diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6e381b..0c6a8b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,6 +156,13 @@ jobs: run: npm run typecheck working-directory: web + # node --test, running the TypeScript directly by stripping the types. Needs + # Node 22.18 or newer for that to be on without a flag; NODE_VERSION is "22", + # which setup-node resolves to the latest 22.x. + - name: Test + run: npm test + working-directory: web + - name: Build run: npm run build working-directory: web diff --git a/.gitignore b/.gitignore index 9eb5de7..b8ad458 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,16 @@ config.local.yaml /web/node_modules/ /web/dist/ /web/.vite/ +# Where `vite build --ssr reader-check.tsx` and `search-check.tsx` put their +# bundles. Both render a screen to HTML on the command line, which is what a +# headless browser is then pointed at to take a screenshot of an authenticated page. +/web/.reader-check/ +/web/.search-check/ +# Where `vite build --config folio-browser-check.vite.ts` puts its bundle. Unlike +# the two above this one is a whole page a browser opens directly, with a real +# document's conversion JSON compiled into it — which is the other reason it is +# never committed. +/web/.folio-check*/ *.tsbuildinfo # go @@ -38,6 +48,11 @@ coverage.html /tmp/ /scratch/ +# Agent worktrees. Root-anchored for the same reason as /manualbox above: these +# are checkouts of this repository, and `git add -A` will otherwise stage one as +# an embedded repository. +/.claude/worktrees/ + # Vite builds the SPA into internal/frontend/dist/app so it can be go:embed-ed. # It writes into the app/ subdirectory rather than dist/ itself because Vite's # emptyOutDir wipes its output directory on every build — which silently deleted diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b96b7e8 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,464 @@ +# manualbox — working notes for Claude + +Self-hosted household manual and maintenance manager. Go 1.25 + React 19, single +binary with the SPA embedded, SQLite, no external services required. Public repo, +MIT. Module `github.com/gordon2/manualbox`. + +## Read before changing an area + +The design docs carry the reasoning **and the measurement** behind each decision. +Read the relevant one first; do not re-derive it. + +| | | +|---|---| +| [CONTRIBUTING.md](CONTRIBUTING.md) | Conventions that have already caused real bugs here | +| [docs/design/ingest.md](docs/design/ingest.md) | The funnel: how a 560-page, 34-language manual is reduced to the pages you actually read, before any model is called | +| [docs/design/layouts.md](docs/design/layouts.md) | How a manual is arranged — sequential sections or parallel columns — and the one seam that varies | +| [docs/design/regions.md](docs/design/regions.md) | Storing a language that is part of a page: the contract, what building it settled, and what it still does not solve | +| [docs/design/conversion.md](docs/design/conversion.md) | Turning a manual into readable blocks — reading order, headings, tables from ruled lines, and the five pages nothing can see | +| [docs/design/search.md](docs/design/search.md) | Finding the sentence you need: why the tokeniser is `trigram`, what that costs and what it cannot do, and why the index is kept by triggers | +| [docs/design/language-detection.md](docs/design/language-detection.md) | The five language signals, what each costs and how accurate each is, and why the detector choice is still open | +| [docs/design/providers.md](docs/design/providers.md) | Why a subscription CLI or local model comes before a metered key, and why a CLI adapter must batch a whole document | +| [docs/design/privacy.md](docs/design/privacy.md) | What manualbox holds, ranked by how it actually leaks | +| [docs/design/keys.md](docs/design/keys.md) | Encryption keys: choosing, storing, recovering | + +## Commands + +```sh +make web-install && make build # build the binary with the SPA embedded +make check # test + lint + typecheck — everything CI runs +make sqlc # regenerate DB code after editing queries or migrations +./bin/manualbox doctor # what is configured, which optional tools are present +./bin/manualbox serve # http://localhost:7745 +``` + +`poppler` and `tesseract` are optional at runtime; features needing them report +why they are unavailable instead of failing. Install them to work on the document +pipeline (`brew install poppler tesseract`). + +Fixture-backed tests need `MANUALBOX_TEST_FIXTURES=1`; they fetch a real 15 MB +manual on demand and skip without it, so the default suite is hermetic and offline. + +## How this project expects to be worked on + +**Measure, don't estimate.** Every number in the design docs came from running +something, and several contradicted the first guess. Before asserting a cost, a +size, or a behaviour, run something and quote the result. + +**Verify where the user will see it.** A clean clone for anything touching the +build or `go:embed`; a real browser for UI; server or container logs for config. +Read structured results, not scrolled log tails. + +**After fixing a bug, revert the fix and confirm the test fails.** This has caught +two worthless tests and one bug that was reported as covered but had no test. + +**Say plainly what was deliberately not done, and why.** Leaving it implicit reads +as a claim that it was done. + +**Prefer delegating bulk implementation** to subagents once a contract is stable, +and keep the main thread for framing, design decisions, integration and +verification. + +## Architecture + +Packages under `internal/`: `config` `id` `db` `store` `jobs` `auth` `api` +`frontend` `fixture` `keyring` `extern` `logging` `doc` `registry` `ingest` +`testpdf`, plus `web/` (React 19 + Vite + Tailwind v4, embedded via `go:embed`). + +- `store` — content-addressed blob store. Originals are immutable, mode 0400, and + the filename **is** the SHA-256. +- `jobs` — SQLite-backed queue with leases, so a killed worker's job is reclaimed. +- `doc` — reads a document and reports facts about it. Knows nothing about + databases and calls nothing remote. +- `registry` — the inventory: locations, devices, documents. +- `ingest` — runs the pipeline as background work and answers the pre-flight gate. +- `testpdf` — generates small valid PDFs in memory for tests, because no PDF may + be committed. + +## Conventions that bite + +These are the ones that have already caused bugs. CONTRIBUTING.md has the full +list and the story behind each. + +- **`db.Read()` for queries, `db.Write()` for statements that modify.** The writer + is capped at one connection; that is what prevents intermittent "database is + locked". +- **Timestamps are integer milliseconds**, converted only through `internal/db`. +- **Wrap SQLite aggregates in `CAST(... AS INTEGER)`** or sqlc emits `interface{}` + and every caller pays for a type assertion. +- **Job handlers must be idempotent.** A worker can die after doing the work but + before recording success. Derived tables use composite natural keys and upserts + so a second run converges instead of duplicating. +- **Count runes, not bytes**, wherever text size matters. Half of a real manual is + Cyrillic, Greek, Hebrew, Arabic or CJK, where bytes run a third higher. +- **Strip Unicode format characters before matching text.** A right-to-left page + wraps Latin furniture in bidi controls, so a tab reading `HE` is really + `RLE LRE H E PDF PDF`. Missing this silently loses whole sections. +- **`gocritic` rejects ranging over large structs by value.** Use + `for i := range pages` and take a pointer. +- **Anchor `.gitignore` patterns** — an unanchored `manualbox` once matched + `cmd/manualbox/`. +- **Look at UI changes in a browser.** Three real bugs shipped past a green + typecheck. + +## Never commit + +- **Real documents or photos.** Manuals are copyrighted and manualbox's own + principle is not to redistribute them. Fixtures are manifests describing where + to fetch a document; tests that need a PDF generate one with `internal/testpdf`. + CI rejects any committed `.pdf`, `.jpg`, `.jpeg` or `.heic`. +- **Anyone's personal data.** Tests use `example.com` (RFC 2606) and documentation + IP ranges (RFC 5737) — never a real address, which is how a shared project + starts reading as one person's private inventory. +- **Absolute paths from a developer's machine** — they carry an OS username. +- Databases, `data/`, or `.env`. + +CI enforces these with a `hygiene` job. It is a grep, not a guarantee. + +## Current state + +M0 and most of M1's pipeline are done: registry, upload, the free probe that reports +what a document contains and stops at the gate, conversion behind that gate, and +full-text search over what it produced. The reader and export are still to come — +see the roadmap in [README.md](README.md). + +**Regions are computed and stored.** A page can hold several languages, so the unit +of the language map is a region rather than a page: `internal/doc/runs.go` reads +where text sits with `pdftohtml`, `regions.go` divides a page on language, and +`doc_regions` persists it. Verified on both real manuals — the parallel-columns one +stores all five of its languages across its columns, the sequential one stores +exactly one whole-page region per page and its 34-section map is unchanged. + +**The gate answers from regions, and characters lead.** `ingest.Gate` reads +`registry.Regions` where a document has them and falls back to the per-page runs +where it does not, so the parallel-columns manual now reports its five languages +with 47,641 characters of German — 20% of the document's text, on 26 of 68 pages it +shares with four other languages — instead of "68 pages, but no language could be +identified". The sequential manual reports exactly what it did, plus the new fields. +`Gate.UnlabelledPages` and `CostEstimate.Chars` were declared and never assigned; +both are now derived from stored rows. + +**The gate screen leads with characters too.** `web/src/screens/DeviceDetail.tsx` +reports each language as its character count and its share of the document's text, +with the pages underneath as a locator, and words those pages by `sharesPages` — +"appears on 26 pages, sharing each with other languages" against "pages 23–38, all +its own". The stat row and the import button count characters rather than pages, for +the same reason. A language under 1% of the text keeps a decimal and loses its +emphasis rather than being filtered, which is what the 289 characters of Finnish in +the columns manual need. + +**The gate has a second door, and conversion runs behind it.** +`POST /documents/{id}/approve` is `decline`'s opposite: it moves the document to +`converting` and queues `doc.convert`, which re-runs `doc.Analyze`, calls +`doc.Convert` for the household's configured languages, and stores the result with +`registry.SaveConversion` — the target state travelling as a parameter so `ready` +lands in the same transaction as the blocks that justify it. There is no language +argument anywhere on that path: the gate showed a specific scope, and approving must +mean that scope. `GET /documents/{id}/conversion?lang=de` serves the blocks and +figures, `GET /documents/{id}/figures/{sha256}` the PNG bytes; `/content` still serves +the original, unchanged. Measured through the API when this was written: the column +manual's German was 431 content blocks and 53 figures, the sequential manual's Russian +431 content blocks and 65 figures over pages 517-538. + +**Two of those three numbers are now stale, and the way they went stale is the lesson.** +Both 431s were superseded by the two-strip reading of a page with no second column, and +they did not move together: German became **460** and Russian **449 content plus 58 +furniture**. The figure counts still hold. Anyone quoting a block total from this file +must re-measure it first — `TestOptingOutIsTodaysConversionExactly` pins Russian's, and +`431` appearing twice for two different documents is exactly the "a total needs its +sequence beside it" trap recorded further down. + +**And 449 is stale too, which is the third entry in that sequence.** Russian content is +**404**, having been 409 when a figure's callout labels first left the block flow, with +the 95 that left still in `Conversion.Blocks` as `Callout` blocks and still counted by +coverage. Read the sequence 431 → 449 → 409 → 404, not any one value. + +**A section title is served once, where the section starts.** The furniture pass has a +third clause: the page's first printed line is a running head when the page before it +in the same language section printed the identical line, so the first page of each +consecutive run keeps its title and every page after it loses one. That dissolves a +blocker the design doc had recorded as measured-and-refused — separating a running head +from a repeated heading by the occupancy of its height is 0.77 against 0.63 with one +document on each side — because both of its options were wrong: remove them all and the +sequential manual loses its titles, keep them all and every page reprints one. 61 claims +on the column manual over 20 titles, 184 on the sequential over 77, and **0 of the 245 +has no surviving content copy**, which is the invariant that replaces a list of 97 +strings in 39 languages. Two recorded claims turned out to be wrong and are corrected in +[conversion.md](docs/design/conversion.md): the column manual **does** have a running +head (its grey banner's chapter name), and sequential page 24's pinned "one heading and +12 list items" was itself the defect — page 23 starts that section and page 24 only +reprints its title. + +**A contents page reads as a list of entries.** The columns manual's `Оглавление` was +one run-together paragraph of dot leaders; each printed line is now its own block, +drawn as a title, a leader rule and the page the paper prints. The signal is a dot +leader of 8+ plus a page reference, and it has a rare thing under it — a real gap: +over both manuals every dot run is 3, 3, 3, 4 then 34 to 91. It is **not** a sixth +`BlockKind`, because that reaches a CHECK on `doc_blocks` and widening it costs a +rebuild of the table the FTS index is external-content over; the note carries the fact +instead. + +**And the page number is a link.** The printed folio maps onto a PDF page by one +constant per document, derived from `doc_pages.printed_folio` rather than stored — no +migration, one grouped scan per conversion response. It is the **mode** of +`page_no - printed_folio`, the same estimator and the same reason as `columnPitch`'s +line gaps: the sequential manual is **6 on 552 of its 558** folio-bearing pages and the +columns manual **0 on 65 of 67**, with the runner-up covering exactly one page in each, +and one back cover misread as folio 2735 would put a mean 40 pages out. The mode must +hold **0.6** of those pages or no offset is served at all — had the sequential manual's +34 sections each restarted at 1, the best offset would have held 22 of 553 pages, 4.0%. +`Conversion.folioOffset` is **absent**, never 0, when there is no answer: the columns +manual's real offset IS 0 and the two must not be confusable. An entry stays plain text +when no offset was served, when the line has no number, or when the target is not a +page this language's conversion holds; a range links to its first page. Verified in +Chrome: 17 of 17 German entries link, `Fehlerbehebung 57` scrolls to page 57, and +withholding the offset returns all 17 to plain text. The brief's expectation that a +German entry could point at a Russian page is **refuted** — each language's contents +page prints its own folios. + +**A page can have two columns and no second column, and reading order now says so.** +**20 blocks on 6 of the sequential manual's 22 Russian pages** crossed the gutter, and +page 530 read `"Мешок для сбора пыли Основная щетка"`, two section banners spliced. The +cause is not the projection but the two gates a **`Column`** has to pass, which are +right for a published fact language attribution reads and wrong for reading order — +page 530's right column holds **6 runs** against `minColumnRuns = 8`, and at 17 usable +runs no x of that page is crossed by more than the **4** `maxGutterCrossings` allows, so +the whole right-hand half reads as one gutter. `readingStrips` is the same projection +with both bounds at their limit, used only where `DetectColumns` has already declined; +`DetectColumns` itself does not move, so no region and no language attribution does. + +A **threshold on the gap was measured first and refused**: over the pages the detector +does split, within-column gaps reach 22.1 times the line's font size and across-gutter +gaps start at 0.0, and there is no gap to put a number in. Two things came out of this +sideways — a right-to-left region's columns were being read left to right on 16 Hebrew +and Arabic pages, and a table whose drawn box overhangs its own words fell to the banner +band and was read before the page's title. Measured with every language converted: +`reading-order` findings **38 → 24** on the sequential manual and its glued-word count +6 → 5, blocks 16,097 → 16,132 over 28 pages and 2,345 → 2,407 over 7, **no word gained +or lost on any page**. The column manual's one new finding is the check's shape, not a +defect, and is explained where it is pinned. Two shapes still weld and neither is two +printed columns — a diagram callout sitting in a gutter reads with the banner (page +521), and the unruled interval grid is the old class (page 528). See +[conversion.md](docs/design/conversion.md). + +**The pages no language owns are offered at the gate, and that is how `A-1` became +reachable.** The funnel converts the pages a household's languages occupy, so anything +outside every language was never converted — including the sequential manual's PDF page +5, an exploded parts diagram that **31 places in its content pages point at**. The gate +now also offers those pages as one opt-in scope. Measured on both manuals, and the second +row is why the feature nearly stopped: + +| | neutral pages | chars | figures | what they are | +|---|---|---|---|---| +| sequential | **7** | 1,639 | **61** | cover, 3 contents pages, **2 diagram plates**, colophon | +| columns | **2** | 10,782 | **0** | a print code, and service addresses in 12 languages | + +**Characters rank those two in exactly the wrong order** — the junk set has seven times +the text — so the offer leads with the picture count, and the probe measures it: +`doc.countNeutralInk` reads ink for the unowned pages only, 7 spawns and 2, bounded at 32 +because hundreds of them is a language map that did not work. `doc_pages.figures` is +**nullable and nil is not 0**, `folioOffset`'s rule again: a 0 default would call a +diagram plate empty. A partial census is withheld rather than summed, because an +understated count is what makes a user decline the pages worth taking. + +These pages contribute **pictures and no text** — their regions are unnamed, so +`RegionsBlocks` skips them while `attribute` hands every picture to every language in +scope. Verified in Chrome on the real manual: German went from **16 pages and 0 figures +to 23 and 61** with its 449 content and 37 furniture blocks *identical*, and a German +reader now has `Abb. A-1` on page 27 and the 31 drawings it names on page 5. Russian +65 → 126 figures, 449/58 blocks unchanged. + +**The crop is the band the page prints, and the label text is its description.** This +**reverses** the two designs before it, and the sequence is the argument: grow the crop +onto the labels, then carry each label as text with a position and let the reader draw +it, then crop the band and stop re-laying out a page the paper had already laid out. +Drawing labels as positioned elements produced four defects — collisions on page 522, +stranded siblings, orphaned wrapped tails, and per-figure placement blind to the +neighbour — and every one of them existed only because the reader was rebuilding an +arrangement that is already correct in the source. + +`doc.labelBand` is the drawing unioned with **every run `claimLabels` reaches, gated or +not**, taken whole, with the top and bottom moved out so no line of type is cut across. +The gate stays exactly where it was and now decides only the **alt text**, because the +two questions have opposite failure modes: a paragraph printed inside a picture that +already prints it is untidy, and a paragraph *described* to a screen reader as a +diagram's callout is a lie. + +**The page band was the first choice and the measurement refused it.** A band prints +prose the block flow also emits, so a few lines arrive twice; +`TestABandCropDuplicatesThisMuchProse` counts it in runes over both whole documents: + +| crop rule | columns manual | sequential manual | +|---|---|---| +| the drawing alone, as it was | 773 — 1.0% | 51 — 0.3% | +| **drawing + every claim — SHIPPED** | **1,481 — 1.8%** | **1,714 — 9.8%** | +| the page's full width | 67,662 — **84.0%** | 11,377 — **65.4%** | + +**59,618 of the full band's 67,662 runes are a neighbouring column's text**, which on +the columns manual is a different language — the one failure `attribute` says the funnel +may not have. That manual carries **0 labels**, all nine of its claims being false, so it +would pay the whole cost of a feature it cannot use. + +It fixes a defect this file recorded as unfixable. `Кнопка сброса`, `Индикатор Wi-Fi` +and `Датчик края` are printed by the crop — **45 of the sequential manual's 50 refused +claims are** — and both refusals recorded below were about *drawing* them, not about +cropping. + +The fourth name is a **correction, and the user photographed it**: `Датчики перепада +высоты` is not a refused claim of figure 2, it is claimed by **nothing**, so the leaders +in that drawing's top edge end in empty paper. It is **40.44** units below figure 0 +against `labelCorridor` = 40; and from figure 2 above it is within reach, with **a real +3.3×3.3 terminator at (665.3, 339.3)** that `labelAlign` refuses. That refusal is an +**asymmetry, not a threshold**: `labelAlign` is 4 units on both axes, a one-line label is +13–14 units tall and this one is **140 units wide**, so ±4 of its midpoint is 6% of it. +The fix is to ask that the mark fall inside the run's own extent on that axis rather than +near its middle — unbuilt, because it changes `claimLabels` and every count in this file +is expressed in that. See [conversion.md](docs/design/conversion.md). + +**A crop wholly inside another is removed.** Two drawings that reach the same run can +end up with one band inside the other — page 529's figure 7 is figure 4's band with the +top cut off — and serving both shows the picture and then its own lower half. +`absorbNested` drops the contained crop and gives its labels to the survivor: **6 on the +sequential manual, 0 on the columns one**. It is **not** the refused plate merge, which +unions two want boxes and invents a rectangle larger than either; this invents nothing, +the surviving crop being one that already existed unchanged. + +Measured with `manualbox verify` over the sequential manual in all 34 languages, before +and after: blocks **14,749 (1,289 furniture, 165 callout) identical**, labels carried +**166 identical**, median coverage **0.996** both times, `reading-order` **23** both +times, figures **134 → 128** (the absorbed crops), and the one check that moved is +**`figure-clipped` 25 → 14**. Total findings 287 → 276. `Figure.Labels` is `[]string`; the position and side left the type, +the schema (`00009`), the JSON and the reader. + +One more cost is not new but is newly visible: **a wrapped label reads as several +labels**, because one entry is one printed line — page 521's `Вентиляционное отверстие +системы автоопорожнения` is three entries in the alt text. The paper's own arrangement +used to hide it. Joining the chain is not built because the chain is not recorded. + +The costs are stated rather than managed: the band rule produces **35 overlapping crop +pairs** on the sequential manual and runs cut by a crop edge go **16 → 48**; what a +reader is served holds **26** pairs, 11 of them on served pages, so page 521's two +crops +each show a fragment of the other's labels; the crop can **print text the alt text does +not name** (page 522's second figure); and the columns manual pays 708 runes on its +page 22. Merging an overlapping pair is refused on the record by +`TestAPlateMergeOnSharedLabelsIsRefused`. See [conversion.md](docs/design/conversion.md). + +**A callout label leaves the block flow, and that survived the reversal.** Russian +content blocks are **404** with **95** `Callout` blocks carrying the text — read the +sequence 431 → 449 → 409 → 404 and not any one value. The reason did not change when +the labels stopped being drawn: what removes a run from the prose is that something +else shows it, and the something else is now the picture itself. A block-level filter +was measured and refused — **66 of 89 labels arrive inside a bigger paragraph**, so +dropping the block deletes its neighbour. `Block.Callout` is a **second flag, not +`Furniture`**, because `checkCoverage` skips furniture (discarded) and must count a +label (relocated). + +**Two things the claim rule still gets wrong, each pinned rather than hidden:** **7 +claims are prose** (page 529's numbered step) and there is **no threshold** — the claims +run 38, 39 runes with a real label at 38 and a Japanese sentence at 39. The Hebrew and +Arabic sections carry **no labels at all**. + +**A label is a unit: all of its lines or none.** The third thing that list used to +carry is fixed, and it was fixed because the user photographed it — `Монтажные отверстия +для` drawn on page 521's underside drawing with `держателя`, `насадки для` and `швабры` +adrift in the prose a screen below. **The tail was never claimed, not dropped after +claiming.** `claimLabels` asks `runBeyond`, whose band — the extent the drawing occupies +on the other axis — is right for a leader pointing out of a drawing and wrong for a wrap +going downward: that label is level with the drawing's foot, so its last three lines sit +8, 20 and 32 units *below* the box. The continuation pass and `continuesLabel`'s +line-sharing scan now ask `runInCorridor`, the same test without the band, which is the +LARGER set — so aloneness gets stricter, not weaker. Read the sequences **23 → 34 → 37** +on page 521, **268 → 276** carried, **159 → 167** on served pages, and Russian content +blocks **449 → 409 → 404** with callouts **89 → 95**. The columns manual carries **0 +before and 0 after**, all nine false claims still refused. Every one of the 8 gained is a +later line of a label already carried, and +`TestNoLabelIsCarriedWithoutItsLaterLines` pins the invariant rather than the total: no +run left in the flow continues a carried label, with 0 cut by `labelCorridor` and 2 by +`minWrapRunes` — the digits `4` and `2` on the plate pages, asserted by name. + +**The stranded siblings are left in the prose and printed by the crop, and the two +alternatives are still refused.** `labelExtent` is load-bearing for the alt text and for +nothing else now. Drawing page 521 figure 0's right side would describe three bullet +lines as labels, because a continuation has no direction and they chain upward from +`Индикатор Wi-Fi`; making the gate per label chain instead of per side — implemented and +measured — carries **8 lines of German body prose** on the columns manual's page 22. +Captioning them fails identically, because it needs the refused set to be labels and 9 +of 9 on the columns manual are not. A refused claim is a claim of **unknown kind**, and +no signal here separates the real from the false: the columns manual's two false claims +are both terminator claims, and length is the 38-against-39 non-gap already on the +record. What changed is only the cost — the band prints them, so a refusal now costs a +missing sentence in the alt text rather than a label the reader never sees. + +**`approve` grew one boolean and the promise is argued, not waived.** It still takes no +language argument. A yes/no to an offer the gate displayed is admitted because the caller +cannot name a page — the set is recomputed from stored regions — and because the choice +is per document, not per household. It is stored on `documents.include_neutral_pages` and +read back by the job, so `ConvertPayload` stays document-only and its dedupe key stays +sound. `internal/verify` deliberately does **not** take this scope, so those plates are +converted-but-unchecked. See [conversion.md](docs/design/conversion.md). + +**The blocks are indexed, and `GET /api/v1/search?q=` answers which manual says X.** +FTS5 over `doc_blocks` with `content='doc_blocks'`, kept correct by three triggers +because the third path that changes that table — `documents ON DELETE CASCADE` — +runs no Go at all. A hit names the document, the device, the page and the language, +and carries the block's natural key so it can cite the paragraph. + +The tokeniser was the one open question and it is measured, not argued: +**`unicode61` finds nothing in Japanese and nothing in Thai**, because a whole CJK or +Thai run is one token, so the index is `trigram remove_diacritics 1` — 880 KB against +270 KB over the 3,122-block corpus of both manuals. Its named limitation is that no +query under three characters is in the index at all, which is a real hole in Chinese +and Japanese, so those are answered by an `instr` scan instead and the response's +`mode` says which path ran. Verified through the API on both real manuals: German, +Russian, Japanese and Thai all find a real word, and **so does Hebrew, typed +forwards** — `מדריך` finds 5 blocks forwards and 0 backwards, the exact inverse of +what it did, because `internal/doc/bidi.go` stores right-to-left text in logical +order and the **region's language** decides direction. A fixture test pins both +numbers; the claim had lived in prose with nothing under it. +The whole measurement is [docs/design/search.md](docs/design/search.md). + +**Right-to-left text is no longer stored reversed, and that is asserted rather than +believed.** `internal/verify`'s `right-to-left-reversed` check reports **0 pages** +where it reported 32, and `TestNoTextIsStoredReversed` fails on a single word that is +absent from `pdftotext` and present in it backwards. Getting there needed the check +sharpened first: it fired on any right-to-left page with any absent word, which was +the same question as "is this page Hebrew" only while every Hebrew page was broken. +It now needs evidence of a reversal — see `verify.minReversibleWords`, which records +why that is a count and not a share. + +**A zero there means no word is spelled backwards. It does not mean the words are in +the right order.** Four defects were found in `bidi.go`, all by measurement, and the +check whose name describes them caught **exactly the one that was a reversal** — the +direction rule that left six lines unrepaired, which it named to the page and the word +once it was sharpened. It was structurally blind to the other three, which were +reorderings: the word check compares set membership per page, so word order is outside it +by construction. Page 204's URL arrived with its seventeen runs reversed; page 211's +Arabic list marker `1.` arrived as `. 1`, merging six printed list items into one +paragraph on each of six pages; page 204's laser standard arrived as `EN1:2014/ 60825-`. +Two surfaced sideways through the joins check reacting to a side effect, and **one was not +reported at all** — caught only because a pinned block count moved by 43. All four are +fixed. `internal/verify` asks the order question of blocks and nothing asks it of words; +the design of the check that would, and the reason it is not built, is in +[docs/design/conversion.md](docs/design/conversion.md). + +**Pin the counts you cannot yet explain.** That block-count pin caught the regression +nothing else saw, and then confirmed the repair page for page. Its history also shows why +a total needs its sequence beside it: 16,055 appears twice and means opposite things. + +Deliberately not built yet, each for a stated reason: + +- **The printed-index parser cannot read a contents page laid out in columns.** It + returns junk for the Thomas manual, which costs 26 columns of printed-tag + attribution and once labelled two pages `fax`. See language-detection.md; a test + pins the current reading so the gap stays visible. +- AI provider adapters (the `Kind` values are accepted and fail at first use with a + clear message), the statistical language detector (see language-detection.md), + serial numbers and purchase prices in the schema (they need the keyring first), + and login throttling (`TODO(M1)` in `internal/auth/auth.go`). + +**One trap worth knowing before you touch `internal/db/queries/`:** those files must +stay pure ASCII. sqlc v1.31.1 corrupts generated statements when a query file +contains a non-ASCII character, sometimes silently — valid Go, invalid SQL, failing +at PREPARE time in a background job. Two tests guard it; the header of +`queries/docregions.sql` has the measurement. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7a40fd..5452ee4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,6 +86,9 @@ measurements: | | | |---|---| | [ingest.md](docs/design/ingest.md) | How a 560-page, 34-language manual is reduced to the 16 pages you actually read, before any model is called | +| [layouts.md](docs/design/layouts.md) | Manuals are arranged differently — sequential sections or parallel columns — and where the seam between them goes | +| [regions.md](docs/design/regions.md) | Contract for storing a language that occupies part of a page, and what it deliberately leaves unsolved | +| [language-detection.md](docs/design/language-detection.md) | The four signals that say what language a page is in, what each one costs, and why the detector choice is deliberately still open | | [providers.md](docs/design/providers.md) | Why a subscription CLI or a local model comes before a metered API key, and why a CLI adapter must batch a whole document | | [privacy.md](docs/design/privacy.md) | What manualbox holds, ranked by how it actually leaks | | [keys.md](docs/design/keys.md) | Encryption keys: choosing, storing, and recovering them | diff --git a/Makefile b/Makefile index 9a10b04..7628341 100644 --- a/Makefile +++ b/Makefile @@ -49,10 +49,17 @@ web-build: ## Build the frontend into web/dist web-typecheck: ## Typecheck the frontend cd web && npm run typecheck +# `node --test`, which runs the TypeScript directly by stripping the types. No test +# framework and no new dependency: the alternative was adding a runner to assert +# rules that are a few lines each. +.PHONY: web-test +web-test: ## Run frontend tests + cd web && npm test + ## ---- quality ---- .PHONY: check -check: test lint web-typecheck ## Everything CI runs +check: test lint web-typecheck web-test ## Everything CI runs .PHONY: test test: ## Run Go tests with race detector diff --git a/README.md b/README.md index 0a8142c..b2170dd 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,12 @@ Self-hosted. Free. MIT. Single binary, SQLite, no external services required, and **no API key needed** to get real value out of it. -> ⚠️ Early development. The server runs, you can create an account and sign in, and background -> jobs report live progress — but **you cannot add devices or manuals yet**. That is M1, the next -> milestone. See the [Roadmap](#roadmap). +> ⚠️ Early development. You can create an account, add devices, and upload a manual — manualbox +> reads it locally and tells you what it contains, in which languages, before anything is +> converted or sent anywhere. Approve it and the pages you read are converted to blocks, and +> `GET /api/v1/search?q=` finds the sentence you need across every manual in the house. +> **It stops there for now:** the reader screen, the search screen and export are still to +> come. See the [Roadmap](#roadmap). ## Try it @@ -75,7 +78,7 @@ Then it gets out of the way: notifications where you already look, and a calenda | | | |---|---| | **M0** ✅ | Skeleton: config, SQLite + migrations, blob store, job queue, auth, API, frontend shell, Docker, CI | -| **M1** | Registry, document pipeline (convert → language-segment → index), reader, full-text search, **export** — see [ingest design](docs/design/ingest.md) | +| **M1** | Registry ✅, document probe and language map ✅, conversion ✅, full-text search ✅, then the reader and **export** — see [ingest](docs/design/ingest.md), [conversion](docs/design/conversion.md) and [search](docs/design/search.md) | | **M2** | Maintenance: schedules, battery charge cycles, service log, notifications, ICS calendar feed | | **M3** | Translation: per-block, glossary, translation memory, side-by-side, post-editing | | **M4** | Extraction: maintenance plans with citations, printable per-device cheat sheets, error-code lookup | @@ -101,8 +104,10 @@ Optional external binaries, used when present and degraded gracefully when not ( | `tesseract` | OCR for scans and photos | `brew install tesseract tesseract-lang` | Design decisions, with the measurements behind them, are in `docs/design/`: -[ingest](docs/design/ingest.md) · [providers](docs/design/providers.md) · -[privacy](docs/design/privacy.md) · [keys](docs/design/keys.md). +[ingest](docs/design/ingest.md) · [layouts](docs/design/layouts.md) · +[language detection](docs/design/language-detection.md) · +[providers](docs/design/providers.md) · [privacy](docs/design/privacy.md) · +[keys](docs/design/keys.md). Conventions and the things that have already caused bugs here: [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/cmd/manualbox/main.go b/cmd/manualbox/main.go index 2340ec5..36bab70 100644 --- a/cmd/manualbox/main.go +++ b/cmd/manualbox/main.go @@ -24,8 +24,10 @@ import ( "github.com/gordon2/manualbox/internal/config" "github.com/gordon2/manualbox/internal/db" "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/ingest" "github.com/gordon2/manualbox/internal/jobs" "github.com/gordon2/manualbox/internal/logging" + "github.com/gordon2/manualbox/internal/registry" "github.com/gordon2/manualbox/internal/store" ) @@ -68,6 +70,8 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { return cmdServe(ctx, rest, stdout, stderr) case "doctor": return cmdDoctor(ctx, rest, stdout) + case "verify": + return cmdVerify(ctx, rest, stdout, stderr) case "version", "--version", "-v": fmt.Fprintf(stdout, "manualbox %s (%s)\n", version, commit) return nil @@ -89,6 +93,7 @@ Usage: Commands: serve Run the web server and background workers doctor Report configuration and which optional tools are available + verify Convert a PDF and report what is wrong with the conversion version Print the version help Show this help @@ -98,6 +103,10 @@ Flags (serve, doctor): Flags (doctor): -redact Replace your home directory with ~, for pasting into a bug report +Flags (verify): + -limit How many findings of each kind to print (default 20) + -all Print every finding + Configuration comes from defaults, then the config file, then MANUALBOX_* environment variables. Run "manualbox doctor" to see what was resolved. `) @@ -197,19 +206,31 @@ func cmdServe(ctx context.Context, args []string, stdout, stderr io.Writer) erro queue := jobs.NewQueue(database, log) defer queue.Broker().Close() + registryService := registry.New(database, registry.Options{Logger: log}) + ingestService := ingest.New(ingest.Deps{ + Config: cfg, + Registry: registryService, + Store: blobs, + Jobs: queue, + Logger: log, + }) + pool := jobs.NewPool(queue, cfg.Jobs, log) - // M1 registers the real conversion, OCR, translation, and extraction - // handlers here. Until then the pool runs with none, and a job of an unknown - // kind fails with a clear message rather than hanging. + // The document pipeline's handlers. Conversion, OCR, translation, and + // extraction register here as they land; a job of an unknown kind fails with a + // clear message rather than hanging. + ingestService.Register(pool) server := api.New(api.Deps{ - Config: cfg, - DB: database, - Store: blobs, - Auth: authService, - Jobs: queue, - Logger: log, - Version: version, + Config: cfg, + DB: database, + Store: blobs, + Auth: authService, + Jobs: queue, + Registry: registryService, + Ingest: ingestService, + Logger: log, + Version: version, }) httpServer := &http.Server{ diff --git a/cmd/manualbox/verify.go b/cmd/manualbox/verify.go new file mode 100644 index 0000000..5f1f573 --- /dev/null +++ b/cmd/manualbox/verify.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "sort" + "text/tabwriter" + "time" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/verify" +) + +// cmdVerify converts a PDF and reports what is wrong with the conversion. +// +// It exists to be run on a manual the fixtures do not contain, which is where +// this check earns its keep: the two measured documents are the two whose defects +// are already written down. Nothing is stored and nothing is uploaded — the same +// stance doctor takes — so it is safe to point at a file and read the answer. +// +// The document is converted for EVERY language it holds rather than for the +// household's, because coverage is measured against all the text on a page and a +// page of a parallel-columns manual holds five languages. See [verify.Check]. +func cmdVerify(ctx context.Context, args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("verify", flag.ContinueOnError) + fs.SetOutput(stderr) + limit := fs.Int("limit", 20, "how many findings of each kind to print") + all := fs.Bool("all", false, "print every finding rather than the first few of each kind") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errors.New("usage: manualbox verify [-limit n] [-all] ") + } + path := fs.Arg(0) + + start := time.Now() + res, err := doc.Analyze(ctx, path) + if err != nil { + return err + } + langs := res.Languages() + names := make([]string, 0, len(langs)) + for i := range langs { + names = append(names, langs[i].Lang) + } + fmt.Fprintf(stdout, "%d pages, %d language(s): %s\nprobed in %v\n", + res.Info.Pages, len(langs), join(names), time.Since(start).Round(time.Millisecond)) + + start = time.Now() + conv, err := verify.ConvertAll(ctx, path, res) + if err != nil { + return err + } + fmt.Fprintf(stdout, "%s\nconverted in %v\n", conv.Summary(), + time.Since(start).Round(time.Millisecond)) + for _, n := range conv.Notes { + fmt.Fprintf(stdout, " note: %s\n", n) + } + + start = time.Now() + rep, err := verify.Check(ctx, path, conv) + if err != nil { + return err + } + fmt.Fprintf(stdout, "\n%s\nchecked in %v\n\n", rep.Summary(), + time.Since(start).Round(time.Millisecond)) + for _, n := range rep.Notes { + fmt.Fprintf(stdout, " note: %s\n", n) + } + + tw := tabwriter.NewWriter(stdout, 0, 8, 2, ' ', 0) + fmt.Fprintf(tw, " kind\tfindings\tpages\n") + kinds := rep.Kinds() + for _, k := range verify.AllKinds { + if kinds[k] == 0 { + continue + } + fmt.Fprintf(tw, " %s\t%d\t%d\n", k, kinds[k], rep.PagesFlagged(k)) + } + tw.Flush() + + // The worst pages by coverage, whether or not they were reported: the + // measurement is what a person reads this for, and a document whose worst page + // scores 0.99 has been told something by that number. + cov := make([]verify.PageCoverage, len(rep.Coverage)) + copy(cov, rep.Coverage) + sort.Slice(cov, func(a, b int) bool { return cov[a].Ratio < cov[b].Ratio }) + fmt.Fprintf(stdout, "\nmedian coverage %.3f; least covered pages:\n", rep.MedianCoverage()) + for i := 0; i < len(cov) && i < 5; i++ { + fmt.Fprintf(stdout, " page %d: %.3f (%d block characters against %d from pdftotext)\n", + cov[i].Page, cov[i].Ratio, cov[i].Blocks, cov[i].Text) + } + + shown := make(map[verify.Kind]int, len(kinds)) + fmt.Fprintln(stdout) + for i := range rep.Findings { + f := &rep.Findings[i] + if !*all { + if shown[f.Kind] >= *limit { + continue + } + shown[f.Kind]++ + } + fmt.Fprintf(stdout, "%s: %s\n", f.Kind, f.Detail) + if f.Sample != "" { + fmt.Fprintf(stdout, " %s\n", f.Sample) + } + } + if !*all { + for k, n := range kinds { + if n > shown[k] { + fmt.Fprintf(stdout, "… %d more %s finding(s); -all prints them\n", n-shown[k], k) + } + } + } + return nil +} diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 3d2f451..2bd3aa6 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -35,8 +35,74 @@ tags: - name: setup - name: auth - name: jobs + - name: documents + - name: search paths: + /search: + get: + tags: [search] + summary: Which manual says X, and where + description: | + Full-text search over every converted manual in the household. A hit names + the document, the device, the page and the language, and carries enough text + to recognise — "page 47 of something" does not solve the problem this + endpoint exists for. + + A GET with `?q=`, so a search is a URL: linkable, bookmarkable, and in the + browser's history where a user expects it. + + **`q` is text, never a query language.** SQLite FTS5 has an expression syntax + — bare `AND`, `OR`, `NOT`, `NEAR`, colons, prefix stars — and passing a + user's words to it would make an ordinary query a syntax error and silently + reinterpret another. Each word is quoted as a phrase and the phrases are + ANDed, so two words mean a block containing both. + + **Two modes, and the response says which one answered.** The index is a + trigram index, chosen because the corpus is not English: Chinese, Japanese + and Thai do not put spaces between words, and a word-boundary tokeniser + finds nothing at all inside them. Its cost is that no token shorter than + three characters exists in it, so a query where any word is shorter than + that is answered by scanning instead — `mode` is then `substring`, and + `bm25` and `score` are 0 on every hit because there is no index term to + weigh. See `internal/db/migrations/00006_block_search.sql` for the + measurements. + + A query matching nothing returns `indexed`, which is how many blocks exist + to search at all: "no manual says that" and "nothing has been converted yet" + are otherwise the same empty list. + parameters: + - name: q + in: query + required: true + description: What to look for, as typed. Whitespace-only is refused rather than answered. + schema: { type: string, examples: [Saugkraft] } + - name: documentId + in: query + required: false + description: | + Narrow to one manual, which is what a reader already inside a document + asks. Omitted, the search spans every document, because a household + looking for the descaling interval does not know which manual to open. + + An unknown id is a search of nothing rather than a 404: this parameter + scopes a search, and answering 404 would turn it into a way to test + whether an id exists. + schema: { type: string, examples: [doc_01JQ8ZK3M4N5P6R7S8T9V0W1X2] } + - name: limit + in: query + required: false + description: 1 to 100. Defaults to 25. + schema: { type: integer, minimum: 1, maximum: 100, default: 25 } + responses: + "200": + description: The hits + content: + application/json: + schema: { $ref: "#/components/schemas/SearchResults" } + "400": { $ref: "#/components/responses/Error" } + "401": { $ref: "#/components/responses/Error" } + /health: get: tags: [system] @@ -294,6 +360,147 @@ paths: description: Already finished (`not_cancellable`) $ref: "#/components/responses/Error" + /documents/{documentID}/gate: + parameters: + - name: documentID + in: path + required: true + schema: { type: string, examples: [doc_01JQ8ZK3M4N5P6R7S8T9V0W1X2] } + get: + tags: [documents] + summary: The pre-flight question, before anything is spent + description: | + What manualbox is holding, what it would process, and what that would + cost. Built entirely from stored probe results, so it survives a restart, + costs nothing to render, and never re-reads the document. + + The rest of the document and device surface is not written up here yet; + this path is documented because the gate is the screen a user decides on + and its shape is what the SPA reads. + responses: + "200": + description: The gate + content: + application/json: + schema: { $ref: "#/components/schemas/Gate" } + "401": { $ref: "#/components/responses/Error" } + "404": { $ref: "#/components/responses/Error" } + + /documents/{documentID}/approve: + parameters: + - name: documentID + in: path + required: true + schema: { type: string } + post: + tags: [documents] + summary: Authorise the conversion the gate described + description: | + The moment the user agrees to spend, and the opposite of `decline`. The + document moves to `converting` and a `doc.convert` job is queued; progress + arrives on the existing job event stream. + + **There is no request body, and there must not be one.** The scope + converted is the household's configured reading languages, which is what the + gate showed the user. Accepting a language list here would let the thing + approved differ from the thing shown, which is the one promise the funnel + makes. + + Refused with `invalid` when there is nothing honest to convert: the document + has not been read yet, it is password-protected, it has no text layer and so + needs OCR, or none of its languages are ones this household reads. A refusal + does not move the document. + responses: + "202": + description: Queued + content: + application/json: + schema: + type: object + required: [document] + properties: + document: { type: object, description: The document, now in `converting`. } + jobId: { type: string } + "400": { $ref: "#/components/responses/Error" } + "401": { $ref: "#/components/responses/Error" } + "404": { $ref: "#/components/responses/Error" } + + /documents/{documentID}/conversion: + parameters: + - name: documentID + in: path + required: true + schema: { type: string } + get: + tags: [documents] + summary: The readable blocks and the pictures a conversion produced + description: | + Deliberately not served from `/content`, which is the stored original byte + for byte and stays that way — that path is the "own your data" promise at + its most literal. Overloading it would mean either content negotiation, + which is invisible in a URL so the derived view could not be linked, or a + query parameter that silently changes the response from bytes to JSON. + + Figures come back as their own list rather than as blocks of kind `figure`, + because a language-neutral picture has no region and so no natural key. A + reader merges the two by page and vertical position. Each figure's bytes are + at `/documents/{documentID}/figures/{sha256}`. + parameters: + - name: lang + in: query + required: false + description: | + One language, as a BCP-47 tag. Returns that language's blocks together + with its own pictures **and every picture belonging to no language** — a + reader must not lose a diagram because the diagram has no language of + its own. + + Omitting the parameter returns everything stored, which is already only + what the household's scope charged for. Passing it empty is a third, + real question: the content nothing could name, which no other language's + answer contains. + schema: { type: string, examples: [de] } + responses: + "200": + description: The conversion + content: + application/json: + schema: { $ref: "#/components/schemas/Conversion" } + "401": { $ref: "#/components/responses/Error" } + "404": { $ref: "#/components/responses/Error" } + + /documents/{documentID}/figures/{sha256}: + parameters: + - name: documentID + in: path + required: true + schema: { type: string } + - name: sha256 + in: path + required: true + description: The figure's digest, as `Figure.sha256` reports it. + schema: { type: string, pattern: "^[0-9a-f]{64}$" } + get: + tags: [documents] + summary: One rendered figure's PNG + description: | + Addressed by digest because the content is the name: the ETag is exact and + the bytes are immutable for ever. + + The digest is checked against this document's own figures rather than handed + to the blob store directly. The store holds every original anyone has + uploaded, so a route that opened any digest a caller named would serve + another household's manual to whoever could guess a hash. + responses: + "200": + description: The PNG + content: + image/png: + schema: { type: string, format: binary } + "400": { $ref: "#/components/responses/Error" } + "401": { $ref: "#/components/responses/Error" } + "404": { $ref: "#/components/responses/Error" } + components: securitySchemes: sessionCookie: @@ -366,7 +573,7 @@ components: kind: type: string description: What the job does. - examples: [convert, ocr, translate, extract] + examples: [doc.probe, doc.convert] state: { $ref: "#/components/schemas/JobState" } priority: { type: integer, description: Higher runs first. } progress: { type: number, minimum: 0, maximum: 1 } @@ -440,6 +647,334 @@ components: type: object additionalProperties: { $ref: "#/components/schemas/ExternalTool" } + LanguageRun: + type: object + description: | + One language as a stored run of pages: what a signal concluded about a + contiguous stretch of the document, collapsed to one entry per language. + required: [source, code, lang, name, start, end, pages, confidence, conflict] + properties: + source: + type: string + enum: ["", page-tag, index, script, repertoire, detector, reconciled] + description: Which signal named it. Empty where nothing could. + code: + type: string + description: The label the document itself prints, which need not be a valid tag. + examples: [DE, UA, KAZ] + lang: { type: string, description: "`code` normalised to BCP-47, empty when it could not be." } + name: { type: string, description: English display name., examples: [German] } + title: { type: string, description: The section title from the manual's own contents table. } + start: { type: integer, description: First 1-based PDF page. 0 means the language could not be placed. } + end: { type: integer } + pages: { type: integer } + printedPage: { type: integer, description: The start page the printed index claims, often 1-2 off. } + confidence: { type: number, minimum: 0, maximum: 1 } + conflict: { type: boolean, description: The signals disagreed. Shown, never silently resolved. } + note: { type: string } + + GateLanguage: + description: | + One of a document's languages as the gate reports it: the stored run, plus + what only the region map can say about size. + + Characters lead and pages are context. A language occupying one of three + parallel columns on 26 of 68 pages is not 26 pages of reading, and + `sharesPages` is what says so. A language the per-page signals never named + has no run behind it, and then `title`, `printedPage` and `confidence` are + absent or zero because regions store none of them. + allOf: + - $ref: "#/components/schemas/LanguageRun" + - type: object + required: [chars, share, sharesPages] + properties: + chars: + type: integer + description: Runes, not bytes — the same amount of Cyrillic or CJK writing runs about a third more bytes. + examples: [47641] + share: + type: number + minimum: 0 + maximum: 1 + description: "`chars` as a fraction of the document's named text." + sharesPages: + type: boolean + description: | + This language does not have its pages to itself: somewhere it + occupies a box on a page another language also occupies. + + CostEstimate: + type: object + description: | + What the scope would cost. `chars` is measured and always present; + `available` stays false while there is no honest token or money figure to + give, and `reason` says why. + required: [available, chars] + properties: + available: { type: boolean } + chars: { type: integer, description: Characters in scope. The same number as `Gate.scopeChars`. } + reason: { type: string } + + Gate: + type: object + description: | + The pre-flight answer for one document. + + The language map is read from the stored regions where there are regions, + and from the per-page runs otherwise. On a parallel-columns manual a page + holds three languages, so no per-page answer about it can be right; on a + sequential manual the two sources agree. + required: + [documentId, deviceId, kind, state, probed, pages, encrypted, hasTextLayer, + medianChars, chars, household, inScope, other, scopePages, scopeFraction, + scopeChars, scopeCharFraction, conflicts, unlabelledPages, requiresApproval, + maxPagesAuto, cost, summary] + properties: + documentId: { type: string } + deviceId: { type: string } + filename: { type: string } + kind: { type: string, enum: [manual, receipt, warranty, photo, other] } + state: { type: string, enum: [uploaded, probing, awaiting_scope, declined, converting, ready, failed] } + probed: { type: boolean } + pages: { type: integer } + encrypted: { type: boolean } + hasTextLayer: { type: boolean, description: False means this is a scan and reading it needs OCR. } + medianChars: { type: integer } + chars: + type: integer + description: | + The document's named text: the characters of every language something + could name, and the denominator of every `share`. Text nothing could + name is excluded, so a failed signal cannot silently shrink a share. + household: + type: array + items: { type: string } + description: The configured reading languages, echoed back so the UI can explain scope. + inScope: + type: array + items: { $ref: "#/components/schemas/GateLanguage" } + other: + type: array + items: { $ref: "#/components/schemas/GateLanguage" } + description: | + Present but not read by this household. Listed, never discarded: the + original is kept whole, so importing one later is a button rather than + a re-upload. + scopePages: + type: integer + description: | + Distinct pages carrying an in-scope language, not a sum over + languages — on a columns manual a sum reports 133 pages of a 68-page + document. + scopeFraction: { type: number, description: "`scopePages` over `pages`." } + scopeChars: { type: integer } + scopeCharFraction: + type: number + description: | + `scopeChars` over `chars`. The honest measure of how much of a + document a household reads: on the measured columns manual German is + 38% by pages and 20% by characters. + conflicts: + type: integer + description: | + How many runs the signals disagreed about — the document's own contents + table against its pages. A region's disagreement (a column's alphabet + against the page's printed tab) is a different thing and reaches the + user through `conflict` on the language itself. + unlabelledPages: + type: integer + description: | + Content pages carrying text that no signal could name. Front matter and + a back cover are excluded: they legitimately belong to no section. + requiresApproval: { type: boolean } + maxPagesAuto: { type: integer } + cost: { $ref: "#/components/schemas/CostEstimate" } + summary: { type: string, description: One line, characters-led, for a person to read. } + + Block: + type: object + description: | + One piece of readable content, in document order, with the original's page + number kept so a reader can say "page 47" and mean what the paper means. + + Keyed naturally on the document, the page, the region's left edge and the + index within that region, so re-converting converges rather than inserting a + parallel set — and so extraction has stable block IDs to cite. + required: [page, regionX0, index, kind, text, x0, x1, y0, y1, lines, chars] + properties: + page: { type: integer, description: 1-based PDF page. } + regionX0: + type: integer + description: | + The left edge of the language region this block was read from, rounded + because it is part of the key. The block's own box is not rounded, + because nothing keys on it. + index: { type: integer, description: Position within the region, from 0. } + kind: + type: string + enum: [heading, paragraph, list-item, table, figure] + description: | + `figure` is declared and never produced, deliberately: a + language-neutral picture has no region, so a figure block would either + invent a key or collide with a real block's. Pictures come back as their + own list. + level: { type: integer, description: Heading level, 1 for the most prominent. 0 for anything else. } + text: { type: string } + lang: { type: string, description: The region's language, empty where none was established. } + name: { type: string, description: "`lang` for a person to read: German, not de." } + x0: { type: number } + x1: { type: number } + y0: { type: number } + y1: { type: number } + lines: { type: integer } + chars: { type: integer, description: Runes, not bytes. } + note: { type: string, description: Why the block was read the way it was, for inspecting a misreading. } + + Figure: + type: object + description: | + One illustration, rendered to PNG. + + There is no language field, and that is the contract rather than an omission: + a picture belonging to no language belongs to every language, so a reader + scoped to German selects German's own pictures plus every neutral one instead + of selecting pictures by language. + + The pictures in a manual are not the images in the file. `pdfimages` yields + zero illustrations across all 628 pages of both measured fixtures — every one + of them is vector — so a figure is found from what the page draws and its + bytes come from rendering the crop. + required: [page, index, x0, y0, x1, y1, ink, textFraction, dpi, pixelWidth, pixelHeight, sha256] + properties: + page: { type: integer } + index: { type: integer, description: Position in the page's reading order, from 0. Not a document-wide figure number. } + x0: { type: number } + y0: { type: number } + x1: { type: number } + y1: { type: number } + ink: { type: integer, description: How many drawn shapes it holds — the shape guard's evidence, kept rather than reduced to the verdict. } + textFraction: { type: number, description: How much of its area is covered by text — the text guard's evidence, which is what separates a table from a grid of framed illustrations. } + dpi: { type: integer } + pixelWidth: { type: integer, description: Read back out of the PNG rather than computed. } + pixelHeight: { type: integer } + sha256: + type: string + description: | + The blob store's name for the PNG, which is also the PNG's digest. Fetch + the bytes at `/documents/{documentID}/figures/{sha256}`. + + Conversion: + type: object + description: | + A document's converted content: the ordered blocks and the pictures. + required: [documentId, state, blocks, figures] + properties: + documentId: { type: string } + state: + type: string + enum: [uploaded, probing, awaiting_scope, declined, converting, ready, failed] + description: | + Present because no count can distinguish "converted and empty" from "not + converted". A document that has not been through the gate has no blocks, + and that is not the claim that it has no content. + lang: { type: string, description: Echoed back only when the request filtered to one language. } + blocks: + type: array + items: { $ref: "#/components/schemas/Block" } + figures: + type: array + items: { $ref: "#/components/schemas/Figure" } + folioOffset: + type: integer + description: | + How far this document's PDF pages run ahead of the numbers printed on its + paper: the PDF page for a printed page number is `printed + folioOffset`. + One constant for the whole document, derived from the pages that print a + folio by taking the offset a clear majority of them agree on. + + OMITTED when they agree on no such offset, which is a different answer + from zero and must not be read as it: a manual whose page 1 is its cover + really does have offset 0, and a client defaulting the missing field to 0 + would turn every contents entry of an unmappable document into a link to + the wrong page. + lastError: { type: string, description: Why the conversion failed, when `state` is `failed`. } + + SearchResults: + type: object + description: The hits, plus what was actually asked and how it was answered. + required: [query, mode, limit, truncated, hits] + properties: + query: { type: string, description: The text as typed, echoed so a client need not keep its own copy in step. } + mode: + type: string + enum: [index, substring] + description: | + Which path answered. `index` is the FTS5 trigram index with bm25 ranking. + `substring` is the scan that covers a query the index cannot represent — + one where some word is shorter than three characters — and it does not + rank. + limit: { type: integer, description: The limit applied, after clamping to 100. } + truncated: + type: boolean + description: | + The limit cut the list off, which is the difference between "these are + the hits" and "these are the first hits". + hits: + type: array + items: { $ref: "#/components/schemas/SearchHit" } + indexed: + type: integer + description: | + How many blocks exist to search. Present **only** when nothing matched, + because that is when "no manual says that" and "nothing has been + converted yet" are otherwise indistinguishable. + + SearchHit: + type: object + description: | + One match. `page`, `regionX0` and `index` are the block's natural key — the + same citation `Block` carries — so a hit can be deep-linked to the exact + paragraph and still points there after a re-conversion. + required: [documentId, deviceId, deviceName, state, page, regionX0, index, kind, snippet, chars, bm25, score] + properties: + documentId: { type: string } + filename: { type: string, description: What the uploaded file was called. Half of "which manual". } + deviceId: { type: string } + deviceName: { type: string, description: The other half. A household recognises the device, not the id. } + state: + type: string + enum: [uploaded, probing, awaiting_scope, declined, converting, ready, failed] + description: The document's pipeline state, so a hit from a manual that is mid-re-conversion is visible as such. + page: { type: integer, description: 1-based PDF page, the number the paper means. } + regionX0: { type: integer } + index: { type: integer } + kind: + type: string + enum: [heading, paragraph, list-item, table, figure] + level: { type: integer } + lang: { type: string } + name: { type: string, description: "`lang` for a person to read: Japanese, not ja." } + snippet: + type: string + description: | + About 64 characters of the block around the match — enough to recognise. + `chars` is the whole block's length, so a client can tell a snippet from + a complete block and fetch the rest from + `/documents/{documentID}/conversion`. + chars: { type: integer, description: The block's full length in runes, not bytes. } + bm25: + type: number + description: | + FTS5's own relevance, negative and lower-is-better. 0 in `substring` + mode, where there is no index term to weigh. + score: + type: number + description: | + What the results are ordered by: `bm25` minus 1.0 for a heading. The + heading bonus is a judgement — a heading names a section and so answers + "where does it say this" better than a passing mention — and both numbers + are reported so it can be argued with rather than merely trusted. + security: - sessionCookie: [] - bearerToken: [] diff --git a/docs/design/conversion.md b/docs/design/conversion.md new file mode 100644 index 0000000..21b45c6 --- /dev/null +++ b/docs/design/conversion.md @@ -0,0 +1,1883 @@ +# Turning a manual into something you can read + +Contract for the next change, written before it is built, in the pattern +[regions.md](regions.md) set. The probe answers *what is in this document*. +Conversion answers *let me read it* — and it is the first stage that produces +something a user looks at rather than decides on. + +Prerequisites are done and committed: the language map, regions (which part of a +page is which language), and the font each run is set in. Conversion is what those +were for. + +## What it produces + +**Blocks, not a page image and not a PDF viewer.** A block is one piece of +readable content: a heading, a paragraph, a list item, a table, a picture, a +caption. In document order, with the original's page number kept so a reader can +say "page 47" and mean the same thing the paper does. + +That choice is already implied by what comes after. [ingest.md](ingest.md) says +extraction must be able to cite *"a paragraph rather than a document"*, and +full-text search wants the same units. A rendered page image satisfies neither, and +a single blob of text per page satisfies neither. + +## The decisions + +**Only the regions in scope are converted.** This is the whole funnel. A household +that reads German gets the German column of each page — not the page, and not the +other four languages sharing it. The measurement that justifies the funnel is +already recorded: 47,641 characters of the column manual against 240,622 in all its +languages, so converting one language is a fifth of the work. + +**Reading order comes from the columns inside a region, not from the region's box.** +The first version of this contract said "within a region's box" and that was wrong — +corrected here because it was caught in implementation rather than in review. + +A region is not always one column. [regions.md](regions.md) rule 3 deliberately +stores a page whose columns are all the *same* language as one whole-page region: +the column manual's page 62 is two columns of German at x=43-443 and x=463-863, and +it is stored as a single region spanning 0-892. Sorting runs down-then-across inside +*that* box interleaves the two columns line by line — which is precisely the +`pdftotext -layout` failure this section exists to avoid, committed under another +name. Measured on that page: its two columns run body lines at a 16-unit pitch whose +baselines drift apart across the gutter (102/102, then 118/120), and interleaving +produces `"rial bitte umweltgerecht. sich bei gewerblicher Benutzung oder +gleichzusetzender Beanspruchung…"`. The sequential manual has the same shape on the +199 pages that read as three columns. + +So a region is subdivided by [DetectColumns] first, and reading order runs down each +column in turn. Lines within a column are grouped by shared baseline, using the same +rule `columns.go` already uses to fold a list marker into its text. + +That rule 3 is still right — a page of same-language columns is one *language* +territory — is what makes this a seam rather than a contradiction: the region says +which language and how much text, the columns inside it say in what order to read it. + +**And reading order needs its own pair of bounds, because a page can have two columns +and no second column.** Committed as `DetectColumns` alone, the paragraph above +reproduces the failure it exists to avoid on every *sparse* page of either document. +Measured on the sequential manual's Russian section: **20 blocks on 6 of its 22 pages** +crossed the gutter, and page 530 came back with its two section banners as one block, +`"Мешок для сбора пыли Основная щетка"`, and its two columns' first step as one +sentence ending `"…мешок для сбора 1. Надавите на"`. It is 0 on those six pages now. + +The cause is not the projection. It is the two gates a *Column* has to pass, and both +are right for what a Column is — a published fact that language attribution reads — +and wrong for reading order: + +- `minColumnRuns = 8`. Page 530's right-hand column holds 6 runs and page 533's + left-hand one holds 6. Neither is called a column, so the page reports ONE, and + `readingGroups` falls back to a single strip across the whole page. +- `maxGutterCrossings = 4`, which is a count and not a share. On a dense page that is + 2% of its runs; page 530 has 17 usable runs in total, so *every* x on the page is + crossed by at most four and the projection reports the whole right-hand half as one + gutter. Page 531 loses its right column that way rather than to the run count. + +So `readingStrips` runs the same projection with both bounds at their limit — no run +may cross a corridor, and one run is a strip — and `readingGroups` uses it only where +`DetectColumns` has already declined to answer. `DetectColumns` itself does not +change, so no `Column`, no region and no language attribution moves. + +**A threshold on the size of the gap was measured first, and refused.** The obvious +line-level guard is "split a line where the gap between two of its runs is far larger +than a word space". Taking the pages where `DetectColumns` finds two or more columns as +ground truth — a gap between two runs of one column against a gap between runs of +different columns — the two distributions overlap completely and no number separates +them. Over the sequential manual, in multiples of the line's own font size: + +| | n | p50 | p90 | p99 | max | +|---|---|---|---|---|---| +| within one column | 2,436 | 0.0 | 1.6 | 17.1 | **22.1** | +| across a gutter | 5,838 | 12.9 | 34.7 | 52.0 | 67.2 | + +The overlap is real on both ends and both ends are ordinary printing. A spec table sets +`Model` and `RLL77SE` 216 units apart on one line of one column; a left column whose +line runs the full measure ends 9 units before the right column's line begins, and on +page 543 the two overlap by 8. There is no gap to put a number in, so none was chosen. + +**Right-to-left regions read their columns right to left.** The strips reached the +Hebrew 189-200 and Arabic 205-216 pages and made visible something that had always been +there: `readingGroups` ordered columns left to right whatever language they were in. +Page 216 prints its disposal warning in a right column and its numbered removal guide +in a left one, so an Arabic reader was handed step 1 before the paragraph introducing +it. The direction comes from the region's language, the same source `lineIsRightToLeft` +uses. 16 pages reorder and not one block is added, removed or rewritten. + +**A table is placed in the strip it mostly sits in, not the one that contains it.** A +table's box is drawn by `PageTables` from the strokes; a strip's bounds are where its +words reach. They come from different inputs and do not nest — the sequential manual's +page 537 draws the base station's spec table x=478-862 in a strip whose text reaches +845 — so under containment the table belonged to no strip and fell to the banner band, +which is read first. The reader was shown the whole spec table and then the page's own +title. + +What all of this cost and bought, measured over both documents with every language +converted, and with no word gained or lost on any page: + +| | column manual | sequential manual | +|---|---|---| +| blocks | 2,345 → 2,407 over 7 pages | 16,097 → 16,132 over 28 pages | +| `reading-order` findings | 0 → 1 | **38 → 24** | +| `join-glued-words` | none either way | **6 → 5** | +| figures | 59, unchanged | 134, unchanged | + +The sequential manual's 14 lost findings are one whole class — every two-column +disposal and product-overview page — and what is left is the routine-maintenance +interval grid, which is a different defect recorded below. Its glued word was +`סוללות|מדריך` on Hebrew page 200, two columns' words meeting inside a block. The +column manual's single new finding is the check's shape rather than a defect and is +explained where it is pinned: page 58's two header cells are read left to right, +level, because the table they head has no drawn top border, so they are prose. + +**Two shapes still weld across a gutter, and neither is two printed columns.** Both are +on the sequential manual's Russian section and both are named here so they are not +rediscovered as this defect: + +- **A label sitting in a gutter reads with the banner.** Page 521 is an exploded + diagram whose callouts land in the gaps between the three columns the detector does + find, so `columnOf` returns −1 and they join the banner band, where a shared baseline + still merges them: `"• Постоянно горит белым: уборка Боковая щетка"` is a left-column + bullet and a diagram label. Reading an unassigned run with the banner is deliberate — + losing it is worse — and giving those runs strips of their own is a further step this + change does not take. +- **The unruled interval grid.** Page 528's `"Раз в 2 недели Раз в месяц Раз в 3–6 + месяцев /"` is the class the reading-order check still reports on 18 pages, already + recorded here as invisible to the table detector. + +**A heading is found by weight and by length, not by size — and there is no size +floor either.** Size alone is known to be wrong here, and the counter-example is +measured: on the sequential manual, 17pt text is 11.4% of the document at 70 +characters per run — safety body copy, which "larger than body means heading" would +promote to a heading. (Its weight reads as *unknown* rather than regular: the face is +plainly `MiSans` and states nothing, and unknown sorts below light, which matters to +any rule comparing weights.) Its real headings are 15pt semibold, 1,268 runs at +**15.5 characters per run**. + +The tempting corollary — that a heading is at least as large as the body — is also +wrong, and costs 80 real headings. The sequential manual's safety pages are set +entirely in 17pt, so on those pages 17pt *is* the body, and their real subheadings +(`Nutzungsbeschränkungen`) are 15pt semibold: **smaller than the text they head.** Characters per run is what +separates a heading from same-size emphasis, and on the column manual the same holds: +18pt bold at 17.8 characters per run are headings, while 14pt medium at 43.8 is +emphasis and table labels. + +**Both weight signals are needed, because the two manuals disagree about which one +exists.** The column manual names its faces honestly — `FuturaCon-Bol`, +`FuturaCon-Med` — and 93.4% of its characters are in a face that states a weight. +The sequential manual does not: 73.2% of its characters are in a face called plainly +`MiSans`, and poppler's own `` marking is the only weight there is. Either signal +alone fails on one of the two documents. + +**Tables come from the ruled lines, which are a different input rather than a better +use of the old one.** [layouts.md](layouts.md) and [regions.md](regions.md) both +record that *geometry* cannot tell a table cell from a text column, and that remains +true. Vector rules are not geometry of the text; they are the lines the document +draws. `pdftohtml -xml` reports none of them; `pdftocairo` reports them exactly, in +PDF points, which is this space divided by 1.5. Measured against renders: + +| page | printed cells | recovered | +|---|---|---| +| column manual 57 | 29 | 25 — the misses are two header rows whose top border is not drawn | +| sequential 20 | 12 | 12 | +| sequential 100 | 16 | 16 | +| sequential 21 | 32 | 32 | +| sequential 15 | 47 | 37 — the misses are exactly the vertically merged cells | + +**A table needs a text guard as well as a shape guard, and this is not optional.** +"Has ruled lines" fires on 68 of the column manual's 68 pages. Requiring a table +shape — at least two columns, two rows, four cells of a legible size — leaves 13, +and three of those are false in an instructive way: pages 22, 38 and 44 are **grids +of framed illustrations**, ruled by exactly the evidence a table gives. What +separates them is whether the cells hold words: 14 of their 15 cells contain zero +characters, while all 12 cells of page 57's table contain text. With both guards: +10 pages of the column manual, 170 of the sequential one — which is 34 languages +times 5 table pages, exactly. + +**A figure is not a block, and `BlockFigure` stays unproduced.** A block's natural key +is the page, the region's left edge and the index within that region — and a +language-neutral figure has no region, so giving it one would either invent a key or +collide with a real block's. Figures come back as their own list, and a reader merges +the two by page and vertical position. `blocks.go` says `BlockFigure` is declared and +never produced; that is still true and is now deliberate rather than pending. + +**Blocks are keyed naturally, so re-converting converges.** Same reasoning as +`doc_regions`, and the same reason: a job handler can run twice. The key is the +document, the page, the region's left edge and the block's index within it. A +surrogate ID would make a second conversion insert a parallel set. This is also +what gives extraction the stable block IDs ingest.md asks for. + +**Conversion runs after the gate, never before it.** It is the first thing in this +pipeline that is not free, and the gate exists precisely so a user authorises it. +The document states `converting` and `ready` have been in the schema since `00002` +with nothing setting them; this is what sets them. + +**Cost — and the free probe now pays some of it, which this section originally +denied.** Reading the ruled lines costs 6.1 s over all 68 pages of the column manual +and 36 s over all 560 of the sequential one, against a probe of about 4 s for either. + +The first version of this said conversion runs only over the pages in scope, so the +pre-flight is never slowed. That became false the moment tables were needed to derive +regions — which they are, because a table's cell dividers would otherwise be read as +language boundaries, and regions are computed by the probe. Reading every page's rules +there would take the sequential manual's probe from 3.6 s to about 46 s: a tenfold +regression of the one thing the design insists is free. + +So the rules are read **lazily, only for a page that just divided into more than one +column** — the only place a table can change a stored answer. Measured: 44 of the +column manual's 68 pages, and **0 of the sequential manual's 560**, since none of its +pages divide by language. Its probe is untouched at 3.60 s. The column manual's goes +from 4.09 s to **8.02 s**, and that is the honest price of reading a +parallel-columns manual correctly. + +Converting the pages in scope is still separate, still after the gate, and still only +over the pages asked for — with one cost this section also omitted: `Convert` pays +**one `pdftohtml` pass over the whole document**, because the probe's `Result` +deliberately does not carry the runs. That is why converting the sequential manual's +German is 3.3 s for 16 pages while its Russian is 13.1 s for 22: the difference is 81 +figure renders, not pages. + +A 6.5x saving exists for that later pass and is not yet verified: one +`pdftocairo -ps` call renders all 560 pages in 5.15 s where per-page SVG spawns take +33.5 s, and the strokes survive, but no PostScript parser has been written. + +## What this deliberately does not solve + +Recorded so the next person does not think these are unsolved by accident. + +**A table with no ruled lines is invisible.** The column manual prints +`Technische Daten` as label/value pairs — `Spannungsversorgung: | 230 V, 50 Hz` — +with no rule anywhere, and nothing detects them. Not five separate spec pages, as an +earlier draft of this said: it is one spec table repeated per language, a block within +the disposal-and-warranty page — 62 German, 63 Polish, 65 Ukrainian. This is accepted rather than solved, and the softening is real: those +pages still *read* correctly, as lines of text, which is how they read on paper. What +is lost is answering "what is the tank capacity" from a cell later. + +A text-only signal was looked for and not found. Row alignment points the wrong way: +the table page scores 29-40% mutual band alignment while three parallel translated +columns score 67% and 100%, because a translated paragraph corresponds to its +neighbour and a two-line question cell does not correspond to a ten-line answer. A +per-column tab-stop streak does find all five specification tables, but also fires on +three pages of numbered lists — it separates a spec table from body text, not from a +list. + +**Vertically merged cells are dropped.** 10 of 47 on one measured page. An omission +in the row walk rather than a limit of the data; the fix is a column-direction twin +of the same walk. + +**A header row with no top border loses its cells.** Four of 29 on the measured page. +The shaded cell backgrounds are filled rectangles present in the same output and +would recover them. + +**Framed illustrations are geometrically identical to tables.** Only the text guard +separates them, and a figure with a caption inside its frame would defeat it. + +**A picture that belongs to no language belongs to every language — of the pages that +language was already going to read.** Decided by the user, and it settles what +regions.md left open: language-neutral content is included in **every** language's +conversion rather than assigned to one or dropped. A reader must not lose a diagram +because the diagram has no language of its own. + +**The scope of "every" is a page, not the document, and that is a deliberate limit +with a measured cost.** The column manual sets pages 14 and 15 as a spread: page 14 is +German and Polish *plus two photographs of the machine*, and page 15 is Russian, +Ukrainian and Kazakh with **no pictures at all**. The same instructions in five +languages, illustrated once. So the pictures serving all five sit physically on the +German page, and a page-scoped rule cannot reach them from Russian: + +| household | figures from the column manual | +|---|---| +| German | 53, of which 51 neutral | +| Polish | 54, of which 51 neutral | +| **Russian** | **1** | +| **Ukrainian** | **1** | + +Those first two numbers were 40 and 41 when this was written and the shape of the +finding is unchanged; reading the clip split merged drawings apart and took that +document from 46 figures to 59, so every per-household count here rose with it. + +Closing that automatically costs either every page's ink for every household — 68 +`pdftocairo` spawns where 52 were charged here, and 1,120 on the sequential manual to +find its 3 pages of neutral figures — or a facing-page association, which would be a +rule invented from one manual's binding. + +**Neither is being built, and the intended answer is different.** The user's direction: +let a reader skim the original and choose pages to convert by hand — having found those +photographs on page 14, ask for exactly them. That handles this case and every case +like it, without a heuristic guessing at what a spread is, and it is the right shape +for a feature that has to be honest about a document it has never seen. Unbuilt, and +recorded here so nobody builds the expensive guess instead. + +**A TENTH OF THAT PICKER IS NOW BUILT, AND IT IS THE TENTH THAT CONTAINS `A-1`.** The +full picker is still unbuilt for the reason above. What exists is the one subset that +needs no new vocabulary and no new UI for choosing pages, because it is already +computable: **the pages no language region claims**, offered at the gate as a single +opt-in. + +The measurement that justified it, and that nearly killed it, is over both fixtures: + +| | neutral pages | chars | figures | what they are | +|---|---|---|---|---| +| sequential | **7** | 1,639 | **61** | cover, 3 contents pages, **2 diagram plates**, colophon | +| parallel-columns | **2** | 10,782 | **0** | a print code, and service addresses in 12 languages | + +Those character counts are the **regions'**, which is the measurement the rest of the +gate screen uses; the same pages measured with `pdftotext` are 1,656 and 11,256. The +two tools disagree by the few percent `scopeChars` already records, and the ordering — +which is the whole point here — is the same either way. + +Two things follow, and the second is the one worth remembering. + +**It is exactly the set that holds the reported defect.** 59 of those 61 figures are on +pages 5 and 6 — the plates this document already records as "never converted" — and +page 5 is the `A-1` exploded parts diagram. `pdftotext` finds **31** `Fig. A-1` / +`Abb. A-1` cross-references in the content pages, one per language section that has +one, and until now every one of them pointed at nothing. + +**Characters rank the two sets in exactly the wrong order.** The junk set holds seven +times the text of the valuable one. A gate offering "7 pages, 1,656 characters" against +"2 pages, 11,256 characters" would invite a user to decline the plates and accept the +address page. So the offer leads with the **picture count**, which means the probe has +to measure it: `doc.countNeutralInk` extracts ink for the unowned pages only — 7 spawns +and 2 on the fixtures — and stores it on `doc_pages.figures`. That column is **nullable +and nil is not 0**, the same absent-versus-zero rule `folioOffset` records; a 0 default +would tell the gate a diagram plate is empty. Bounded at 32 pages, because hundreds of +unowned pages is a language map that did not work rather than front matter, and a spawn +per page there would make the free pre-flight ten times slower on the document already +going badly. + +**These pages contribute pictures and no text, deliberately.** Their regions are +unnamed, `RegionsBlocks` filters on the languages in scope, so they yield no blocks — +while `attribute` reaches its neutral arm and hands every picture to **every** language +in scope, which is rule 2 unchanged. Measured through the running server on the +sequential manual: German goes from **16 pages and 0 figures to 23 pages and 61**, with +its 449 content blocks and 37 furniture blocks **identical**; Russian from 65 figures to +126, its 449 and 58 identical. A German reader now has `Abb. A-1` on page 27 and the 31 +drawings it names on page 5, in one reader. + +Serving their text as well would mean storing blocks with no language and teaching +`BlocksByLang` to union those into every language's answer — a read-path change +touching every document already converted, for three contents pages that need the +tab-stop parser this document already records as unbuilt. Not done, on purpose. + +**Approving grew one boolean, and the promise it had to keep is argued rather than +waived.** `ingest.Approve` deliberately takes no scope argument. This is admitted +because the gate displayed **both** outcomes, because the caller can only say yes or no +to a set the server recomputes from stored regions — it cannot name a page — and +because the choice could not have been configuration: one manual's unowned pages are +diagram plates and the other's are addresses. The decision is written to +`documents.include_neutral_pages` and read back by the job from that row, so the handler +still takes its whole scope from stored state, `ConvertPayload` stays document-only, and +the dedupe key stays the document — which a flag in the payload would have broken, since +approving false then true would dedupe onto the first job. + +**`internal/verify` does not cover these pages.** `ConvertAll` passes +`doc.ConvertOptions{}` on purpose: an opt-in scope taken unconditionally would move every +count pinned in this document for a reason that is not a change in the pipeline. So the +plates are converted-but-unchecked until that call takes an option. + +**An earlier version of this section claimed the opposite of the truth and is +corrected here.** It said the sequential manual's 229 figures were "every one in front +or back matter", so a language-scoped conversion of it would show no pictures at all. +That was read off page numbers without checking which pages the language sections +actually occupy, and it is wrong. Measured properly, over all 560 pages: + +| | | +|---|---| +| figures | 195 | +| figure pages **inside** a language section | **20** | +| figure pages outside one | 3 | +| Russian | **65 figures** | +| Japanese | **69 figures** | +| the other 32 languages | none | + +That total read 229 when this was measured, and the two per-section counts 81 and 82. +All three fell when candidate boxes that overlap were merged — a drawing that had +clustered in pieces is one figure now — and the page counts did not move at all, +which is what says these are the same pictures counted correctly rather than +pictures lost. + +So a Russian or Japanese reader of that manual gets a heavily illustrated section, and +the other 32 get none — because those two sections genuinely carry illustrations and +the rest genuinely do not. The lesson is narrower than the claim it replaces: figures +outside a section are the exception here, not the rule. + +**Some languages have more content than others, and it must not be lost.** Russian +occupies 22 pages of that manual and Japanese 21, where the other 32 languages get 16 +— the extra pages are an illustrated maintenance section that exists only in those +two. PDF page 533 is an example: Russian prose with eight line drawings of the robot, +the waste tank and the vents. Verified: those pages fall inside the stored Russian +region span of 517-538, and their figures are found. + +No attempt is made to audit every language for such extras. The requirement is +weaker and achievable: **whatever a household's own language contains must be read and +processed, however unlike the other languages' sections it is.** Nothing may assume +the sections are alike, and the 16-page assumption is exactly what would have hidden +this. + +**The pictures in a manual are not the images in the file.** `pdfimages` — the +obvious tool, registered in `extern` since before any of this — yields **zero +illustrations across all 628 pages of both manuals.** What it does yield is 1,358 +gradient-mesh slivers of 12x4 pixels on two pages, a 97x73 corner logo, some CE marks +and recycling symbols. Page 42 of the column manual prints four framed line drawings +and reports zero embedded images. Every illustration in both documents is **vector**, +so a figure is found the same way a table is — from what the page draws — and its +bytes come from rendering the crop. + +Two consequences worth stating. `pdfimages` is not useless, but its role is the raster +path for a photographed or scanned manual, which neither fixture is. And a caller that +wants both tables and figures pays `pdftocairo` twice for the same page; that is +accepted for now and recorded rather than optimised. + +**Clip paths ARE read now, and it was the largest visible defect in the output.** This +section previously recorded the opposite as an accepted cost. A figure's box was a +path's *unclipped* extent, so drawings merged and were cropped through their own +artwork. The verifier put a number on it — 22 of 46 figures and 74 of 163 cut off — and +`clip.go` now resolves each shape's effective clip and intersects the path's extent +with it. Measured end to end with `manualbox verify`: + +| | columns manual | sequential manual | +|---|---|---| +| figures | 46 → **59** | 163 → **168** | +| pages carrying figures | 27 → 27 | 20 → 20 | +| cut off by their own crop | 22 → **15** | 74 → **71** | +| carrying a blank band | 4 → **0** | 6 → **2** | + +The count rises while the page count does not, which is what tells a split from newly +admitted furniture: page 42 returns its four printed drawings where it returned three, +page 22 three for three, and page 16 **four for four** — that page prints four framed +panels, not the three an earlier version of this document twice claimed. + +It also fixed a table: page 38 draws a frame edge to y=268.7 while the paint stops at +y=239.06, and those 30 units of phantom rule were closing a cell. Verified against a +432 dpi render. All five ground-truth cell counts are unchanged. + +Two simplifications, stated: a clip is reduced to its **bounding box**, which can only +ever make a figure's box smaller than the unclipped extent and never wrongly larger; and +an unresolvable reference or an `objectBoundingBox` clip means *no clip*, which is the +old recorded wrongness rather than a guess that could erase a picture. + +**The residual cut figures were not the clip either — they were `trimToPicture`**, the +patch written for the cause the clip removed, and it is now fixed rather than removed. +It cut into drawings to exclude labels printed inside them: page 16's third panel lost +its right third, arrow and hose tip, to exclude the label `»click«`. + +**A trim now only pulls in an edge that a text line actually reaches PAST**, and that +rule follows from where the box comes from rather than being tuned. The box *is* the +bounding box of the drawn ink, so a line the box merely reached over must stick out of +it, while a label set inside the artwork cannot. Result: cut figures **15 → 3** on the +columns manual and 71 → 70 on the sequential, with figures, pages and blocks all +unmoved. Seven of the columns manual's thirteen bad trims go away and all six good ones +stay — page 52 still loses the German prose line above its diagram, and now keeps the +nozzle top and three labels the old rule amputated. + +**The obvious rule — a label inside a drawing has ink on more than one side — was tried +and is wrong.** `»click«` on page 16 has ink on all four sides, but the same label on +pages 24, 26 and 36 sits flush at a drawing's right edge with ink on only two, while +page 1's `GEBRAUCHSANLEITUNG`, which is genuinely prose, also has ink on two. Those need +opposite answers and that signal gives them the same one. + +**Two measurements here were misleading and are corrected.** "Figures overlapping prose" +cannot judge this: it counts any run of five runes or more, so a picture keeping its own +seven-rune `»click«` scores exactly like one swallowing a paragraph — it rises 9 → 14 +*because* the fix works, while the prose genuinely excluded stays at 6. And the fixture +pin recording the smallest figure's short side as 128 units was measuring page 52's +diagram **amputated by the trim**; the document's real smallest drawing is page 48's at +130.4, which no trim ever touched. + +The three residual cut figures on the columns manual are pages 11 and 12, where a +page-sized path cannot be attributed to one figure, and page 1, whose cover art genuinely +runs behind the title block. + +**A drawing was also being served in pieces, which a user reported before any test +caught it.** Page 524 of the sequential manual returned six boxes for four printed +drawings, and one of them was *a hand* — part of the robot's underside, cut out and +served as its own picture while still inside the parent. + +The cause was not a missing merge step. `clusterInk` joins *shapes* whose boxes meet, but +a group's box is the union of its shapes and is far larger than any one of them, so two +groups can share most of a rectangle while no shape of one touches a shape of the other. +The same rule run again over the clustering's own output, to a fixpoint, closes it. + +| sequential manual | before | after | +|---|---|---| +| figures | 168 | **134** | +| overlapping pairs | 53 | **0** | +| wholly inside another | 7 | **0** | +| cut off by their crop | 70 | **25** | + +The columns manual is identical in every number; it never had an overlapping pair at any +threshold. Clipped falling to 25 is not a detector agreeing with itself: a piece of a +drawing is genuinely crossed by the shapes of the piece beside it, so removing the split +removes the crossing. + +**Any positive overlap merges; merely touching does not.** No fraction was chosen, +because the measurement offers nothing for one to separate: the 53 pairs run from 1.00 +down to 0.01 with no gap, and every one, rendered and looked at, is a single printed +drawing that clustered in pieces — at 0.91 the hand, at 0.57 a base station split at its +waist, at 0.01 a water tank and the magnified detail its leader lines run to. The cases +needing the opposite answer are untouched because their boxes do not overlap at all: +page 524's two robot views are 23 units apart, page 522's two mop pads 46. + +**Containment alone would not have fixed the reported fault.** The hand is 90.8% inside +its parent, not 100% — its pin pokes 4 units past the edge. A containment-only rule leaves +that page at six boxes with the hand still served as a picture. + +**And the merge exposed a determinism bug.** The groups came out of a map, which is +harmless while merging only grows a box, and is not once a threshold is involved: the +same page returned between 194 and 200 figures across runs. Clustering now sorts into +reading order before merging. That matters beyond a flaky test — these bytes go into a +content-addressed store, so a box that moves means the same page yields different files. + +Two eye counts already in the repo were counting boxes rather than drawings and are +corrected: a page recorded as 8 drawings prints 4, and one recorded as 8 prints 9. + +**A CALLOUT NUMBER WAS BEING CROPPED AWAY, and it made a labelled diagram +unreadable.** Reported by the user against the sequential manual's RU product +overview: the crop keeps the leader lines and loses every label, so the leaders end +in nothing and the drawing cannot be read against its parts. + +The cause is not a bad box, and that is the useful part. A figure's box is the +bounding box of the drawn **ink**; a label is a **text run**. On PDF page 521 the +lidar drawing's box ends at x=263.0, its leader terminators are the marks at +259.6–263.0 that *set* that edge, and all eleven of its labels begin at **266.0**. +Three units, every one. The box does not need to find the leader's end — it is +already sitting on it. It needs to cross the gap, and nothing in `findFigures` ever +grew a box: `trimToPicture` only ever pulls edges in. + +**What says a run is a label is the terminator, not the distance.** Both documents +draw a small open circle where a leader stops — 3.3 to 3.4 units square, measured — +and one sitting in the corridor between the box's edge and a run, on that run's +midline, is what claims the run. The case that rules the distance out is a document: +page 11 of the columns manual prints its **parts list**, 39 numbers and 39 German +names, 22.3 units to the right of the exploded view — *inside* the range page 521's +underside diagram holds its own labels at, 20.3–35.3. It is not that the legend sits +further away; it is that no distance separates the two, so any "grow onto text within +N units" rule swallows the whole list. The terminator test refuses all 78 of its +runs, because a legend is not pointed at. + +**A label wraps, and its later lines are the obstacle.** Page 521's lidar drawing +claims nine of its eleven labels by terminator; the two continuation lines +(`на основе ИИ`, `3D-датчики`) carry no mark of their own and, left unclaimed, block +the edge from moving at all. So a run flush with a claimed label, on the adjacent +line, **alone on its baseline**, is part of it. Alone is what separates a label from +a bulleted description: a bullet has its text beside it 1 unit away, a continuation +line does not. Comparing bands rather than baselines gets this wrong in a way worth +recording — two consecutive lines of one label overlap vertically, because a run is +taller than the pitch it is set at, so a band test reports a label's own third line +as something sharing the second's line and blocks every growth on the page. + +That rule reaches only along the corridor **without** the band the leader is judged in, +and the reason is a fault the user reported off the same page — see "A wrapped label kept +its head and lost its tail" below. + +**The conservative half is that prose stops an edge dead.** An edge moves only if +everything the growth region touches is a claimed label; one line of prose in the way +and the edge stays. That is why page 521's lid-open drawing keeps its three +right-hand labels cropped — the corridor holds `Кнопка сброса` and then the five +bullet lines explaining it — while its left edge takes nine labels. + +**A claimed label may be cut short; prose may not.** On a page whose two label +columns interleave in x this is the difference between the fix working and not +working: the lidar drawing reaches x=397 where its own longest label ends at 469, +because the neighbouring drawing's labels start at 400. Refusing to cut a label at +all was measured, and it costs the whole page — that drawing does not grow, and +neither does its neighbour's left edge. A leader ending in a word cut short is a +large improvement on a leader ending in nothing; a picture with a paragraph in it is +not. + +What it is worth, over both whole documents: + +| | columns manual | sequential manual | +|---|---|---| +| figures | 59 | 195 | +| figures with a label outside them | 2 | 79 | +| figures grown | **0** | **55** | +| labels taken in | 0 | **229** | + +Those 229 are labels the crop **reaches**, and the gap between reaching and +containing is exactly the clipping this accepts: page 521's three drawings reach 9, 11 +and 14 labels and hold 3, 8 and 12 whole, so 23 of its 34 arrive uncut. It is the +number that would become 34 if a label were carried as text instead. + +**The columns manual does not move at any setting**, which is the same shape of +evidence the merge rests on, so this is the other document's change entirely. Both of +its claims are FALSE and both are blocked, and that is the clearest statement of what +the conservative rule is for: page 1's cover figure claims the title +`РУКОВОДСТВО ПО ЭКСПЛУАТАЦИИ` and page 22's claims eight lines of German prose about +emptying the DryBOX. The terminator signal is not precise on its own — a small shape +near a line of text will do — and what makes it safe is that an edge does not move +unless the region it would add holds nothing but claims. Of the sequential +manual's 55, **22 are on pages 5 and 6** — the front-matter diagram plates, which +fall outside every language region and are never converted — leaving 33 on pages a +reader is served. Page 521's three drawings take 9, 11 and 14 labels. + +**The cost is overlapping crops, and it is confined to those plates.** Eleven pairs +of grown boxes overlap — nine on page 5 and two on page 6, where 31 and 28 figures +share one sheet with labels between them, and one of page 5's is a crop wholly inside +another crop; none is on any page a conversion serves. The **drawn** boxes are +untouched: measured over both documents on every page they still overlap in 0 pairs +and nest in 0, so the merge pass's property holds of the rect it is about. +Widening the corridor is what would change that: at 80 units a pair appears on page +524, which a reader is served, and that is the upper bound's evidence. +Not fixed, because arbitrating which of two drawings a shared corridor belongs to +would be a rule invented for one plate. + +**The rendered rectangle and the drawn one were different things while this pass ran, +and one caller must not confuse them.** `Figure.Rect` is what was rendered and is what +the stored pixel size describes; `Figure.InkRect` is the drawn extent the two guards +judged. Attribution reads the drawn one, through `Figure.DrawnExtent`: a box grown +sideways onto a label could otherwise reach out of its own language column and be handed +to every household, which is the one failure the funnel may not have. A picture's +language is a property of the picture, not of how much of the page around it was +rendered. The ink box is not stored — nothing reading a conversion back asks the +language question again — so there is no migration. + +*Now that the crop does not grow the two are equal on every figure of both documents, +and the distinction is kept anyway.* It costs nothing, `DrawnExtent` is what attribution +must ask whatever the crop happens to be, and a pass that widens a crop again would +otherwise re-open a hole the funnel may not have. + +**The order of the two guards and the label pass is deliberate.** A diagram's own labels +are text, so the extent covering them is legitimately over `maxFigureTextFraction`: page +521's lidar drawing reaches 0.162 with its eleven labels. Judging the drawing plus its +labels would reject the very pictures the labels complete, so the guards judge the +drawing and the labels are collected from a drawing that has already passed. +`TestTheGuardsJudgeTheDrawingNotTheCrop` asserts that the labelled extent WOULD have +failed the guard, which is what keeps the test meaningful now that no box grows. + +**A perfectly horizontal leader line is invisible to all of this, and fixing that was +measured and rejected.** `onPageInk` drops a shape whose box has no extent on one +axis, and an axis-aligned hairline is exactly that: page 521 carries 52 such shapes 8 +units or longer, its leaders among them, and that is why the underside drawing's +terminators sit 28 units *outside* its box while the lidar drawing's sit on the edge. +Keeping them was tried. It costs the sequential manual **16 figures, 195 → 179**, and +the reason is that the restored shapes include the page's own furniture: page 5 draws +a zero-width column separator 402 units tall, and it bridges that page's middle +column into one 244×402 box where ten drawings were found before. The columns manual +does not move (59 → 59, identical per page). Not taken, because growth reaches the +labels without it — a terminator survives the filter on its own, being a circle +rather than a line. + +**What is still cropped is counted, so it can only go down.** 41 figures of the +sequential manual held 88 labels their leaders point at and their crop did not +reach — 16 of those figures and 29 of those labels on the two plates — and the columns +manual's 2 are its two false claims. + +**EVERYTHING FROM HERE TO THE PLATE MERGE IS NOW HISTORY, and it is kept because it +is the measurement the replacement had to beat.** The crop no longer grows: +`maxLabelGrowth` is off, `growToLabels` survives only as the thing `TestGrowSweep` and +`grownGuards` measure, and a label reaches a reader as text. Read the section above for +why growing was tried and what it cost; read below for what replaced it. The residual +count is gone with the pass it belonged to — every label is now outside every crop, so +"labels outside the crop" would report all of them while a reader sees all of them too. + +## THE CROP IS THE BAND THE PAGE PRINTS, AND THE LABEL TEXT IS ITS DESCRIPTION + +**This reverses the section below, which is kept because the sequence is the argument.** +Read the three designs in order — grow the crop, draw the labels, crop the band — and +the thing that changed is not the claim rule, which has never moved. It is what a claim +is *for*. + +The user's instruction, after looking at pages 521 and 522 in the reader: it "becomes +too complicated in terms of formatting — maybe just do screenshots, not trying to parse +helper text for images. We can parse it and add as alt for images." + +They were right, and the reason is worth keeping because it undoes work. **The page had +already solved the layout, and re-laying it out is what produced every defect in this +area.** Labels drawn as positioned DOM elements gave four: collisions (page 522 +overlapped in two places), stranded unclaimed siblings, orphaned wrapped tails, and per +figure placement with no knowledge of the neighbour. All four existed only because the +reader was rebuilding an arrangement the paper prints correctly. Take the arrangement +instead and there is nothing left to place, to cut, or to collide. + +### The band is bounded by the claim rule, and the page band was measured and refused + +`labelBand` is the drawing unioned with every run `claimLabels` reaches for it, taken +whole, with the top and bottom moved out so no line of type is cut across. **Every +claim, gated or not** — because the two questions have different right answers: + +| | | +|---|---| +| what becomes ALT TEXT | only what `labelExtent` believes is a label | +| how wide the PICTURE is | everything the page laid out beside the drawing | + +Being wrong about the second costs a paragraph printed inside a picture that already +prints it. Being wrong about the first tells a screen reader that eight lines of German +body prose are a diagram's callouts. That asymmetry is the whole design, and it is why +the gate stays exactly where it is while the crop ignores it. + +**The first choice was the page's full width and the measurement refused it.** The cost +of a band is prose the crop prints as pixels that the block flow ALSO emits as text, so +a few lines arrive twice. Counted in runes over the pages that serve a crop, by +`TestABandCropDuplicatesThisMuchProse`: + +| crop rule | columns manual | sequential manual | +|---|---|---| +| the drawing alone, as it was | 773 — 1.0% | 51 — 0.3% | +| drawing + the GATED labels | 773 — 1.0% | 116 — 0.7% | +| **drawing + every claim — SHIPPED** | **1,481 — 1.8%** | **1,714 — 9.8%** | +| the printed column, over that band of y | 45,736 — 56.8% | 4,355 — 25.0% | +| the page's full width | 67,662 — 84.0% | 11,377 — 65.4% | + +**59,618 of the full band's 67,662 runes are a NEIGHBOURING COLUMN's text**, and on the +columns manual a neighbouring column is a different language — the one failure +`attribute` says the funnel may not have, restated for pixels. That document also +carries **0 labels**, all nine of its claims being false, so it would pay the entire +cost of a feature it cannot use. The plan's estimate before measuring was "page 521's +top band is almost nothing; page 522 would be more"; it was wrong by two orders of +magnitude, and structurally rather than by accident. + +The gated-labels row is the cheap alternative and it is refused for the opposite +reason: at 0.7% it is nearly free, and it leaves the stranded siblings exactly where +they were. + +### What it fixes, which is a defect this file recorded as unfixable + +The section below says `Кнопка сброса`, `Индикатор Wi-Fi` and `Датчик края` float +mid-page, with **both** ways of drawing them measured and refused. Neither refusal was +about cropping. The band prints all three: verified in Chrome on page 521, where +figure 0 now shows every label the paper prints beside it, including the bullet +description under `Кнопка сброса`. **45 of the sequential manual's 50 refused claims are +printed by the crop.** + +**And a correction to what this file recorded — twice, the second time by the user's +photograph.** The fourth name on that list, `Датчики перепада высоты`, was written down +as a claim of figure 2 whose side the gate refused. It is not: it is claimed by +**nothing**, so no gate ever sees it and the leaders in the drawing's top edge end in +empty paper. Why nothing claims it is the part worth having: + +| | | +|---|---| +| figure 0, below | **40.44** units past its box, against `labelCorridor` = 40 | +| figure 2, above | 27.43 units, within reach, and **a real terminator, refused by `labelAlign`** | + +The second row is the finding. There IS a mark: 3.3 by 3.3 units at (665.3, 339.3), +exactly the size this file measures a leader's end circle at, sitting between the +drawing's top edge and the label. `terminatorAt` refuses it because it asks that the +mark be within `labelAlign` = 4 units of the label's **midpoint**, and the midpoint of +`Датчики перепада высоты` is x=680. The mark is 14.7 units off it. + +**That is an asymmetry rather than a threshold.** `labelAlign` is the same 4 units on +both axes, and a label's extent along those axes differs by an order of magnitude: a +one-line label is 13 to 14 units tall, so ±4 of its midline is most of it, and this one +is **140 units wide**, so ±4 of its midpoint is 6% of it. A leader arriving at a top +label from below meets it wherever the drawn part is, not at the centre of the phrase. +The rule that would fix it is the mark falling inside the run's own extent on that axis +rather than near its middle — the same test, expressed against the box instead of the +point. + +It is **not built here**, and the reason is scope rather than doubt: it changes +`claimLabels`, which every count in this file is expressed in — 327 claims, 276 carried, +the block totals, coverage — so it needs its own measurement over both whole documents +rather than a patch on the end of this one. `labelCorridor` is deliberately **not** +moved to 41 either: a bound set from the one sample it has to admit is the fitted +threshold `minFigureWidth` refuses on the record, and this label does not need it. + +### What it costs, stated rather than managed + +**A crop wholly inside another is removed, and that one is not a cost but a fix.** +Two drawings that reach the same run can end up with one band inside the other: page +529's figure 7 is figure 4's band with the top cut off, because both claim the numbered +step underneath, and serving both shows a reader the picture and then its own lower +half. `absorbNested` drops the contained crop and gives its labels to the one that +swallowed it — **6 on the sequential manual, 0 on the columns one**, taking crops served +from 195 to **188** and, over all 34 languages, figures 134 to **128** with the label +count **166 identical**. It is not the refused plate merge: that unions two want boxes +and invents a rectangle larger than either, and this invents nothing at all. The +surviving crop is one that already existed, unchanged to the unit, so there is no +cascade to bound. `TestNoFigureOverlapsAnotherOnEitherManual` asserts nesting at zero +everywhere, which is the half of that test the band did not weaken. + +**Two crops may still overlap, and a neighbour's label can be cut at an edge.** The band +rule produces **35** overlapping pairs on the sequential manual and runs cut by a crop +edge go **16 → 48**; after `absorbNested` what a reader is served holds **26** pairs, 15 +of them on the two front-matter plates and **11 on pages a conversion serves**. The +columns manual has 0 either way. +It is visible on page 521: the lidar drawing's crop shows `Вен` / `отвер` / `автоо` at +its right edge, which are the robot drawing's labels, and the robot's crop shows the +tail of the lidar's. Merging the pair is the obvious answer and is refused on the +record by `TestAPlateMergeOnSharedLabelsIsRefused`, where the transitive closure of +overlapping want boxes takes page 521's three drawings into one crop 0.629 of the page +and joins page 522's front view to a cutaway 328 units away. + +**The crop can print text the alt text does not name.** Page 522's second figure carries +no labels and its picture prints `Зажим бака для воды` and `Поплавковый уровнемер`, +which are claims nothing gated in. That is the honest shape of a conservative gate over +a generous crop, and it is the right way round: a description that says less than the +picture shows is a gap, one that says more is a lie. + +**A wrapped label reads as several labels.** One entry is one printed LINE, which is +`claimLabels`' own unit — a continuation is its own claim — so page 521's +`Вентиляционное отверстие системы автоопорожнения` is three entries and the alt text +says `Вентиляционное; отверстие системы; автоопорожнения`. This is not new; it was +invisible while the reader placed each line against the drawing, because the paper's own +arrangement put them back together. It is visible now that the strings are the only text +a screen reader gets. Joining a chain back up is a real improvement and is not built, +because the chain is not recorded: `continuesLabel` answers one pair at a time and +nothing keeps which line continued which. + +**The columns manual pays 708 runes for a feature it cannot use.** Its page 22 crop +takes in the eight lines of German prose behind its false claims, and the same text is +emitted below it as blocks. Verified in Chrome; it is one page, and it reads as a +screenshot of the page followed by the page's own words. + +### What did not move, which is most of it + +Measured with `manualbox verify` over the whole sequential manual in all 34 languages, +the same binary path before and after: + +| | before | after | +|---|---|---| +| blocks | 14,749 (1,289 furniture, 165 callout) | **identical** | +| labels carried | 166 | **identical** | +| figures | 134 | **128** — see `absorbNested` | +| median coverage | 0.996 | 0.996 | +| `reading-order` | 23 | 23 | +| `invented-text`, `join-*` | 160, 5, 72 | identical | +| **`figure-clipped`** | **25** | **14** | +| total findings | 287 | **276** | + +Only the crop moved, and `figure-clipped` fell because the crop is now wider than the +drawing rather than equal to it. `attribute` still asks `Figure.DrawnExtent`, so no +picture changed language: `TestTheCropDoesNotChangeWhichLanguageAPictureBelongsTo` used +to assert `Rect == DrawnExtent` and now asserts the distinction that matters — the drawn +box is present, and it is the smaller of the two. + +`Figure.Labels` is `[]string`. The position and the side are gone from the type, the +schema (`00009`), the served JSON and the reader; `Figure.LabelBoxes` hands the run's +own coordinates to `Callouts.Mark` in-process and is never stored, which retires the +fraction-reconstruction hazard `calloutAt` documents rather than absorbing it. The order +is the page's own — down, then across — because that is the order alt text should read +in. `Block.Callout` is unchanged and still load-bearing: the picture prints the label, +so the prose must not. + +## A CALLOUT LABEL IS CARRIED AS TEXT, AND THE CROP IS THE DRAWING — SUPERSEDED, SEE ABOVE + +The complete answer was never a wider crop. A crop is a rectangle and a diagram's +labels are not arranged in one, so the growth pass could only reach a label by taking +in everything between — which cut the ones it could not reach cleanly and merged crops +on the plates. `Figure.Labels` keeps each claimed label as a string with a position +relative to the crop, `doc_figure_labels` stores it, the conversion response serves it +and `Reader.tsx` draws it beside the picture. *(Superseded: the crop is now the band +and the reader draws nothing — see the section above. What follows is the design this +replaced, kept because its measurements of what IS a label are unchanged and still +load-bearing.)* + +**The claim rule does not move.** `claimLabels` — terminator plus continuation line — +is reused unchanged, so every measurement in the section above about what IS a label +still stands. What changed is what happens to a claim once it is made. + +**Growth's conservative half is KEPT and its cutting half is DROPPED**, and which is +which is the whole design. + +The conservative half is not optional, and the columns manual is the proof. That +document has **9 claims and every one is false** — page 1's cover figure claims the +book title, page 22's claims eight lines of German body prose about emptying the +DryBOX. What refuses all nine is not the crop; it is `labelExtent` finding no distance +whose growth region holds nothing but claims, because real prose sits in the corridor. +So `figureLabels` gates on that test **per side**. Drop it and the manual whose +pictures were counted by eye gains nine labels it must not have. + +The cutting half was an artefact of the rectangle. Growth moved an edge only as far as +it could go cleanly and let the labels past that point be cut — the lidar drawing +reached x=397 where its own longest label ends at 469, because the neighbour's labels +start at 400. A text run has no rectangle, so once a side is established as labels +rather than prose, every claim of it is carried WHOLE. Nothing is cut and nothing has +to arbitrate a shared corridor. **The side is the unit of the decision; the label is +the unit of the answer.** + +What it is worth, over both whole documents. Read the sequential column as the sequence +it is: the second value is what carrying a label as text bought, and the third what +dropping the band off the wrap bought after it, measured in the section below. + +| | columns manual | sequential manual | +|---|---|---| +| claims `claimLabels` finds | 9 | 319 → 327 | +| claims carried as text | **0** | **268 → 276** | +| held whole by the grown crop | 0 | 186 | +| carried on pages a reader is served | 0 | **159 → 167** vs **99** the crop held | +| page 521, held whole | — | **23 → 34 → 37** | + +Page 521 carried 34, which is every label its crop ever *reached*, and held all 34 +instead of 23. 109 of the 276 are on the two plates, which yield no blocks at all — so +before this their labels reached a reader only by being inside a grown crop, and now +they are the only text those pages contribute. `TestALabelReachesAReaderWhole` pins +both columns and asserts the replacement is not worse than what it replaced. + +**Verified in Chrome on the real manual.** Page 521 before: `Вентиляционное отверстие +системы автоопорожнения` arrives as `яционное / е системы / орожнения` and +`Вспомогательная светодиодная подсветка` as `Вспомогательная светодио`, seven labels cut +mid-word. After: 34 labels rendered, **0 clipped and 0 off-screen**, measured from the +DOM rather than by eye. + +**The position is a fraction of the crop, and it is routinely negative or over 1.** +*(Superseded: `Label` no longer carries a position at all, and migration `00009` took +the columns out. The paragraph is kept because the fact it turns on — a label sits +OUTSIDE the drawing — is exactly why the crop had to become a band.)* +That is the normal case, not bad data — a label sits outside the picture, which is why +widening was the only way to reach it. Fractions rather than the 1.5-scaled units +everything else here carries, because a label is compared against nothing: its only +consumer draws it against a rendered image whose size it chose itself, and given +fractions it needs neither the page box, the dpi, nor the crop's rectangle. +`00008_doc_figure_labels.sql` records why there is deliberately no `CHECK` bounding +them. + +**The labels leave the block flow, marked rather than deleted.** Page 521 read +`Датчики перепада высоты` as a floating paragraph, so with the reader also drawing it +the label would arrive twice. `Block.Callout` marks the runs that left, exactly as +`Block.Furniture` marks page furniture, and `ContentBlocks` filters them at the save +boundary. + +The cheap version — drop the finished block whose text equals a label — **was measured +and does not work.** Of the 89 labels on the sequential manual's Russian pages, 23 are +a block of their own and **66 arrive inside a bigger block**: page 522's `Бак для +отработанной воды` and fifteen more sit inside larger paragraphs, because a label +column sets its lines at the body pitch and the paragraph rule has nothing to separate +them by. Dropping those blocks deletes the neighbouring content. So the run leaves +before anything is grouped, which is what `splitFurniture` records for the language +tab. Russian content blocks go **449 → 409 → 404**, with **89 → 95** callout blocks +carrying the text. The last step is the wrapped-label tail below, and the two do not move +by the same amount on purpose: 6 runs left the flow and only 5 blocks went with them, +because one of the six was already inside a bigger paragraph — which is the 66-of-89 +measurement above, seen from the other side. + +**`Callout` is a SECOND flag rather than reusing `Furniture`, and the difference is +coverage.** `checkCoverage` deliberately does not count furniture: furniture is +discarded, and counting it would hide a rule that wrongly claimed a paragraph. A label +is **relocated, not discarded** — the reader still shows it — so its characters must +stay in the numerator, or coverage would fall by exactly the labels on every +illustrated page and nothing could tell that from lost text. Measured end to end with +`manualbox verify` on the sequential manual across all 34 languages: **0 coverage +findings before and after, median 0.996 both times, and the five least-covered pages +identical.** + +`checkOrder` needed the opposite treatment: a callout block is appended after its +region's content, so judging it asks where in the reading order something outside the +reading order comes. Page 521's two label columns are disjoint in x and descending in +y, which is that check's violation shape exactly. Excluding callouts takes +`reading-order` **24 → 23** — one finding that was two labels compared against each +other and predates this change. Page furniture is *not* excluded there, and that is +worth knowing: it escapes by position rather than by rule, because a tab or a folio +sits at the top or bottom of a page so the block after it is further up. + +**The one number that went the other way is `figure-clipped`, 24 → 25.** The crop is +now the drawing exactly, so a leader running past the drawn extent is no longer inside +it. That is the class already recorded above — leader lines on crowded diagram pages — +and it is a consequence of the crop being what it says it is. + +### What this still gets wrong, measured + +**Seven claims are prose, not labels, and there is no threshold to refuse them.** Page +529's numbered step 6 and its continuation, page 531's instruction sentence, and four +more over 546, 550 and 553 are drawn beside a picture and taken out of the list they +belong to. Sorted by length the claims run 37, 38, **39**, 40, 42 runes, where 38 is a +real label (`Вспомогательная светодиодная подсветка`) and 39 is a Japanese sentence — +**one rune apart**, which is a coincidence and not a gap, and cutting there would be +the fitted threshold `minFigureWidth` refuses on the record. Only 2 of the 268 end in a +full stop — 268 was the total when that was counted, and it is 276 now for the reason the +next section gives, with the same 2. So that signal is worse. It is a misplacement rather +than a loss: every one is still shown and still counted by coverage. `TestASentenceShapedClaimIsTheResidual` +pins the count and the one-rune non-gap, so a change that opens a real gap is noticed. + +### A wrapped label kept its head and lost its tail, and that is FIXED + +This was recorded here as a deferral and it was the wrong call: the user photographed +it. `Монтажные отверстия для` was drawn on page 521's underside drawing while +`держателя`, `насадки для` and `швабры` stayed in the prose as three floating +paragraphs. **A label is a unit, and half of one is worse than either answer** — the +reader saw half a label on the picture and the other half adrift in the text a screen +below it. + +**The tail was not lost between claiming and carrying. It was never claimed.** The +deferral guessed at `continuesLabel` and the cause is one line further out: +`claimLabels` asks `runBeyond`, which requires a run to sit inside the band the drawing +occupies on the other axis. That band is right for a **leader**, which points out of the +drawing, so its label is level with some part of it — and it is the thing that stops a +terminator coincidence elsewhere on the page from claiming a run. It is wrong for a +**wrap**, which goes downward: this label is level with the drawing's foot, so lines 3, +4 and 5 of it sit 8, 20 and 32 units *below* the box and were invisible to the +continuation fixpoint. Its first two lines were level with the box, so they were +carried. + +So the continuation pass asks `runInCorridor` — `runBeyond` without its band — and so +does `continuesLabel`'s scan for a run sharing the candidate's line. Both halves of that +test must see the same set, and the band-free set is the **larger** one, so the +aloneness test gets stricter rather than weaker. Nothing else moves: a continuation must +still be flush to 3 units with a claimed line, adjacent to it to 6, alone on its own +baseline, and reachable along a chain that starts at a leader. + +| | columns manual | sequential manual | +|---|---|---| +| claims found | 9 | 319 → **327** | +| claims carried | **0 → 0** | 268 → **276** | +| carried on served pages | 0 | 159 → **167** | +| carried on the two plates | 0 | 109 → **109** | +| page 521 | — | 34 → **37** | + +Every one of the 8 is a **later line of a label whose earlier lines were already +carried**, so nothing new is claimed: page 521's three, `Бак для | чистой воды` and +`Вентиляционное | отверстие системы автоопорожнения` on 522, `モップパッドホル | +ダー取り付け穴` on 542, and the second line of page 546's bullet caution — one of the +seven prose claims above, now wrong *whole* rather than wrong by halves, which is the +point. + +**The invariant replaces the count, because what went wrong has no total that describes +it.** `TestNoLabelIsCarriedWithoutItsLaterLines` asks of every carried side of every +figure of both documents: is there a run left in the flow that `continuesLabel` says is +the next line of something carried? A chain is grown to a fixpoint over exactly that +predicate, so the answer can only be yes where a bound cuts it, and both bounds are +measured: **0** continuations anywhere in either document are refused by +`labelCorridor`, and **2** by `minWrapRunes` — the single digits `4` on plate page 5 and +`2` on page 6, which are their own numbered callouts and not any label's second line. +They are asserted by name so a real truncation cannot hide inside the allowance. + +Verified in Chrome on page 521 of the real manual, before and after: the whole five-line +label is drawn against the drawing and the three floating paragraphs are gone. Measured +through the API on the same document, page 521's flow blocks go **32 → 29**. + +**Mutated one way at a time, to see which test dies.** The last row is the one worth +knowing: + +| mutation | caught by | +|---|---| +| the band back on the continuation pass — the fix reverted | `TestNoLabelIsCarriedWithoutItsLaterLines` names all five truncated labels by page and text; `TestALabelReachesAReaderWhole` 167 → 159; `TestTheReportedPageKeepsItsCalloutLabels` figure 2 at 14 | +| no `labelCorridor` bound on a continuation | `TestALabelReachesAReaderWhole` 167 → 171 — and *not* the invariant test, which asks the opposite question by design | +| `minWrapRunes` = 0 | `TestNoLabelIsCarriedWithoutItsLaterLines`, because the floor's two named digits vanish from the allowance; `TestALabelReachesAReaderWhole` 167 → 175 | +| the band back on the **aloneness scan alone** | **nothing** | + +That last mutation is unobservable on both manuals, and the change is kept anyway with +the reason on it: a run outside the band that shares a later line's baseline is a shape +neither fixture prints, and the first document that prints one gets a label with a +neighbour's words in its chain. It is consistency between the two halves of one test, not +a measured gain, and it says so in `continuesLabel`. + +### `Датчики перепада высоты` still floats, and the alternatives were measured + +The gate that refuses its side is `labelExtent`, per side, and the question asked here +was whether that refusal is still load-bearing now that **nothing is widened** — the +crop no longer grows, so a refusal costs a drawn label rather than a swallowed +paragraph. Three options, measured rather than argued. + +**Which gate refuses what, first.** All nine of the columns manual's claims are refused +by `labelExtent` and by nothing else: page 1's cover title is 1 terminator claim with no +continuation, page 22's eight lines of German prose are 1 terminator claim plus 7 +continuations. Page 521 figure 0's right side is 6 claims — `Кнопка сброса`, `Индикатор +Wi-Fi` and `Датчик края`, which are real labels with leaders, plus the three lines of the +bullet description under `Кнопка сброса`, chained **upward** from `Индикатор Wi-Fi` +because a continuation has no direction. What blocks the extent is the bullet `•` in the +corridor. + +- **(b) Draw them too — REFUSED.** Admitting that side draws the three bullet lines as + labels, because they are claimed: `нажмите и удерживайте кнопку в течение 3 секунд, + чтобы восстановить заводские настройки` would be printed on the base-station picture. + Making the gate **per label chain instead of per side** — the strip from the drawing's + edge out to that chain, restricted to the chain's own lines, holding nothing else — + was implemented and measured, and it carries **8 lines of German body prose** on the + columns manual's page 22. That is the document whose 9-for-9 false claims are the + recorded safety property, so the refusal is still load-bearing and this is what depends + on it. +- **(c) Keep them in the flow but adjacent, as the figure's caption — REFUSED for the + same reason.** It needs the refused set to be labels, and it is not: 9 of 9 on the + columns manual are not, and 3 of the 6 on page 521 are bullet prose. Presenting page + 22's German paragraph as a figure's caption is the same false claim as drawing it, + moved into the note. +- **(a) Leave them. TAKEN.** A refused claim is a claim of **unknown kind**, and there is + no signal here that separates the real ones from the false ones. Terminator-vs- + continuation does not: the columns manual's two false claims are both terminator + claims. Length does not, and that is already on the record one section above — 38 + against 39 runes. Counting terminators per side separates these two documents by + exactly one sample and is the fitted threshold this file refuses elsewhere. + +So the leftovers stay in the prose, which is where text of unknown kind belongs, and the +cost is stated rather than managed: page 521 shows four label-shaped lines adrift. +`TestTheReportedPageKeepsItsCalloutLabels` names two of them so the cost stays visible. +What would change the answer is not a better threshold but a better signal — the leader +**line**, not its end mark. Those lines are in the raw ink and `onPageInk` drops them, +because a perfectly horizontal hairline has zero height; a pass that recovered them +could ask "is a line drawn from this drawing to this run", which is the question +`terminatorAt` approximates. Probed on page 521: reachable for 8 of that page's 26 +claims with a first cut, which is not enough to gate on and is why it is written down +rather than built. + +**The Hebrew and Arabic sections carry no labels at all.** Measured: a Hebrew reader's +labelled pages are the two plates and page 560, nothing in 185-196. So the +right-to-left path is exercised by the plates and by tests rather than by a content +page, and page 189 was checked in Chrome to confirm it is unchanged. + +**Placement is physical, not logical, and that is the one exception in the reader.** A +label printed to the left of a drawing is to the left of it on a right-to-left page +too: the picture is a picture, and mirroring its labels moves each one off the part it +names. The label's own text stays direction-aware. + +**The one-file alternative to that was measured and refused.** The idea is good and +worth writing down: if two drawings' label claims interleave, the page laid them out +as ONE plate with a shared label field, so serve them as one picture — a merge +criterion in `figures.go` instead of a schema, an API and a reader. Give every figure +a *want box*, its drawn extent plus every run `claimLabels` claims for it, and merge +the figures whose want boxes overlap. The columns manual does not move at all: it has +no page where two want boxes overlap. On the sequential manual: + +| | today | interleaving want boxes | two shared claimed runs | +|---|---|---|---| +| crops served | 195 | **162** | 190 | +| merged groups | 0 | **17** | 4 | +| claimed labels held whole | 186 of 319 | **258** | 199 | +| largest merged crop | — | **0.629 of its page** | 0.049 | +| crops over `maxFigureTextFraction` | 0 | **9** | 0 | +| page 521 | 23 of 40 | **40 of 40** | 23 of 40 | + +**Page 521 cascades.** Merging the pair the diagnosis names does produce one crop +holding both drawings and all 20 of their labels uncut, and it was rendered and is a +real improvement. But figure 1's want box also clips figure 2's — by 14×5 units, an +overlap of 0.001 — so the transitive closure takes all three drawings into a crop +0.629 of the page holding two columns of button-description prose, cut mid-line. + +**Page 522 does not cascade, and the expectation that it would was wrong.** Its seven +printed pictures come back as 9 figures and 6 of them *do* grow — growth does not +decline that page. Only one pair merges, and that pair is the worse failure: the base +station's front view in the left column joined to its cutaway in the right, 328 units +apart, in a crop carrying a fragment of the page's `Примечание.` line and a fragment +of a bullet list, and **still** clipping labels at its right edge. A reader would be +served that instead of two crops that each lose a word. + +**Nothing separates the two.** Sorted by how much of the smaller want box lies inside +the larger, the 35 overlapping pairs run 1.000, 0.965, … 0.148 (page 521's pair), +… 0.063 (page 522's), 0.001, with no gap; page 5's plate pairs sit at 0.173 and above, +*over* the case this is for. The gap between the drawn boxes orders the two the wrong +way round to be a bound — 276 units for the pair that should merge against 328 for the +pair that must not — and any bound between them also admits page 546's 99, page 552's +131 and page 553's 174 and 223, whose merged crops are a drawing plus a paragraph: +page 553's three drawings arrive with a whole three-line note under them and its first +glyph clipped, at 0.256 text. + +**The literal reading of "a shared label field" is the only version with evidence +under it, and it is worth 13 labels.** A run claimed by BOTH figures is 0 on page +521's pair and 0 on page 522's. At two or more it fires on 4 groups, every one on PDF +pages 5 and 6 — the front-matter plates, served only under the neutral-pages opt-in — +and moves nothing on any content page. Its floor is a 2..3 plateau, which mutation +testing found and the sweep did not. + +The whole measurement re-runs in `TestAPlateMergeOnSharedLabelsIsRefused`, on the +variant most favourable to the idea: a merged crop is the union of its members' want +boxes, because re-growing the union instead re-applies the conservative half to the +merged box and page 521 then holds 26 of 40 rather than 40. + +**A CONTENTS PAGE IS READ AS THE LIST OF ENTRIES IT IS**, which it was not: the +columns manual's `Оглавление` arrived as one run-together paragraph of dot leaders, +`Мы поздравляем Вас ...........2 Использование по назначению ......4`, because its +seventeen entries sit at exactly the line pitch and the paragraph rule has nothing +else to separate them by. + +**The signal is a dot leader plus a page reference, and it has a real gap under it** — +which almost nothing else in this package does. Measured over both whole documents, +every run of two or more dots: **3, 3, 3, 4, then 34 to 91, with nothing between.** +The four short ones are ellipses in prose and none carries a page number; the 85 long +ones are the contents entries, 17 per language across five languages. Both halves are +required anyway, because "a leader" and "a page at the end of it" are what an entry +is, and the next document gets no say in which of the two it breaks. + +**The sequential manual cannot trigger it at all**, and that is a property rather +than luck: its longest dot run anywhere is two. Its own contents page sets the page +number in a separate column at x=851 against a title at x=89, with no leader between, +so it needs the tab-stop signal this does not attempt — and it is front matter that +falls outside every language region, so no conversion serves it. + +**It is not a sixth `BlockKind`, and the reason is a cost worth knowing.** A contents +entry IS a list item, the paper prints a list, and the note says which sort — exactly +as it already says `opens with the list marker "•"`. A kind of its own would reach a +database column whose CHECK lists the five by name, and widening a closed set there +costs a table rebuild. For `doc_blocks` that means dropping and recreating `00006`'s +three FTS triggers and reindexing a search table that is external-content over this +table's rowids. Migration `00003` is the precedent and records the procedure. Nothing +here needs it; a later change that wants the kind knows the price. + +Measured: **+16 content blocks per language, +80 over the columns manual's five**, and +coverage does not move — the dots are still in the block's text, only grouped +differently. The reader drops them from the DOM and draws the leader with a rule, +because a row of literal periods is noise to a screen reader. + +**The page number is a link, and the mapping under it is one constant per document.** +`doc_pages.printed_folio` already held the answer, so nothing is stored and no +migration was needed: `DocPageFolioOffsets` groups the document's pages by +`page_no - printed_folio` and `registry.FolioOffset` takes the **mode**, for the same +reason `columnPitch` takes the mode of its line gaps — a few readings that are not +measurements of the thing at all destroy an average. One page of the columns manual +is misread as folio 2735; that alone puts the mean 40 pages out. + +Measured over both manuals' stored pages: + +| manual | pages printing a folio | modal offset | pages agreeing | +|---|---|---|---| +| sequential, 560pp | 558 | **6** | **552** (98.9%) | +| columns, 68pp | 67 | **0** | **65** (97.0%) | + +The runner-up covers exactly one page in each, so the margin is 552-to-1 and 65-to-1, +and every deviation is a short line misread as a folio — two contents pages reading +their own body numbers, two diagram plates reading a callout, a back cover reading +2735. The folios do **not** restart per section: the sequential manual's run +continuously across all 34. + +The mode is offered only if it holds **0.6** of the folio-bearing pages. That floor is +chosen from the failure it exists to catch rather than from the successes: had the 34 +sections each begun again at 1, the best offset would have held 22 of 553 pages — +**4.0%**. The case a bare plurality would get wrong is a document bound in two halves +where the larger is near 50%, so the floor must be above a half; being above a half +also makes the mode unique by construction, so no tie-break policy is needed. It costs +a document that really has one offset but whose folios are read so badly that fewer +than three in five agree — a long way from 6-in-558 and 2-in-67, and the right side to +fail on, because a link to the wrong page is worse than no link. + +`folioOffset` is **absent** from the conversion response where there is no confident +answer, and that is not fussiness: the columns manual's real offset IS 0, so a client +reading a missing field as zero would turn every entry of an unmappable document into +a link to the wrong page. + +Three things keep an entry as plain text, and only the third is interesting: no offset +was served, the line carries no number, or **the target is not a page this language's +conversion holds**. A range entry links to its first page — `Сухая уборка … 15 – 23` +goes to 15. + +Verified in Chrome against the columns manual's own conversion: all **17** German +entries render as links, clicking `Fehlerbehebung 57` scrolls to page 57 and marks it, +and with `folioOffset` withheld all 17 fall back to grey plain text with the printed +number still readable. Forcing offset 2 splits the same list **9 links to 8 plain**, +which is the fallback doing its job in the middle of a list. + +One expectation this refuted: the columns manual prints five languages' contents +pages, so a German entry was expected to be able to name a Russian page. It cannot — +each language's contents page prints **its own** folios, which are its own pages, and +all 17 German and all 17 Ukrainian entries resolve inside their own language. The +containment check stays because it is what makes a misread folio and an out-of-range +target safe, but it does not fire on this document. + +The printed-index parser still needs the same page for a different purpose — see +language-detection.md, where its columns defeat it. + +**No translation, no search, no OCR.** Translation is M3. Search needs an FTS5 table +that does not exist yet — SQLite has the extension compiled in and nothing uses it. +A scanned manual with no text layer needs OCR before any of this applies, and the +tesseract binary is registered but called from nowhere. + +**RIGHT-TO-LEFT TEXT IS EXTRACTED BACKWARDS, and no amount of care in the view can +fix it.** Found by building the reader, and it is a defect in the pipeline rather than +a limitation of it. + +`pdftohtml -xml` — the tool every block, column and region reads from — returns a +right-to-left line in **visual** order. Page 185 of the sequential manual, its Hebrew +section, arrives as `שומיש תולבגה`; reversed rune for rune that is `הגבלות שימוש`, +"usage restrictions", which is what the page prints. `pdftotext` on the same page +returns the logical string correctly, wrapped in the bidi controls U+202B and U+202C. +Arabic is worse: it arrives both reversed and unshaped, in isolated rather than +presentation forms. + +No `dir` value repairs it. The bidi algorithm reorders a strong RTL run under either +base direction, so `dir="rtl"` displays the mirrored letters and `dir="ltr"` displays +the same mirrored letters flush left. + +And reversing in the view would be wrong twice over: it mangles the Latin words and +digits these manuals mix into RTL prose, and it double-reverses the day the extraction +is fixed. **The fix belongs in `internal/doc`** — either reverse an RTL run's runes at +extraction, or take the order from `pdftotext`'s bidi-controlled output. + +**It is now built, at the one place a line's order is decided, and it is measured to +zero.** `internal/doc/bidi.go` reverses a right-to-left line and puts its +left-to-right islands back, so `8` stays `8` and `MopExtend` stays `MopExtend`; that +file's header carries the measurements. Over the whole sequential manual, in the two +steps it took: + +| | before | line's own majority | region's language | +|---|---|---|---| +| pages `verify` reports reversed | 32 | 6 | **0** | +| words absent from `pdftotext` on them | 8,120 | 80 | — | +| ...of those, present when reversed | 7,938 | 18 | **0** | +| `מדריך` found typed forwards | 0 blocks | 4 | **5** | +| `מדריך` found typed backwards | 5 blocks | 1 | **0** | + +The middle column is worth keeping because it is a lesson about authority. Deciding a +line's direction by the majority of its own strong characters looks safe and is not: a +Hebrew line carrying a URL has more Latin than Hebrew, so it was read left to right +and never repaired. Those six lines — the support URL under a Hebrew sentence on page +188 and its Arabic twin on 204, `Dreamehome תייצקלפא` on 191, +`Dreamehome App قيبطت ليزنت` on 207, a Wi-Fi label on 189 and 205 — were the entire +residual. The **region's language** decides now, with the majority as fallback, which +is the right authority: a document-wide answer to a question one line cannot settle, +and establishing it is what the probe is for. + +Two independent checks hold it there, off comparisons sharing no code: +`verify.TestNoTextIsStoredReversed` fails on one word absent forwards and present +backwards, and `registry.TestHebrewIsFoundTypedForwards` fails if the word for +"manual" is findable backwards. + +### The run-level order of a mixed line, which is where both remaining defects live + +Getting a line's DIRECTION right does not get its RUN ORDER right. Three defects lived +here, one after another, each uncovered by fixing the one before it; all three are now +fixed and all three were found by `internal/verify` rather than by reading the code. None +was found by the check that names the defect — see the next section, which is the part of +this worth more than any of the bugs. + +**First: a left-to-right island spanning several runs was reversed.** *Fixed.* Page 204 +prints the support URL + +``` +https://global.dreametech.com/pages/user-manuals-and-faqs +``` + +and poppler cuts it into **seventeen runs**, breaking at every `:`, `/`, `.` and `-` +because the punctuation is set in a different font from the words. Reversing the order +of a line's runs is right for the Arabic prose around it and wrong for those seventeen, +whose visual order already *is* their logical order, so the line came out +`faqs-and- manuals-user/pages/com.dreametech.global://https`. Page 188's Hebrew twin +never showed it, because there the same URL is a single run — the same way the +character-level version of this bug hid from the `pdftotext` comparison. A line is not a +reliable witness to how poppler will cut it up. + +**Second: a run of neutrals belonging to the right-to-left text was treated as part of +the island.** *Fixed.* The fix for the first defect held a maximal stretch of runs with +no right-to-left character in printed order, and `hasRightToLeft` asks whether a run +contains a right-to-left **letter** — so a run of only digits, punctuation or spaces +answered no, and neutrals that take their direction from the surrounding Arabic or Hebrew +were frozen in printed order as though they were left-to-right content. + +Page 211's Arabic maintenance list is the worked example. Its first item is five runs: + +| x | run | +|---|---| +| 728 | `)` | +| 732 | `LDS` | +| 752 | `( رزيللاب ةفاسملا رعشتسم` | +| 850 | ` .` | +| 859 | `1` | + +Only the run at 752 holds a right-to-left letter. So `1` and ` .` formed one island and +kept their printed order, giving `. 1` where the page prints `1.`; `)` and `LDS` formed +another, giving `)LDS` where the page prints `(LDS)`. The block read +`. 1 مستشعر المسافة بالليزر ( )LDS` instead of `1. مستشعر المسافة بالليزر (LDS)`. + +The cost was not cosmetic. `leadingMarker` does not recognise `. 1` as a marker, so the +six printed list items on that page merged into the paragraph above them, and the same +happened on pages 189, 194, 195, 205 and 210: **43 blocks and the list structure of six +Hebrew and Arabic pages.** + +The distinction the run-level test was missing is one `leftToRightIsland` already draws at +character level: a lone neutral does not start an island, and a space joins one only when +the runes on **both** sides do. Lifted to runs, an island is now the part **between the +outermost runs that carry a left-to-right letter**, and a run of only digits, punctuation +or spaces at either end belongs to the right-to-left text beside it and reverses with it. +That keeps the URL's punctuation runs inside their island, because each sits between +`https`, `global` and `com`, and puts `1`, ` .` and `)` back with the Arabic, because +nothing left-to-right stands on the far side of them. Both cases are pinned on their real +geometry in `bidi_internal_test.go`, and the 43 blocks came back page for page. + +**Third: a run kept in printed order was still repaired as though it were reversed.** +*Fixed.* A run the island rule keeps in printed order is already in logical order, and +`joinRunsRightToLeft` passed it through `visualToLogical` anyway. For a run of Latin +letters that is a no-op — the whole string is one island, so reversing it and putting the +island back is identity. For a run of only digits and punctuation it is not, because a +space is an island member only when the runes on both sides are strongly left-to-right. + +Page 204 prints the laser standard `IEC 60825-1:2014/EN 60825-1:2014/A11:2021`, and +poppler cuts it at the font changes into `IEC`, `EN`, `A` and two runs that are pure +digits and punctuation. Those two were correctly held in printed order and still arrived +damaged: + +| run | printed | stored, before | +|---|---|---| +| x=180 | `" 60825-1:2014/"` | `"60825-1:2014/ "` — leading space ends up trailing | +| x=316 | `" 60825- 1:2014/"` | `"1:2014/ 60825- "` — halves swapped | + +In the second, the space in `- 1` has `1` on one side and `-` on the other, so it was not +an island member and became a split point. The line was stored as +`معيار 11:2021 IEC60825-1:2014/ EN1:2014/ 60825- A`; the missing space is the one the +glued-words check reported as `iec|60825`. + +Only a run that **reverses** gets `visualToLogical` now, and the standard reads +`IEC 60825-1:2014/ EN 60825- 1:2014/`. + +#### The stated limit, corrected by the corpus + +A digits-only run at the OUTER edge of an island goes with the right-to-left text and is +emitted outside the island in reading order. `bidi.go` records this as a limit on the +grounds that neither document prints the shape. **It does print it — three times — so the +reason is wrong, though the decision is right.** Found by scanning every line of both +manuals that holds a right-to-left character, 1,136 of them, for a digits-only run +outside the left-to-right span of its stretch with right-to-left text *adjacent* beyond +it. Adjacency matters: grouping a page's runs by their top alone merges a table's label +column with its value column, and `blocks.go` groups lines within a region and a column +group, so those are not one line. With that filter the corpus holds three, all on the +sequential manual and none on the columns one: + +| page | stretch | stored | verdict | +|---|---|---|---| +| 196 | `GHz`, ` `, `5` | `בחיבור Wi-Fi של 5 GHz` | **correct** | +| 205 | `AI`, ` `, `IR`, ` .`, `11` | `11. AI IR كاميرا` | marker leads, `leadingMarker` sees it | +| 205 | `AI`, ` `, `HD`, ` .`, `12` | `12. AI HD كامير` | same | + +The reason the decision survives is sharper than the reason recorded. Putting an outer +digits run outside the island is **right when the digits lead the phrase** — a quantity +like `5 GHz`, or a list marker `11.` — and wrong only when they **trail** a Latin token, +which is the synthetic `A` + `11:2021` shape. All three real instances are the leading +kind, and that is not luck: a trailing number is part of the Latin token beside it and a +printer sets it in the same run, which is exactly what page 204 does — `11:2021`, the tail +of the standard, is inside the **same run** as the Arabic that follows it. So the trailing +case is unreachable at run level there, and separating it would need a mixed run split at +character level, which nothing here does. + +Page 205 is also where `pdftotext` stops being a usable referee: it reads that line +`AI IR11.` with U+202A/U+202C embedding markers around the Latin, so its byte order is not +plain logical order either. Neither tool is clean, which is why the verdict column above +says what the marker does rather than claiming a match. + +### THE CHECK THAT NAMES THIS DEFECT SAW ONE OF THE FOUR BUGS + +This is the most useful thing the whole exercise produced, and it is a statement about +`internal/verify` rather than about bidi. + +Four defects were found in `bidi.go`, all by measurement rather than review. **The check +whose name describes them caught exactly the one that was a reversal, and was structurally +blind to all three that were reorderings.** + +The one it caught is the direction rule: a line whose Latin outweighed its Hebrew was +never repaired, so its words were stored as their own reverse. That is what +`right-to-left-reversed` names, and once it was sharpened to require evidence of a +reversal rather than merely a right-to-left page, it named those six pages and 18 words +precisely. The sharpening earned itself here — see `verify.minReversibleWords`. + +The other three are word ORDER, not word spelling, and the word check compares **set +membership per page** — for the reason `checkText` gives, that a multiset would report a +legitimate difference of one occurrence and a sequence would report the reading order +`checkOrder` is about. So a reordering that preserves the word set is invisible to it *by +construction*: + +| defect | word set | how it actually surfaced | +|---|---|---| +| URL's 17 runs reversed | unchanged — `https`, `global`, `com`, `pages` all present | sideways, as one `join-hyphen-space` | +| list marker `1.` → `. 1` | unchanged — `1` is still on the page | **not at all**; 43 blocks vanished from a pinned count | +| `EN 60825- 1:2014/` → `EN1:2014/ 60825-` | unchanged for the transposition; the lost space merges two tokens | sideways, as `join-glued-words` on `iec|60825` — the side effect, not the defect | + +Read the right-hand column. Two were caught by a *different* check reacting to a side +effect, and one was not reported at all. A `right-to-left-reversed` count of 0 was true +throughout every one of them. + +#### Pin the counts you cannot yet explain + +This is the practice that actually found the damage, and it earned its place twice. + +The block count of the sequential manual is pinned at an exact number with no theory +attached to most of its history — it has been 15,951, 16,055, 16,097, 16,098, 16,055 and +16,098 again. Twice that pin was the only thing standing between a regression and a +release: + +- It **caught** the neutral-run island defect. Nothing else did. 43 blocks disappeared, + the number moved, and the cause was six pages of Arabic and Hebrew list markers turned + round. +- It **confirmed** the repair, and more precisely than a total could: the distribution + came back page for page identical to the earlier reading, which is what says the same + 43 blocks returned rather than 43 different ones appearing somewhere else. + +It also showed why a total is not enough on its own. 16,055 appears twice in that +sequence and means opposite things — once the honest count before any right-to-left +repair, once a regression in which page 194 sat 7 blocks *below* its original. A pinned +number needs its sequence recorded beside it, which is why +`verify_fixture_test.go` carries the whole history in a comment and its failure message +tells the reader to read it before deciding which direction is good. + +`checkOrder` asks the order question of **blocks** and nothing asks it of **words**. That +is the gap, and it is deliberately still open. + +#### What a word-order check would have to compare against + +The reference is usable, and that is the part worth recording, because it is not obvious. +`pdftotext` returns a right-to-left line in logical order wrapped in U+202B…U+202C — so +once those controls are stripped, **the reference's byte order already IS reading order**, +and `tokensMin` strips them today for a different reason (CONTRIBUTING.md records that a +bidi control is not a separator). No bidi algorithm has to be reimplemented to get a +ground-truth sequence; it is already sitting in `Input.Text` unused. + +What makes it real work is everything around that: + +- **Sequence, not set, and per line rather than per page.** A block joins printed lines + that the reference keeps separate, and reflows them, so exact position cannot be + compared. The tractable form is a longest common subsequence over one page's tokens, + reporting **pairs present in both readings whose relative order differs** — `https` + before `global` in the reference and after it in ours; `1` before `.` and after it. +- **Lines have to be matched first.** The reference is per-page text and the defects are + per-line, which is the step that makes this non-trivial and is the honest reason it is + not built. +- **Two legitimate reorderings must not report.** Hyphenation and reflow already cost + `maxInventedShare` its 0.34 rather than 0, and the same disagreements would show up + here as transpositions. Column interleaving is worse: it is a real transposition of + every word between two columns, so a page already reporting `reading-order` must not + also report it as word transposition for the same cause. +- **What it would have bought:** both defects on this page, by name, at the line they + are on, instead of one hyphen finding and one silent count change. + +Until then the honest statement is the one at +`verify.minReversibleWords`: a zero from `right-to-left-reversed` means no word is +spelled backwards, and it does not mean the words are in the right order. + +Worth knowing what none of this breaks: the language signals are unaffected. The +character-repertoire and script signals count characters, so order is irrelevant to +them, and the printed page tag already strips bidi controls for the reason +`stripFormatting` documents. It is the readable text, and therefore search and +translation later, that is wrong. + +**Right-to-left is postponed for the app and built into the reader.** The frontend has +no direction handling at all — no `dir` attribute, no logical properties, every margin +physical — and converting the five existing screens is deliberately not being done. + +The reader is the exception, because for a *new* screen the cost is nil: Tailwind's +logical utilities (`ms-`, `me-`, `ps-`, `pe-`, `text-start`, `text-end`) are the same +length to type as the physical ones, and the block model already carries each block's +language, so setting `dir` from it is one attribute. Writing it that way costs nothing +today and saves rewriting the one screen where direction actually matters — this +document's manuals include Hebrew and Arabic sections. + +## Two corrections to what was already recorded + +Found while measuring, and both concern the column fixture: + +- Its tables are on **pages 52-61, not 57-61**. Pages 52-56 carry genuine small + tables (`Anwendungsfall | Düse/Zubehör`) that the manifest never recorded. +- It prints an **unruled specification table** that nothing had recorded at all, as a + block within its disposal-and-warranty page, once per language: 62 German, 63 + Polish, 65 Ukrainian. + +## What building the first half settled + +**Page furniture IS identified now, per language and across pages.** The printed tab, the +folio and the running head are found by `internal/doc/furniture.go` and no longer served as +content: **172** blocks on the column manual (70 tabs, 41 folios, 61 heads) and **1,289** +on the sequential one (553 tabs, 552 folios, 184 heads). On the sequential manual 471 of +those tabs were arriving as level-2 headings and 533 of the folios as paragraphs. + +**The denominator was the whole problem, and the obvious choice fails.** Counted over the +pages a *household* converted, the German tab is 16 of 59 — 0.27, below any usable cut. +Counted over the pages of *its own language*, 1.00. The rule is per base language. + +The threshold has a real gap under it, which is rare in this codebase: measured over all +39 language sections of both manuals, a tab is on **0.81–1.00** of its language's pages, +and the widest share of anything that is *not* furniture is **0.29** — `Плановое +обслуживание` at 0.27, `Sicherheitshinweise` at 0.25. 0.5 sits 1.7x above the ceiling and +1.6x below the lowest tab with nothing in between. A four-page floor is also needed: on a +two-page section one page out of two is a half, which made ~400 buckets furniture. + +**A folio is confirmed by a second opinion rather than by a share.** The column manual +prints its folio in the outer margin, so German only carries it on 7 of 26 pages. What +replaces the share is `Page.Folio`, which `pdftotext` read through none of this code. + +**Removing the tab makes the heading rule work better.** Level-1 headings on the column +manual *rise* by 29, because an 11pt tab glued onto a heading line was diluting the body +face the rule measures against. Page 14 read `D Trockensaugen` and now reads +`Trockensaugen`; page 57 `D Fehlerbehebung` now `Fehlerbehebung`. + +**Marked in the model, filtered at the save boundary** — `Block.Furniture` plus +`ContentBlocks()`, and `internal/ingest` stores only content. No migration, no change to +`00006`'s search triggers, no filter in the reader or the index. The verifier deliberately +excludes furniture from its coverage sum so that a rule wrongly claiming a *paragraph* +shows up as a drop rather than being invisible; coverage moved 0.974 → 0.973 and 1.000 → +0.997 against a threshold of 0.75. + +**The tab is on 556 of 560 pages of the sequential manual, not 110.** That 110 is quoted +in four shipped files and is wrong in two ways: it undercounts by a factor of five, and it +attributes the repetition to the *column* manual, which has 68 pages. + +**The RUNNING HEAD is clause 3, and what unblocked it was the sequence rather than a +better threshold.** This was recorded here as measured-and-refused, and the refusal was +correct on its own terms: separating a running head from a genuinely repeated heading by +the occupancy of its height is **0.77 on the column manual against 0.63 for a real body +line**, and that cut removes the sequential manual's section titles, because there the +running head *is* the section title, printed identically where the section starts and on +every page after. Twelve points apart with one document on each side. + +What that framing missed is that both of its options were wrong. "Remove them all" loses +the titles, "keep them all" is the defect, and the answer is neither: **the page's first +printed line is a running head when the page before it in the same language section +printed the identical line.** The first page of each consecutive run keeps its title, every +page after it loses one. The sequential manual's Russian prints `Плановое обслуживание` on +pages 528 to 533 with a different grey-pill sub-heading under it on each; page 528 keeps +the title and the other five stop repeating it. No occupancy figure is consulted and no +new constant is introduced. + +Three things had to be got right and each is a measurement: + +- **Consecutive is in the *section's* page order, not the PDF's.** The column manual's + German holds every even page, so its head runs 14-16-18-20-22 with Polish pages in + between; a rule reading PDF adjacency finds no run at all. +- **The position test is a vertical overlap of the two head bands, not a tolerance.** Text + equality alone is not enough — a stock phrase like `Hinweis:` opens a note and slides down + the page — but a tolerance would be another constant to defend. Measured, a real head + moves **0 units on 202 page pairs, 1 on 35, and never more than 8** — the sequential + manual supplies every non-zero, the column manual's 61 are all exactly 0 — against a head + 29–33 units tall in the sequential manual and 19 in the column one. Overlap is + scale-free: the head measures itself in its own type size. +- **One line, not the matching prefix.** How far two consecutive pages agree from the top: + 0 lines on 400 page pairs, 1 on 207, 2 on 38, never 3 with the probe allowed to look 8 + deep. All 38 second lines are one thing — a troubleshooting table's repeated + `Problem | Solution` header on a continuation page, which a reader wants. The named cost + is the column manual's Polish head, two printed lines in one banner: pages 42 and 44 keep + the orphaned `AQUA-Box`. That is an **under-removal**, which is the only direction this + clause can fail in. + +**The column manual does have a running head, and it always did.** Read at 108 dpi, its +grey top banner is `D Trockensaugen` / `PL Odkurzanie na sucho` — the chapter name, on +every page of the chapter. It is claimed on 61 blocks over 20 distinct titles; the +sequential manual on 184 over 77. + +**The false-positive check for clause 3 is an invariant, not a list.** A list would be 97 +titles in 39 languages and would assert only that they had been copied out of a previous +run. Since the first page of every run keeps its head, **every string clause 3 removes must +still be served as content in the same language** — and 0 of the 245 is not. One wrinkle: +`FindFurniture` does not put right-to-left text back into logical order, because furniture +reaches neither a reader nor the search index, while a content block does. So the Hebrew +and Arabic heads are held the two ways round and a naive comparison reports 10 losses that +are not losses. + +**Sequential page 24's pinned reading was itself a defect, and clause 3 found it.** It was +pinned as "one heading and 12 list items, matching its render". Re-read at 108 dpi: page 23 +opens `Sicherheitshinweise` with an introduction and a sub-heading; page 24 reprints the +title at the same place and then 12 more bullets, with no new section on it. What page 24 +prints is 12 list items. The heading it was serving was a third piece of furniture that had +gone unnoticed because it is a real word. + +**One page cannot identify furniture, which is why this pass is where it is.** The printed +`DE` badge comes back as a level-2 heading, the folio as a one-character paragraph, the +running head as a paragraph. Nothing *on a page* separates those from content — the +sequential manual genuinely titles sections `A`, `B` and `C`, and page 24's running head is +set in exactly the face and place page 23's real title is — and what does identify +furniture is repetition across pages, which is a different input than a single region's +runs. + +**A paragraph break cannot always be found.** The gap factor is 1.2 of the measured +line pitch, and on the column manual's page 62 that resolves paragraphs separated by +20-21 units against a 16-unit pitch — but 17-unit gaps occur both inside and between +paragraphs on that same page, so **no factor separates those**, and two paragraphs 18 +units apart stay joined. Chosen as the smaller error: a missed break reads as a long +paragraph, an invented one splits a sentence. + +**The pitch must be the mode, not the median.** Page 62's left column has nearly as +many paragraph breaks as body lines, so its median gap is 18 against a real pitch of +16 — high enough to swallow the 20-21 unit breaks it needs to find. The mode is 16. + +**A heading's share of the measure is a soft cut with no gap to put it in.** Every +candidate's width as a fraction of its column is a smooth continuum from 5% to 100% +on both manuals — 33 candidates at 60-64%, 25 at 65-69%, 116 at 95-99% — and rune +counts are no better. 0.6 is chosen for precision, and its cost is named: a heading +that fills a narrow column reads as a paragraph, which loses `Fehlersuche`, +`Feilsøking` and `Depanare`. + +**Hyphenation is not undone.** `brud-` / `nej` survives as written, because German +legitimately ends a line with a hyphen and rejoining would corrupt those. + +## The integration this leaves, and the thing it explains + +Blocks and cells were built separately and deliberately do not know about each +other. Joining them is the next step, and measuring both halves against page 57 of +the column manual settled what that join has to be — and incidentally explained a +misreading this project has been carrying since the language work. + +**A table's cell dividers are being read as language columns.** Page 57's four +stored language regions and its two tables' cell columns are the same boundaries: + +| stored region | table cell column | +|---|---| +| 36-178, read as Finnish | 29.7-173.3 — table 1's question cells | +| 179-424, German | 173.3-428.1 — table 1's answer cells | +| 457-589, German | 450.2-593.9 — table 2's question cells | +| 601-846, German | 593.9-848.7 — table 2's answer cells | + +Within about five units on all four. And 173.3 is not incidental: it is the only +interior vertical the left table draws, arriving as six segments broken at each row +and recovered only by merging them. + +So that page has **no language columns**. It has two tables, and the column detector +found their cell dividers. That is the root of the one language error both +`layouts.md` and the fixture record — a German cell read as Finnish — one level +below the explanation already written down. Both causes are real and they compound: +the printed `D` in the page's corner is rejected for want of an index vocabulary, +*and* the thing whose language is being asked about is a column of short table +labels rather than a column of prose. + +**Therefore a table's cell BOUNDARIES are excluded from region derivation, not +reconciled with it afterwards.** Joining tables to regions afterwards would be joining +a table to boundaries the table itself created. This touches `regions.go` and is the +one place the two halves genuinely meet. + +*Boundaries, not area* — an earlier draft said area, and that is not implementable +without a migration: a region is one x-range and cannot have a table-sized hole in it. +What is excluded is the table's interior dividers from the set of candidates a page +may divide on, before any region exists. Two guards stop that welding two genuine +language columns a table happens to cross: only dividers inside the table's own box +are candidates, and a gutter is merged away only if a cell divider sits within 1% of +the page width of it — 8.9 units, against the 5.2 by which page 57's widest +coincidence misses. + +Subtracting the area would also have been actively wrong, and by a lot. Pages 58-61 of +the column manual are tables covering nearly the whole measure, and it is their *cell +columns* — all one language — that name the page under rule 3. Remove the table's text +and what is left is a running head, which falls below `minColumnRuns` and loses 3,502, +3,691, 3,668 and 3,339 characters of Polish, Russian, Ukrainian and Kazakh. Four pages +would have lost their language to tidy one. + +**Result, measured end to end on a clean database.** Page 57 goes from four regions — +`fi` at 36-178 and German at 179-424, 457-589, 601-846 — to **one whole-page German +region of 3,618 characters**. With the page's German finally together the alphabet has +`ü×17` and `ß` to read, so it names German confidently. The document goes from 6 +languages to 5; `fi` is gone and nothing replaced it. The gate reports German at +47,932 characters instead of 47,641. The sequential manual's dump is identical: 560 +regions, 0 boxed, 34 languages, every section's pages and spans unchanged. + +Three constraints on that join, all measured rather than reasoned: + +- **It must be geometric, not by key.** Blocks key on `(page, region left edge, + index)` and a table area has no region left edge, so there is no key to join on. +- **Both halves already draw from the same filtered run set** — each calls + `usableRuns` — so cells and blocks can never see text the other cannot. That is + also what keeps both inside what the gate charged for, since a region's character + count comes through the same filter. +- **A heading printed across a table must not be assigned to a cell.** It lands in + the banner group that is read first and is dropped from every cell, so it appears + exactly once and in the right place. An integration that tries to place it in a + cell, or that suppresses banner blocks wherever a table covers the region, makes + it vanish. + +One trap for whoever builds it: page 57 draws a **real vertical at x=440.2 spanning +the full page height**, between the two tables. The cell grid correctly ignores it — +no horizontal rule spans 428.1 to 450.2, so it bounds no cell — but any page-wide +column projection will find it and read the page as split at 440. A convincing +divider that is neither a language boundary nor a table one. + +## What a free verifier found + +`internal/verify` and `manualbox verify` check a conversion against **`pdftotext`, a +completely independent second extraction of the same bytes** — the block pipeline reads +`pdftohtml`, so every page already has a free second opinion produced by different code. +Five checks, every threshold measured against both manuals and quoted at its constant. +No model, no tokens, runs in CI. + +What it reports today: + +| | column manual | sequential manual | +|---|---|---| +| coverage — did we drop content | **0 findings**, median 0.974 | **0 findings**, median 1.000 | +| reading order | **0** | 37 over 26 pages | +| figures clipped | **22 of 46** | **74 of 163** | +| figures with a blank band | 4 | 6 | +| hyphen-space joins | 276 blocks | 72 blocks | +| words absent from the reference | 4 | 160 | +| right-to-left reversed | none, no such script | **none** | + +The last two rows moved together when `bidi.go` landed, and in opposite directions for +one reason. Reversed pages fell from 32 to 6 and then to 0 because the text is no +longer reversed; absent words rose from 153 to 160 because the pages that are Hebrew or +Arabic but *not* reversed stopped being named as pages and are now judged block by block +like every other page. What is left on them is Arabic shaping and combining-mark +disagreement — not a reversal, and not either tool's to fix. + +**A zero on the last row is a weaker statement than it looks**, and the section above +says why: it means no word is stored as its own reverse, not that the words are in the +right order. + +**Coverage is clean on both, which is the reassuring one:** nothing is being silently +dropped. The least-covered page of either manual is 0.801, and that is the +artifact-heavy front matter `usableRuns` deliberately filters. + +**Clipping is far more widespread than it looked.** 22 of 46 figures and 74 of 163 are +cut off by their own crop — the clip-path limitation above, which had been recorded as a +tidiness problem and is in fact the largest visible defect in the output. Cross-checked +against a second, independent signal, whether the render's own paint reaches the crop +edge: 22 of 22 agree on the column manual, 73 of 74 on the sequential one. + +**37 pages are read in columns rather than rows**, all one class: the routine-maintenance +page of each language section lays its intervals out as an **unruled** grid, which the +table detector cannot see. This is the unruled-table gap above, biting for real rather +than hypothetically. + +**Thai is broken in the document, and neither tool is a reference for it.** The check +flagged 142 absent words across pages 473-488. Investigating rather than trusting the +label: `pdftohtml` returns U+FFFD for some Thai vowels — `ข้อมูลด้�นคว�มปลอดภัย` where the +page prints `ข้อมูลด้านความปลอดภัย` — but `pdftotext` breaks *different* characters and +breaks more of them, 34 against 21 on page 480 and 14 against 6 on page 484, and it +mangles `สำาหรับ` into `สำ�หรับ` where `pdftohtml` gets it right. The PDF's Thai font has an +incomplete character mapping and the two tools recover different partial subsets. So +those findings are mostly the two tools disagreeing, not content we lost, and Thai +cannot be repaired by preferring the other tool. It is a property of the document. + +**Two corrections to what is written above, from the same measurements.** The blank band +on page 14 does not reproduce — its two photographs render with 0.0 and 2.5 units of +margin. The real ones are page 46 figure 1, 64 units blank at the foot, and page 40 +figure 0, 36 at the left. And the sequential manual's conversion yields **163** figures +rather than the 229 counted over the whole document, because 7 figure pages fall outside +every region. + +## Acceptance + +Not "it produces blocks". The column manual's German must come back as readable +content in reading order — headings as headings, its troubleshooting tables as tables +with the right cells — from the German column alone, with no Polish, Russian, +Ukrainian or Kazakh text in it. The sequential manual's German section must come back +the same way from a page it owns outright. Both checked against renders of the pages, +not only against counts. + +And the negative: no page of the column manual may contribute text from a language +that was not asked for. That is the funnel's whole promise, and it is the one failure +a reader would notice immediately. + +**Both halves are met, and both were checked against renders rather than against +counts.** Column manual German: 427 content blocks and 53 figures over 26 of 68 pages, with +page 57's two troubleshooting tables arriving as 25 distinct cells and page 14's two +photographs coming back neutral with the same digest in the German and the Polish +conversion. Sequential manual German: 453 content blocks over pages 23-38, and page 24 +compared against its render matches bullet for bullet — one heading and **12 list +items against 12 printed**. Sequential Russian: 445 content blocks and its 65 figures over +pages 517-538, page 533's eight line drawings among them. + +The two blemishes on that page 24 comparison are the documented page-furniture +limitation, not new: the printed `DE` badge arrives as a level-2 heading and the folio +`18` as a paragraph. Nothing on one page separates those from content. + +## What wiring it to the pipeline settled + +Conversion now runs as a job behind an approval, and three things only became +measurable once it did. + +**The job pays for the probe a second time, and the cost section above understates +it.** That section prices a conversion as "one `pdftohtml` pass over the whole +document" because `Result` does not carry the runs. True, but incomplete: the +handler does not have a `Result` at all. The probe stored its findings as rows, and +`Convert` needs the object, so the job re-runs `Analyze`. Measured end to end +through the API, on a clean database: + +| | analyze | convert | job, start to finish | +|---|---|---|---| +| sequential manual, Russian | 3.78 s | 14.58 s | **18.6 s** | +| column manual, German + Ukrainian | 8.11 s | 17.07 s | **25.4 s** | + +So re-reading is 20% of one job and 32% of the other. It is bought deliberately +rather than cached: `Analyze` is a pure function of the bytes, which is what makes +the probe idempotent, and rebuilding a `Result` from `doc_pages` and `doc_regions` +would be a second implementation of the same object, free to drift from the real one +in ways nothing compares. The alternative worth having later is storing the `Result` +whole, not reconstructing it. + +**A figure's language is derived on read, not stored.** `doc_figures` has no language +column, which is the contract — a picture belonging to no language belongs to every +language — and that is exactly why a household reading two languages cannot be served +by page. The de+uk conversion of the column manual stores 54 figures; page-scoped +filtering would hand a German reader the Ukrainian column's picture off every shared +page. Applying the same geometric test `Convert` used, against the same stored +regions, gives German **53** and Ukrainian **52**, overlapping in the 51 neutral ones +— including page 14's two photographs, which arrive with identical digests in both. + +**The state has to be the transaction's, and reverting it proves so.** Setting the +document to `ready` on its own handle before `SaveConversion`'s transaction leaves a +document claiming to be readable after a save that rolled back — checked by making a +block violate `page >= 1` and watching the row say `ready` with no blocks behind it. +Calling `SetDocumentState` from *inside* the transaction is the deadlock the +`saveFigures` header already measured; the state therefore travels as a parameter and +is written on the transaction's own handle, last. + +Both fixtures came back at the numbers above through the real API: 432 German blocks +with page 57's two tables as 25 cells, and 445 Russian blocks with 65 figures over +pages 517-538, page 533's eight among them. Re-approving a `ready` document produced +byte-identical JSON. diff --git a/docs/design/ingest.md b/docs/design/ingest.md index 7c98e8a..c944267 100644 --- a/docs/design/ingest.md +++ b/docs/design/ingest.md @@ -45,15 +45,30 @@ upload should never silently become a bill. `pdftotext` over all 560 pages took **1.7 s** and produced 1.3 MB of text. The output answers the question that determines everything downstream: **is there -a text layer?** Median extracted characters per page here was 2,241; a scan yields +a text layer?** Median extracted characters per page here is 1,691; a scan yields ~0. That single number selects between a free extraction path and one that costs a vision call per page — a difference of two orders of magnitude. +Count runes, not bytes. The same document measures 2,240 median *bytes* per page, +a third higher, because most of it is Cyrillic, Greek, Hebrew, Arabic or CJK. A +byte-based threshold would judge a Russian page as carrying more text than an +English one containing the same amount of writing. + ### Stage 2 — the language map. Free. -Two independent methods, because **each one is wrong in a way the other catches.** +Several independent signals, because **each one is wrong in a way the others +catch.** The signals, what each costs, and the measurements behind choosing +between them are in **[language-detection.md](language-detection.md)**. The two +that matter most on this document: + +**The printed page tag.** Every content page of this manual prints its own language +code in a corner tab, and it arrives as the first line of the `pdftotext` output +stage 1 already produced — so reading it costs nothing at all. On this document it +labels 553 of 553 content pages correctly, including the two sections a statistical +detector cannot get right. Not every manual prints one, which is why it is the +cheapest signal rather than the only one. -**The printed index.** Pages 2–3 carry a machine-readable contents table. A regex +**The printed index.** Pages 2–4 carry a machine-readable contents table. A regex recovered all 34 sections — ISO code, localised title, start page: ``` @@ -70,8 +85,13 @@ Neither is sufficient alone, and this was measured, not assumed: | The index gets wrong | The detector gets wrong | |---|---| -| **`CZ p.207` is a typo.** Page 207 is Arabic; Czech actually starts at printed 305. | **Indonesian never detected.** Its function words are identical to Malay, so `ID` pages classify as `MS`. | -| **The printed→PDF offset drifts**: +6 at the front, +8 later, because some sections run 17 pages rather than 16. A single global offset lands in the wrong language. | **Danish/Norwegian and Slovak/Czech flip-flop** mid-section for the same reason. | +| **`CZ p.207` is a typo.** Page 207 is Arabic; Czech actually starts at printed 307. | **Indonesian never detected.** Its function words are identical to Malay, so `ID` pages classify as `MS`. | +| **The index's claimed printed pages drift** 1–2 pages from the folio actually printed, on 10 of 34 sections, because `IT` and `PL` run 17 pages rather than 16. Trusting a claimed start lands in the wrong language. | **Danish/Norwegian and Slovak/Czech flip-flop** mid-section for the same reason. **Latin-script Serbian** is read as Croatian or Bosnian on all 16 of its pages, and **Uzbek cannot be labelled at all** — `lingua-go` does not support it. | + +The printed→PDF *offset* itself does not drift: it is a constant +6 across all 34 +sections, because six pages of front matter precede the content. What drifts is +what the index **claims**, which is a different failure and the reason a claimed +start is a hypothesis rather than a boundary. Reconciled, 32 of 34 sections agree and each disagreement is caught: @@ -84,7 +104,7 @@ its provenance. ### Stage 3 — scope, behind a gate. Intersect the languages found with the household's configured languages. For -`de, uk, en`: **48 of 560 pages, 9.6%**. +`de, uk, en`: **48 of 560 pages, 8.6%**. Then ask, before spending anything: @@ -98,6 +118,30 @@ the plan's window for a subscription, nothing at all for a local model. The estimate comes from `count_tokens`, not a character heuristic, so the number shown is the number spent. +**The gate has a second scope, and it is the funnel's own blind spot.** Intersecting +with the household's languages means the pages belonging to *no* language are never +converted. Usually that is right — a cover, a colophon. On the sequential fixture it +is not: PDF page 5 is an exploded parts diagram, **31 places in the content pages say +"see A-1"**, and page 5 falls inside no language region. So the gate also offers the +pages no language claims, with **how many pictures are on them**, and lets the user +include them: + +> Also import 7 pages that belong to no language — 61 pictures on them, and 1,639 +> characters. Pages 1–6 and 560. + +Off by default; including them converts them and nothing else changes. The picture +count is there rather than only characters because characters rank the two fixtures' +sets in exactly the wrong order — the *worthless* set has seven times the text. The +whole measurement, and why this is a tenth of the page picker rather than the whole +of it, is in [conversion.md](conversion.md). + +**Approving still means the scope the gate showed.** `POST /approve` takes no language +argument — the languages are configuration and the gate rendered them from it. The one +thing a caller may send is a yes or no to the offer above, and even then it cannot name +a page: the set is recomputed from the stored region map, and the decision is stored on +the document rather than carried in the job payload, so the handler takes its whole +scope from stored state. + ### Stage 4 — the model, on the slice only. Convert, translate, and extract across 48 pages instead of 560. @@ -124,8 +168,8 @@ costs. ## Test fixture -`testdata/fixtures/l40-ultra.json` records this document's URL, checksum, page -count, and the full expected language map. +`testdata/fixtures/dreame-l40-ultra.json` records this document's URL, checksum, +page count, and the full expected language map. **The PDF itself is deliberately not committed.** It is 15 MB, and it is someone else's copyrighted manual — committing it would break the project's own rule diff --git a/docs/design/language-detection.md b/docs/design/language-detection.md new file mode 100644 index 0000000..d1f0de3 --- /dev/null +++ b/docs/design/language-detection.md @@ -0,0 +1,318 @@ +# Working out what language a page is in + +A multi-language manual has to be split into language runs before anything else +can happen: it decides what gets converted, what gets translated, and what the +user is asked to pay for. See [ingest.md](ingest.md) for where this sits in the +funnel. + +There are five signals. None of them is authoritative on its own, and the whole +design is about combining them and recording which one spoke. + +> **Every number below was measured on a real document, and there are only two of +> them** — the Dreame L40 Ultra (560 pages, 34 languages in sequential sections) +> and the Thomas DryBox Amfibia (68 pages, 5 languages in parallel columns). +> Signals 1–4 were measured on the first, signal 5 on the second. Two manuals are +> not a corpus: treat the numbers as real but not general, and see the open +> question at the end. + +## The five signals + +| | Signal | Cost | Gives | Fails when | +|---|---|---|---|---| +| 1 | **Printed page tag** | free | label **and** boundary, per page | the manual doesn't print one | +| 2 | **Printed index** | free | labels, section titles, claimed starts | claims are wrong or typo'd | +| 3 | **Unicode script** | free | narrows the candidate set | 25 languages share Latin | +| 4 | **Statistical detection** | a dependency | a label per page of text | sibling languages, unsupported languages | +| 5 | **Character repertoire** | free | a language, from the alphabet used | the alphabet is shared, or plain ASCII | + +Numbered in the order they were built, not by price: signal 5 costs nothing and +belongs with 1–3. + +### 1. The printed page tag + +Many manuals print a small tab in a page corner containing that page's own +language code. On the L40 it is white text in the top-left, and — usefully — it +is the **first non-blank line of plain `pdftotext` output**, so reading it costs +nothing beyond the text extraction that already happens. + +Measured: 553 of 553 content pages carried a tag, and all 553 agreed with the +corrected section map. It labelled the two sections statistical detection cannot +(see below). It is the single best signal when present. + +**It is not present in every manual**, so it is a high-confidence input rather +than the answer. Two guards are required, both of which came from real failures +on this document: + +- **Contents pages produce false positives.** Pages 2–4 list language codes in the + same corner, yielding spurious single-page runs for `EN`, `MS` and `RO`. + Requiring a run of **≥2 consecutive pages** removes all three and leaves exactly + the 34 real sections. +- **A two-letter uppercase token is not necessarily a language code.** `ON`, `OK`, + `NO` and `TV` all match `[A-Z]{2}`. Cross-check the tag against the page's + dominant Unicode script before trusting it. + +### 2. The printed index + +Recovers labels and localised titles for every section, which detection cannot do +at all. Its *claimed page numbers* are unreliable: on the L40, 10 of 34 sections +claim a printed page 1–2 off from the folio actually printed, because two sections +run 17 pages rather than 16. So a claimed start is a hypothesis, never a boundary. + +**The parser could not read the Thomas manual's contents page, and read its back +page instead.** That manual's contents are pages 2 and 3, one column per language, +with title-and-dot-leader entries rather than the code/title/page triples the parser +expects. It recovered nothing from them. What it did read — the only page of that +document it read at all — was page 68, a page of service addresses for six countries, +which it took for the contents table: + +| entry | scraped from | title it invented | claimed | +|---|---|---|---| +| `VIA` | *Via Monte Rosa* | "Monte Rosa" | pages 28–45 | +| `FAX` | a fax label | | pages 46–48 | +| `UA` | a Ukrainian postal address | "Telefax" | pages 49–68 | +| `Z` | *Sp. z o.o.* | "o.o. Telefon" | unplaceable | +| `NDE` | *Neunkirchen* | | page 4931 | + +Reconciliation trusted those claims, so a five-language manual reported two languages, +neither of them right, across more than half the document. The user-facing sentence +was *"68 pages in 2 languages, none of them yours. It has fax and Ukrainian."* + +**Half of this is fixed.** An index entry's token must now name a language something +recognises, not merely be shaped like one — the same rule regions.md states, applied +one layer earlier, and here the decision it gates is whether the page is a contents +table at all. `VIA`, `Z`, `NDE` and `GA` name nothing; `FAX` parses as the language +`fax`, which is not among the languages that appear in appliance manuals. Only `UA` +survives, one entry against a floor of three, so the page stops being a contents table +and the fabrication disappears. The sectioned manual is untouched: 34 entries, 34 with +titles. + +**Half is not.** The columnar contents pages still cannot be parsed, so no vocabulary +is recovered from that document at all — and a single-letter printed tab is believed +only where the index lists that code. `D` is not listed, so every German column falls +back to its alphabet and tag-named columns stay at 53 of 169 instead of 79. The total +named barely changes, which is why nothing failed loudly; only the attribution moved. + +The 79 figure is what the commit introducing per-column naming recorded, measured with +a hand-supplied code list rather than through the assembled pipeline. That gap is +pinned by a test so it stays visible, and it has one further cost: the document's one +language error, a German table cell read as Finnish on page 57, happens because the +`D` printed in that page's own corner is rejected for want of the vocabulary. Fix the +parser and that misread goes with it — they are one bug, not two. + +### 3. Unicode script + +Free, and settles more than it looks. On the L40 it resolved 151 of 554 pages +(27%) and uniquely identified six languages — Greek, Hebrew, Arabic, Thai, +Chinese and Japanese (the last two separated by the presence of kana). Cyrillic +narrowed to the three that document contains, out of the seven the script table +lists. + +It cannot help with the remaining 403 pages, which span 25 Latin-script +languages. That residue is what signal 4 exists for. + +### 4. Statistical detection + +`lingua-go` v1.4.0 was the candidate. Measured across all 554 labelled pages: + +| Configuration | Peak RSS | Accuracy | Speed | +|---|---|---|---| +| lazy, 75 languages, **low** accuracy | 130 MB | **93.7%** | 7.3 ms/page | +| lazy, 75 languages, high accuracy | 129 MB | **93.7%** | 8.0 ms/page | +| **preloaded**, 75 languages, high accuracy | **2154 MB** | — | — | +| preloaded, 3 languages, low accuracy | 13 MB | — | — | + +Four things follow, and they are the reason this page exists: + +**Never call `WithPreloadedLanguageModels()`.** It is a 2 GB resident-set footgun +on a machine that may be a NAS, and it buys nothing — see the next point. + +**High-accuracy mode is not worth it here.** Identical 93.7%. Manual pages average +~1700 characters, and lingua's advantage is on short strings. Use low-accuracy +mode: same result, less memory, faster. + +**The binary cost is +118 MB and cannot be avoided.** Linking `lingua-go` takes +the manualbox binary from 11.7 MB to 129.5 MB, because the language models are +`go:embed`ded as a whole directory. Referencing only three languages does *not* +prune them — a build that names German, English and Ukrainian is still 129.5 MB. +128 MB of the binary is `runtime.rodata`. This is the real price, and it is paid +in the Docker image too. + +**Accuracy tops out around 94%, and the residue is systematic, not random:** + +- **Uzbek: 0 of 16 pages.** `lingua-go` does not support Uzbek at all. No + configuration fixes this; the language is simply absent. Detected as + Azerbaijani on 14 pages. +- **Serbian: 0 of 16 pages.** This manual's Serbian is Latin script, which is + near-identical to Croatian and Bosnian. Detected as `hr` or `bs` throughout. +- Japanese 20/22, Czech 15/16 — isolated pages, not systematic. + +Also worth knowing: `lingua-go` has no `no` macrolanguage, only Bokmål and +Nynorsk, so a code map must translate `no → nb`. It covers 32 of this document's +34 languages. + +### 5. Character repertoire + +Languages that share a script do not share an *alphabet*. Signal 3 narrows a +Cyrillic page to seven candidates and stops — that is everything a script table +knows — but the letters only some of those seven can write are already sitting in +the text stage 1 extracted, and counting them costs nothing. + +The case that forced this comes from the *second* fixture, not the L40: 19 pages +of the Thomas manual carry three Cyrillic languages side by side, one per column. +The L40 cannot show this — its languages are sequential, one per page. See +[layouts.md](layouts.md). Counting each language's distinctive letters per column: + +| column | Ukrainian marks | Russian marks | Kazakh marks | verdict | +|---|---|---|---|---| +| left | 0 | 40 | 0 | Russian | +| middle | 83 | 0 | 0 | Ukrainian | +| right | 78 | 111 | 143 | Kazakh | + +Consistent across 18 of the 19 such pages. The exception is a page of contact +addresses, which is not content. + +**The right column is why a maximum over those counts is the wrong reading.** +Kazakh's alphabet *contains* the і it shares with Ukrainian and the ы it shares +with Russian, so overlapping counts are the normal case rather than a conflict. +Two questions decide it instead: + +- **Can this language write everything on the page?** Russian cannot account for + 67% of the right column and Ukrainian cannot account for 76%, so both are out — + ruled out by what they *cannot* write, not out-voted. +- **Does the page exercise this language?** On the *left* column Kazakh can + account for everything, because Russian's alphabet is a subset of Kazakh's. What + settles it is that none of Kazakh's own nine letters appear. + +Both are needed. Either one alone gets the left column wrong. + +**Call this per column, not per page.** A page holding three languages is not text +written by one, so the honest answer for the whole page is nothing — and that is +what it gives: three columns of ordinary Cyrillic manual prose, concatenated, +decline correctly, as does any pairing of them. + +The reason to state it as a rule anyway is that the margin is narrower than the +clean result suggests. On the measured page Kazakh already accounts for 422 of 455 +marks — 7.25% foreign against a 5% threshold — because its alphabet contains most +of what Russian and Ukrainian can write. The guard holds because real Ukrainian +prose uses ї and є often enough to contradict it. Constructed text where one +language's own letters are unusually sparse crosses the line and is named +confidently, which is how this paragraph came to be written: the first version of +this warning claimed a realistic page fails, on the strength of a sample that +repeated one thin sentence. It does not. The margin is a property of the text, not +of a constant, so no threshold fixes it — calling it per column removes the +question instead. + +**Cost, measured.** Linking it adds **1,536 bytes** to the binary (18,475,698 → +18,477,234) and it needs no dependency, no model and no network. A 1,932-rune +page takes **100 µs** on an M2 Pro, of which 53 µs is the Unicode-script pass +signal 3 already runs — 57 ms for all 560 pages, against 1.7 s for the +`pdftotext` that produced the text and 4.0 s for `lingua-go` across the 554 it +was measured on. + +**Coverage.** 34 table entries over 33 languages and two scripts: seven Cyrillic +(ru, uk, be, bg, sr, mk, kk) and 27 Latin. Serbian appears in both, because it is +written in both. On 31 paragraphs of ordinary manual copy — one per language, +hermetic, no PDF — 25 were named correctly, six were declined (four carrying no +distinctive character at all, two as declared ties), and none was named wrongly. + +**Accuracy on a real document, and it is lower than the hermetic figure.** The 31 +paragraphs above are one clean paragraph per language. Measured instead against the +L40's own printed page tab — which is correct on all 553 of its content pages, so it +is ground truth — over the 685 columns of that manual where the signal named a +language at all: **93% correct**. The 7% are largely the sibling groups below, which +that document has in quantity, plus short table cells. + +Two things follow. First, the signal earns its place as a corroborator and a +fallback, not as an authority: where a printed tab exists it must win, which is what +regions.md rule 1 does. Second, and less obvious, **the errors do not correlate with +how much evidence the signal had.** Bucketed by distinctive-character count, accuracy +is flat at every cut from 1 to 50 marks, and one wrong naming carries 118. A +minimum-evidence threshold was designed against this measurement and abandoned by it. +That is worth recording precisely because it is the intuitive fix. + +**What it cannot do, named.** Three groups have byte-identical repertoires, and +the signal reports them tied rather than choosing: + +| tied | why | +|---|---| +| `da` `no` | both æ ø å é. Bokmål and Nynorsk share it too | +| `bs` `hr` `sr` in Latin script | all č ć đ š ž | +| `en` `id` `ms` | nothing outside a-z: invisible, all three equally | + +A second, quieter weakness is one-way rather than symmetric. Where one alphabet +is a strict subset of another — `bg` ⊂ `ru` ⊂ `kk`, `fi` ⊂ `sv`, `sl` ⊂ `hr`, +`nl` ⊂ `fr` — the smaller language wins by exercising all of itself, and it is +only *right* when the text is long enough that the larger one's extra letters +would have shown up. On a heading or a caption it is a coin toss dressed as an +answer, so the runner-up is always returned ranked beneath the winner rather than +discarded, and a floor of three distinctive characters stops one brand name being +read as a language at all. + +**Czech and Slovak are not on that list, and the usual expectation is wrong +here.** They are the standard example of a pair a trigram detector flip-flops on +mid-section, and by repertoire they separate cleanly: Czech ř, ě and ů and Slovak +ľ, ĺ, ŕ, ä and ô are frequent enough in ordinary prose that each contradicts the +other outright. Measured on a paragraph each, Czech is not even admitted as a +candidate for the Slovak text. + +**It does not replace signal 4, and it is not an argument for reopening that +decision.** It reads alphabets, not language: it says nothing about the 25 +Latin-script languages when a page happens to carry no diacritic, and it cannot +tell Indonesian from Malay or English at all. It does dent two of the systematic +failures recorded above, in different ways and neither completely: + +- **Latin-script Serbian** — `lingua-go` scored 0 of 16 pages and answered `hr` or + `bs` with confidence. This signal narrows the same pages from 25 candidates to + exactly three and refuses to pick. That is not a label, but a declared tie is + worth more than a confident wrong answer, and the printed index or the page tag + breaks it. +- **Uzbek** — untouched. `lingua-go` has no Uzbek and neither does this table. + +**Status: implemented, not wired in.** `internal/doc/repertoire.go` is a pure +function with its own tests; nothing calls it from `Analyze` or `Reconcile` yet, +so it changes no stored row and no reconciled outcome. + +## Why detection is still needed + +The page tag worked perfectly on the one manual measured, which is a weak reason +to drop signal 4. Manuals vary enormously in how they are produced, and a signal +that depends on a publisher's layout convention will be absent or different often +enough that it cannot be the only mechanism. The tag is a cheap, high-confidence +*shortcut* — when it is there, take it; when it is not, something has to still +work. + +## How the signals combine + +The rule from [ingest.md](ingest.md) generalises once there are four sources +rather than two: + +> Prefer the cheapest signal that is present. Corroborate it with the next. +> Record every source, its confidence, and its provenance. Where sources +> conflict, surface the conflict — never silently resolve it. + +`doc_langs` stores one row per run per source, so *"the tag says DA, the index +says FI"* is a reportable state rather than a coin toss. That is also what makes a +later, better detector a drop-in addition rather than a rewrite. + +## Open question + +**Every number here comes from one document.** The L40 happens to have a page tag, +a machine-readable index, and a clean text layer. A real library will contain +manuals with none of those. + +So the detector decision — whether `lingua-go`'s +118 MB is worth paying, or +whether a lighter library such as `whatlanggo` (~100 KB, script plus trigram) gets +close enough — is **deliberately deferred until there is a corpus to measure +against**, rather than settled on a sample of one. + +What to measure when that corpus exists: + +- How many manuals print a per-page language tag at all, and in which corner. +- Accuracy of each signal per manual, not averaged across them. +- Whether the sibling-language failures (SR/HR/BS, ID/MS, DA/NB, SK/CS) are + common enough in practice to need the printed index as a tiebreak. +- `whatlanggo` on the same pages, against the same ground truth, before accepting + a 10× binary. + +Until then the pipeline is built on signals 1–3, which need no dependency, and +signal 4 is an interface with no implementation behind it. diff --git a/docs/design/layouts.md b/docs/design/layouts.md new file mode 100644 index 0000000..cec8bae --- /dev/null +++ b/docs/design/layouts.md @@ -0,0 +1,242 @@ +# Manuals are not laid out the same way, and the pipeline has to know it + +The ingest pipeline was designed against one document and worked perfectly on it. +The second real manual broke it comprehensively. This page records what differs, +what does not, and where the seam goes. + +Two ordinary consumer-appliance manuals: + +| | Dreame L40 Ultra | Thomas DryBox Amfibia | +|---|---|---| +| Pages | 560 | 68 | +| Languages | 34 | 5 — German, Polish, Russian, Ukrainian, Kazakh | +| Arrangement | sequential sections, 16 pages each | parallel columns, and not uniformly | +| A page holds | one language | one to three columns | +| Tagged PDF | yes | no | + +Neither is exotic. The second is how European multi-language manuals are routinely +printed. + +## What the second manual did to a pipeline built for the first + +``` +found 1 language (there are 5) +56 of 68 pages unlabelled +content range 49-66 (the content is the whole document) +page-tag runs: 0 +scope for de,uk: 11 of 68 pages, 16% +``` + +Four causes, each a design assumption rather than a bug: + +**The page is the wrong unit.** Languages sit side by side, so a language run +defined as a contiguous span of pages cannot express "German on the left, Polish +on the right". You cannot skip a page to skip a language. + +**Language changes every page.** So `minTagRunPages = 2` — the guard that stops a +contents page becoming a section — deletes the real signal here. + +**Printed codes are one and three letters.** This manual marks its languages `D`, +`PL`, `RUS`, `UA`, `KAZ`. `looksLikeLanguageCode` requires exactly two, so it can +read two of the five. + +**Script cannot separate German from Polish**, both Latin and on the same page. + +One thing worked: `Unlabelled` reported 56. Under its earlier definition — bounded +by a content range derived from the labelled runs — it would have reported 0 and +the document would have looked fine. + +## The column geometry, measured + +Text columns per page, from `internal/doc/columns.go`: + +| Text columns | Pages | +|---|---| +| 0 | 1 | +| 1 | 3 | +| 2 | 31 | +| 3 | 28 | +| 4 | 5 | + +Column *widths* vary within the document — 262px on the three-column spreads, +403px on the wide two-column ones — so nothing may assume a fixed width, count or +pitch. + +### The sectioned manual has columns too, and that was assumed away + +The table above says a Dreame page holds one language, which is true, and it was +read as also meaning one column, which is false. Measured over all 560 pages once +positioned text could be extracted from it: + +| Text columns | Pages | +|---|---| +| 0 | 6 | +| 1 | 148 | +| 2 | 136 | +| 3 | 199 | +| 4 | 71 | + +A test asserting this manual was single-column was written and failed. Pages 20 and +100 rendered at `pdftoppm -r 108` settle what the numbers could not: both are two +side-by-side troubleshooting tables, and the regions returned are the tables' cells, +correctly located. On page 20 only the two wide answer cells come back, the narrow +question cells falling below `minColumnRuns` — that guard working, not failing. + +So the assumption was wrong and the code was right. **Column count is not language +count, in both directions**: on the Thomas manual one page holds several languages, +and on the Dreame manual one language is set across several table cells. Any rule +keyed on how many columns a page has is wrong on one of these two documents. + +An earlier version of this page published 11/16/40/1 for the same document. That +was wrong: it came from an ad-hoc script splitting at gaps wider than 90px, and the +real gutters here are 9 to 17px. Three approaches were needed before the numbers +held, and each failure is a trap worth keeping: + +**Whitespace projection is binary, so one run welds two columns for ever.** A +heading set across the measure is enough. The fix is to count how many runs *cross* +each x and tolerate a few: a gutter is a band few runs cross, not none. Page 63 has +exactly one spanning run, page 68 has two. + +**Left-alignment peaks over-split**, because alignment is a local statistic. Page +13 gives six peaks for three columns, each column having a hanging indent for its +numbered markers; page 63 gives a spurious peak 162px into the left column from a +nested list. No fixed "merge peaks closer than N" rule separates a 30px hanging +indent from a 162px sub-indent while keeping two real columns 280px apart. Crossing +count is page-wide, which is what a column boundary actually is. + +**The text layer contains things that are not on the page**, and both kinds had to +be filtered before any geometry worked: + +- *Production artifacts.* An InDesign filename slug and an export timestamp, 261 + occurrences each across 67 of 68 pages, 8% of all runs, several sitting in + gutters. The obvious filter is wrong in both directions: "repeats across pages" + also matches the printed `UA` and `PL` tags, while "repeats within a page" misses + pages 6 and 41, which carry only two copies each, and would delete 742 of page + 68's 769 genuine runs, since that page legitimately prints a company name a dozen + times. The discriminator is **height** — 522 runs at 2–6px against a body median + of 17, being leftovers scaled down with placed artwork. +- *Off-page runs.* Page 68 parks 218 runs at negative coordinates, invisible in + print, lying across two gutters. Filtering to the page box is the only filter + that changes a column count. + +## Figure callouts are not columns + +A diagram's numbered callouts cluster like text. What separates them is how much +text a candidate column holds: real columns carry 1,116–3,058 characters, the +callouts on one exploded diagram carry 12 and 24. A parts list with short lines +sits between at 1,716, so a character count separates them where a median line +length would not. + +## The seam: a geometry pass, then an assignment rule + +An earlier draft proposed a `Layout` interface with an implementation per +arrangement, chosen per document by a scored `Detect`. That was wrong twice over. +A document contains several arrangements, so a per-file choice is confidently wrong +on every page it does not fit — and the interface bundled two things that fail +separately. + +What exists instead: + +**A geometry pass that knows nothing about language.** `DetectColumns` takes a +page's text runs and returns its columns. It is measured, testable alone, and +correct on all eight pages verified against renders. Whether a document is +"sectioned" or "parallel-column" is not a decision it makes — a sectioned page is +one column, and that falls out rather than being classified. + +**An assignment rule above it**, deciding what a column *is*. That is where the +language signals attach, where a table cell must be told from a text column, and +where an honest refusal belongs. + +The practical payoff of splitting them: the geometry pass shipped and was verified +before the assignment question was answered, and a failure in one is diagnosable +separately from a failure in the other. + +The unit flowing downstream is a region — a page, a box, a language, a source. The +box is what makes the second manual expressible at all. + +One thing changed from this sketch when it was built: the box for a sectioned manual +is not *absent* but spans zero to the page width. An absent box would put a null +check in every caller; a full-width one means a reader clipping text to the box gets +the whole page and needs no special case. See regions.md. + +## Detecting arrangement from the printed index + +The contents table gives it away cheaply. If several entries point at the same +page, the languages must share pages: + +| | Index entries | Distinct targets | Claimed more than once | +|---|---|---|---| +| Dreame (sectioned) | 34 | 34 | 0 | +| Thomas (columns) | 87 | 34 | 14 | + +**Provenance, because it matters here:** the Thomas numbers come from an ad-hoc +script counting lines that end in a page number, not from the shipping index +parser — which requires a two-letter code and could not produce 87 entries in a +five-language manual. The contrast is stark and the signal is real, but the +measurement has not been reproduced by the code that would rely on it. + +Nor is it a general rule. A manual numbering each section from 1, or a +chapter-level contents table in a monolingual document, produces duplicate targets +with no columns at all. Corroboration, not classifier. + +## Failing honestly, without regressing + +An earlier draft made "unclassified" an implementation that labels nothing. That is +a regression rather than a safe fallback: a sectioned manual with page tags and no +contents table is labelled correctly today, and under that design nothing would +score, so it would produce zero languages. **Layout classification must never veto +a stronger signal that is already present.** + +So the fallback is current behaviour — whole-page regions, labelled as they are +now — carrying an unclassified flag. What the flag drives is a *route*, not a +refusal: keep the original untouched, process with the local pipeline, or ask a +model. A document this pipeline finds hard should say so and offer the choice +rather than return a confident wrong answer. + +Whether a model is in fact more reliable on the hard cases is untested. It is +plausible, and it is measurable — two manuals now have recorded ground truth — so +it should be measured before it is offered as the better route. + +## What is still not understood + +**Geometry cannot tell a table cell from a text column.** Pages 57–61 are +troubleshooting tables: two side-by-side tables of two cells each, which is why the +distribution above has five four-column pages. This belongs above the geometry pass, +and it is now answered there — not by learning to recognise a table, but by never +asking. A page divides only where its columns name more than one *language*, so a +table of same-language cells is one region however many cells it has. That disposes +of the case on both manuals at once: Thomas's pages 57–61 and the 406 Dreame pages +that read as two or more columns. + +What remains unsolved is a table whose cells are in *different* languages, which +would divide on language and be wrong to. Neither manual does it. It is recorded +here rather than guarded against, because a guard would be written against an +imagined document. + +The narrow-cell language error survives: a cell of German read as Finnish, sharing +only ä and ö, which gives the repertoire signal too little to discriminate. An +attempt to fix it by requiring more evidence failed on measurement — over 685 +labelled columns the signal is 93% accurate and its mistakes are spread across every +amount of evidence, including one with 118 distinctive characters. There is no +threshold to find. See language-detection.md. + +**Everything here generalises from two documents**, one of which took three +attempts to measure correctly. The numbers are real; their generality is not. + +## Status + +Built and verified: the geometry pass, `internal/doc/columns.go`, correct on all +eight pages checked against renders — and now checked against runs extracted from +the real PDF by `internal/doc/runs.go` rather than typed into a test, which is what +makes that verification non-circular. + +Built: the region model and the assignment rule, `internal/doc/regions.go`. The +column manual's five languages read back across its parallel columns; the sectioned +manual produces exactly one whole-page region per page and its language map is +unchanged. + +Designed, not built: the routing by complexity. + +`testdata/fixtures/thomas-drybox-amfibia.json` records per-page ground truth with +provenance — eight pages verified by eye, the remainder marked as detector output, +so that nothing is ever tested against its own answer. diff --git a/docs/design/regions.md b/docs/design/regions.md new file mode 100644 index 0000000..399f6fa --- /dev/null +++ b/docs/design/regions.md @@ -0,0 +1,142 @@ +# Storing a language that occupies part of a page + +Contract for the next change, written before it is built. It is the riskiest step +in the ingest work so far: it alters tables that already shipped, and a +self-hosted install means a bad migration is other people's data. + +Prerequisites are done and committed: column geometry (`internal/doc/columns.go`) +and per-column language naming (`internal/doc/columnlang.go`). Neither stores +anything. This is what lets them. + +## What is broken today, surveyed rather than remembered + +| | | +|---|---| +| `doc_pages` PK `(document_id, page_no)` | one row per page, so a page cannot hold two languages — `00002:145` | +| `doc_langs` PK `(document_id, source, code, pdf_start)` | **two German columns on one page collide.** Same page, same code, same source, nothing to tell them apart — `00002:205` | +| `doc_langs.source` CHECK | omits `repertoire`, which exists in Go as a `Source` — `00002:163` | +| `Reconcile` | resolves and groups per page throughout — `reconcile.go:69,122,223` | +| `SaveProbe` | calls `PageLang(p.No)` once per page — `documents.go:212` | +| `Scope.Chars` | sums whole-page character counts — `doc.go:343` | +| Nothing anywhere | can slice a page's text by rectangle | + +## The decisions + +**A new `doc_regions` table, and `doc_pages` stays.** They record different +things. A page genuinely has one dominant script, one printed folio, one tag +position — those stay per page. What is not per page is language, and that moves +out. Widening `doc_pages` would make every existing column ambiguous about which +part of the page it describes. + +**Key on geometry, not on the label.** `(document_id, source, page, x0)`. That is +what distinguishes the German left column from the German right column, and it +keeps the natural-key upsert that makes re-probing idempotent — the property +`00002`'s comment calls load-bearing and `CONTRIBUTING.md` makes a rule. A +surrogate ULID would break it: a second probe would insert a parallel set rather +than converging. + +**A whole-page region has no box.** `x0 = 0, x1 = page width`, so a sectioned +manual stores exactly what it stores today and page-only readers keep working. +That is the compatibility stance: absent box means whole page, never null-checks +scattered through callers. + +**Characters replace pages as the unit of size.** Pages stop meaning anything when +a page holds three languages — "48 of 560 pages" was always a proxy. Pages stay a +thing to *show*; characters become the thing to count, which needs the text-slicing +function that does not exist yet. That function is part of this deliverable, not a +follow-on: `Scope.Chars` is wrong the moment regions land without it. + +**`repertoire` joins the CHECK lists**, in an append-only `00003`. `00002` is +committed and shipped; editing it now would diverge from any database already +created from it. + +## What building it settled + +Three things the contract above could not decide in advance. Each was decided by +measurement, and two of the measurements contradicted the first attempt. + +**A page divides on language, never on geometry.** The contract assumed the columns +were the regions. They are not: the column manual sets two columns of one language on +pages 6–10 and three on 52–56, and the sectioned manual reads as two or more columns +on 406 of its 560 pages, every one of them a side-by-side table. Dividing on geometry +stored four regions for a single-language page, on hundreds of pages of a manual with +no parallel columns at all. So a page divides only where its columns name more than +one language, which also disposes of "a table cell is not a text column" below. + +**The per-page answer outranks a column's, where it exists.** Letting a column's +alphabet reading overturn the reconciled page language split 31 pages of the +sectioned manual and contradicted its printed tab on 46 regions — German read as +Finnish, Spanish, Portuguese, every case a short table cell. That tab is right on all +553 of its content pages. On the column manual the per-page signals name *nothing* on +any of the eight verified pages, which is why the columns are trusted there and not +here. A disagreeing column now records a conflict and changes no answer. + +An earlier attempt made this conditional on how much evidence the alphabet had. That +is unsupported: over 685 labelled columns the repertoire signal is 93% accurate and +its errors occur at every amount of evidence, one of them with 118 distinctive +characters. No threshold separates them, so none was added. + +**A page-level answer must name a real language to outrank anything.** BCP-47 +constrains a subtag's shape, not its meaning, so `FAX` parses as the language `fax`, +`TEL` as `te`, `NDE` as `nd`. The column manual prints FAX on its service-address +page; the index parser reads that page as a contents table and offers FAX as an +entry; reconciliation then labelled two pages `fax`, overriding columns that read +correctly as German and Polish. `doc.KnownLanguage` now gates what may outrank other +evidence, and deliberately does not gate what may be stored — a manual printing an +unrecognised code is information worth keeping. + +**Characters are counted with one tool, not two.** A boxed region can only be +measured from positioned runs, so whole-page regions are measured that way too rather +than reusing the existing `pdftotext` count. The two disagree by 3.3% and 2.5% on the +fixtures' totals, 1–2% on a median page, and by up to 51% on a page whose text layer +parks runs outside the page box. One measurement throughout beats two that nearly +agree. Measured payoff: a German-reading household is charged 44,376 characters of the +column manual rather than the 233,849 its pages hold in all five languages — 19%, +where before a single language cost the same as all of them. + +## What this deliberately does not solve + +Recorded so the next person does not think they are unsolved by accident: + +**Regions do not compose across pages.** Nothing says the left column of page 7 is +the same column as page 9. Reading order and stable block IDs will need that, and +it is a separate question about identity rather than storage. Do not invent a +column-identity field here on the guess that it will be right. + +**Language-neutral content has no home.** A diagram, a parts table or a spec block +shared by all five languages must currently be assigned to one, duplicated, or +left unlabelled. None of those is correct. Left open because the honest fix is +probably a region kind rather than a language, and that wants a second document to +design against. + +**A table cell is not a text column.** Geometry cannot tell them apart, five pages +of the measured manual are troubleshooting tables, and it has already caused the +document's one language error. Above this layer — and now handled there, by dividing +on language rather than on cells, so a same-language table is one region. What is +still unsolved is a table whose cells are in *different* languages: it would divide, +and be wrong to. Neither manual does it. + +**A page that both prints a whole-page tab and sets parallel columns of different +languages** would be called one language, with a conflict recorded. Rule 1 gives the +per-page tab precedence and there is no evidence here for doing otherwise: one manual +prints per-page tabs and sets one language per page, the other prints per-column tabs +and names no page at all. The mechanism for the hybrid would be invented rather than +designed. If a third manual is that document, this is the stop condition. + +**Interleaved paragraphs down one column** would need one region per paragraph, +at which point a region stops being a layout partition and becomes a paragraph +annotation. Not seen in either manual. If a third manual does it, this design is +the wrong shape rather than an incomplete one — that is the stop condition. + +## Acceptance + +Not "the migration applies". The pipeline must store and read back the Thomas +manual's five languages across its parallel columns, with the eight +human-verified pages of `testdata/fixtures/thomas-drybox-amfibia.json` matching +column for column — and the Dreame manual's 34 sequential sections must be +unchanged, byte for byte, in what it reports. A change that improves the second +manual by altering the first has broken something. + +Both fixtures already carry the ground truth to check this against, and the +column fixture records per-page provenance so nothing is tested against its own +output. diff --git a/docs/design/search.md b/docs/design/search.md new file mode 100644 index 0000000..2713d6c --- /dev/null +++ b/docs/design/search.md @@ -0,0 +1,227 @@ +# Finding the sentence you need + +The paper pile is unsearchable, and you need the router manual at exactly the +moment the internet is down. That is [README](../../README.md)'s first problem, so +this is the first thing built on top of [conversion](conversion.md): blocks are the +unit the reader stores, and blocks are what is indexed. + +Everything below was measured. The corpus is both fixtures converted for German, +Russian, Japanese, Thai and Hebrew — **3,122 blocks** across the parallel-columns +manual's 68 pages and the sequential manual's 560 — loaded into one database per +FTS5 variant through the same `modernc.org/sqlite` driver the binary ships. + +## What a hit has to say + +Which manual, which page, which language, and enough text to recognise. "Something +matched on page 47" does not solve the problem this exists for, so every hit joins +`documents` and `devices` and carries the filename and the device's name. It also +carries the block's natural key — page, region left edge, index — which is the +citation [conversion.md](conversion.md) specifies, so a hit deep-links to the exact +paragraph and still points there after a re-conversion. + +## Where the index lives + +**FTS5 over `doc_blocks`, external content, maintained by triggers.** + +`content='doc_blocks'` means FTS5 stores the index and reads text back out of the +table rather than keeping its own copy. Measured, whole database file, after +`optimize` and `VACUUM`, against 626,688 bytes for `doc_blocks` alone: + +| | total | index | vs blocks alone | +|---|---|---|---| +| standalone `unicode61` | 1,388,544 | +761,856 | 2.22x | +| external `unicode61` | 897,024 | +270,336 | 1.43x | +| standalone `trigram` | 1,998,848 | +1,372,160 | 3.19x | +| external `trigram` | 1,507,328 | +880,640 | 2.41x | + +The duplicated text is the same 491,520 bytes in both pairs. External content costs +nothing but maintenance, and maintenance is where the decision that matters is. + +**Triggers, not statements next to each write.** Three paths change `doc_blocks` +and only two are visible in Go: `registry.saveBlocks`' wholesale delete-and-reinsert, +its upsert-in-place, and `documents ON DELETE CASCADE`, which runs **no Go at all** — +deleting a device removes its documents, which removes their blocks, entirely inside +SQLite. Triggers cover all three by construction and run inside whatever transaction +the write is already in, which is what `SaveConversion` needs: the blocks, the index +and the document's `ready` state commit together or not at all. + +That the cascade fires them was measured rather than assumed, because SQLite's own +documentation makes trigger firing on a foreign-key action conditional on +`recursive_triggers`, which manualbox does not set. With `foreign_keys(1)` and +`recursive_triggers` off — what `internal/db` actually opens with — the cascade +removes the index rows and FTS5's `integrity-check` passes. + +**And the failure it prevents is not the obvious one.** Dropping the delete trigger +does *not* leave a deleted manual findable: every search joins the index to +`doc_blocks`, so an entry whose row is gone joins to nothing and vanishes from the +results by accident. Measured that way round first, and it made the obvious control +assertion pass over an index FTS5 already reports as malformed. + +The real failure needs one more step. SQLite gives a new row `max(rowid)+1`, so +deleting the highest block frees a rowid the next insert takes, and the stale entry +then points at a real row of a *different* document. Searching for a word from the +deleted manual answers with another manual, another page, and text that does not +contain the word. **A wrong citation rather than a missing one**, which is the +failure this project can least afford, because a citation is what extraction will +hang a maintenance schedule on. `revertCheckTheDeleteTrigger` in `internal/db` is +that run, kept as a test. + +`doc_blocks`' rowid is not an `INTEGER PRIMARY KEY` alias, since its key is +composite, so SQLite does not promise to preserve it across a `VACUUM`. Nothing in +manualbox runs `VACUUM` (grepped, not assumed), and a `VACUUM` of a 3,122-block +database with holes in its rowid sequence left `max(rowid)` and every hit unchanged. +It is still not a promise; the repair is +`INSERT INTO doc_blocks_fts(doc_blocks_fts) VALUES ('rebuild')`. + +## The tokeniser, which is the one decision that could not be reasoned out + +`unicode61` splits on whitespace and punctuation. `trigram` indexes every run of +three characters and therefore matches substrings. The corpus is 34 languages +including Chinese, Japanese, Thai, Hebrew and Arabic, so the description alone +decides nothing. Real words from each script, same corpus, same driver: + +| query | `unicode61` | `trigram` | +|---|---|---| +| `Filter` (de) | 21 | 69 | +| `Saugkraft` (de) | 7 | 7 | +| `Gerat` (de, folded) | 71 | 96 | +| Russian *filtr* | 31 | 96 | +| Japanese *toriatsukai setsumeisho* | **0** | 6 | +| Thai *khu mue* | **0** | 6 | +| Hebrew *madrikh*, as stored | 1 | 5 | + +**`unicode61` finds nothing in Japanese and nothing in Thai.** Not degraded — +absent. A whole CJK or Thai run is one token, so it matches only a query that +happens to be the entire run: the two-character Japanese word for "power" scores 2 +hits against 27 real occurrences, and those 2 are where punctuation isolated it. + +**So: `trigram`, one index for every script.** It costs 880,640 bytes against +270,336 — 3.3x the index, 2.40x a blocks-only database, about 195 bytes per stored +block. The higher Latin counts are substring matches: `Filter` also finds +`Luftfilter` and `Filterdeckel`, which in German is closer to what a person meant +than token-exact matching. + +**Two indexes were rejected.** `unicode61` for the space-separated scripts and +`trigram` for the rest would give the majority of languages token-exact precision +and still serve CJK. It costs 1,150,976 bytes rather than 880,640, both need their +own triggers, and every query must guess from its own characters which index can +answer it — at which point a query mixing a German word and a Japanese one has no +right answer. One index that is somewhat blunt everywhere beats two that are sharp +until the household is multilingual, which every household with this kind of manual +already is. + +### The named limitation + +**A query shorter than three characters is not in the index at all.** Not fewer +results — none. Two characters is an ordinary word in Chinese and Japanese: the +words for "power" and "product" occur in 27 and 24 stored blocks and the index finds +0 of each. That is a real hole in exactly the scripts `trigram` was chosen for. + +So a query the index cannot represent is answered by scanning `doc_blocks` instead, +measured at **1.9 ms** over the 3,122-block corpus against 0.2 ms through the index, +and the API reports `mode: "substring"` so the difference is visible rather than +guessed at. `instr()` rather than `LIKE`, because `%` and `_` in a user's query +would be wildcards and a search box must not have a pattern language. Case folding +there is SQLite's `lower()`, which is ASCII only — exact for the CJK queries this +path exists for, case-sensitive for a two-letter Cyrillic one, which is the honest +limit of a scan that must not build an index to fix. + +One short word sends the **whole** query to the scan rather than being dropped from +it. A search for two words that quietly became a search for one is indistinguishable +from a correct answer. + +### Diacritics: the cost it was weighed against does not exist + +A German household on any keyboard types `Gerat` and should find `Gerät`. The worry +was that `remove_diacritics` also folds Cyrillic and Greek, which would be +expensive here — half this corpus is not Latin. + +Measured, across all three `unicode61` modes and both `trigram` modes: + +| stored | queried | folded? | +|---|---|---| +| `Gerät` | `Gerat` | yes, when on | +| `ещё` | `еще` | **never** | +| `Київ` | `Киiв` | **never** | +| `οδηγίες` | `οδηγιες` | **never** | +| Hebrew with niqqud | without | **never** | + +FTS5's folding table covers precomposed Latin and does not reach Cyrillic, Greek or +Hebrew. **It is on**, and it has to be set explicitly: `unicode61` folds by default +but `trigram` does not, and with it off `Gerat` finds 0 of the 96 blocks holding +`Gerät`. The index is 4,096 bytes *smaller* with folding on. + +### What no tokeniser fixes, and what stopped needing one + +**The stored Hebrew used to be in visual order, and is not any more.** +`internal/doc` reads the runs a right-to-left page paints and the PDF paints them +reversed, so the word for "manual" was stored as its own reverse: findable by a +query typed backwards (5 blocks) and not by one a Hebrew speaker would type (0 +blocks). That was upstream of the index, in extraction, and search could not repair +it and did not pretend to. + +`internal/doc/bidi.go` repaired it there, and the measurement is now the exact +inverse of the paragraph above: `מדריך` typed forwards finds **5** blocks and typed +backwards **0**, over the same Hebrew section. `internal/registry`'s +`TestHebrewIsFoundTypedForwards` is that measurement, run against the real manual, +and it exists because this claim had lived in prose with no test under it. + +It took two steps. The repair first read 4 and 1: one line of page 188 prints the +support URL and a Hebrew sentence together, `lineIsRightToLeft` decided direction by +majority of a line's strong characters, the URL's Latin outweighed the Hebrew, and +that line was joined left to right and left reversed. `internal/verify` reported the +same page from the other side, off a comparison sharing no code with this one. Giving +the decision to the **region's language**, with the majority as fallback, closed both. + +**One thing search still can't be asked about.** Word ORDER. Three now-fixed defects in +`internal/doc` reordered stored text without changing a single word of it — page 204's +support URL, page 211's Arabic list markers, page 204's laser standard. Every word was +present throughout, so the index found them all and only the reading was wrong: **no query +reveals that class of defect**, and none is pinned here. conversion.md carries all three +and why the verifier's word check could not see any of them either. + +## Ranking + +`bm25`, with **1.0 subtracted for a heading**, and both numbers reported. + +bm25 favours short documents, and on this corpus that is often wrong: for `Filter` +in the column manual it puts the parts-list fragments `1. Filter` and `13. Filter` +first and pushes the maintenance heading `Ausblasfilter austauschen` to tenth. A +heading names a section, so it answers *where does it say this* better than a +passing mention. Within one query bm25 spans about −9 to −2 with adjacent hits +differing by 0.05 to 0.5, so 1.0 moves a heading past hits of comparable quality +without overturning a decisively better one — on `Saugkraft` the troubleshooting +cell `Saugkraft ist zu gering` at −8.5 still leads, which is right. + +It is a judgement, so `bm25` and `score` are both in the response and their gap is +the bonus. A number in a response can be argued with; one buried in an `ORDER BY` +cannot. + +## Scope + +**Across documents by default**, because "which manual says X" is a question about +the household and someone looking for the descaling interval does not know which +manual to open. `?documentId=` narrows it, which is what a reader already inside a +document asks. An unknown id is a search of nothing rather than a 404: the parameter +scopes a search, and answering 404 would turn it into a way to test whether an id +exists. + +A query matching nothing returns `indexed`, the number of blocks there are to +search. "No manual says that" and "nothing has been converted yet" are otherwise +the same empty list, and the second is not a search failure — the same distinction +`Service.Blocks` makes between empty and absent. + +## What this deliberately does not do + +- **No language filter.** A household's scope already decided which languages were + converted, so the index holds only those; filtering further is a reader's + question, not a search one. The language is on every hit. +- **No highlight markup.** The snippet is about 64 characters of plain text around + the match. Inventing a delimiter would presume how a screen renders it, and the + search screen is a separate slice. +- **No stemming and no synonyms.** A trigram substring match already covers German + compounding, which is most of what stemming would buy here, and a stemmer is + per-language — a per-language index is the two-index design rejected above. +- **No paging beyond a limit.** `limit` caps the hits at 100 and `truncated` says + the list was cut off. Offset paging over a ranked result set that changes when a + document is re-converted is a promise this cannot keep yet. diff --git a/internal/api/api.go b/internal/api/api.go index a5688e7..aee1e7b 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -15,7 +15,9 @@ import ( "github.com/gordon2/manualbox/internal/config" "github.com/gordon2/manualbox/internal/db" "github.com/gordon2/manualbox/internal/frontend" + "github.com/gordon2/manualbox/internal/ingest" "github.com/gordon2/manualbox/internal/jobs" + "github.com/gordon2/manualbox/internal/registry" "github.com/gordon2/manualbox/internal/store" ) @@ -24,13 +26,15 @@ const sessionCookie = "manualbox_session" // Deps are the collaborators the API needs. type Deps struct { - Config config.Config - DB *db.DB - Store *store.Store - Auth *auth.Service - Jobs *jobs.Queue - Logger *slog.Logger - Version string + Config config.Config + DB *db.DB + Store *store.Store + Auth *auth.Service + Jobs *jobs.Queue + Registry *registry.Service + Ingest *ingest.Service + Logger *slog.Logger + Version string } // Server serves the API and the embedded frontend. @@ -67,6 +71,16 @@ func (s *Server) routes() { r.Use(s.recoverPanics) r.Use(middleware.Compress(5)) r.Use(middleware.Timeout(60 * time.Second)) + // Defence in depth against content sniffing. The document route sets this + // itself and serves anything unrecognised as an attachment, but a browser that + // second-guesses a Content-Type anywhere on this origin can turn stored bytes + // into script running beside the session cookie. + r.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + next.ServeHTTP(w, r) + }) + }) r.Route("/api/v1", func(r chi.Router) { // Reject cross-site state changes before anything else looks at the body. @@ -107,6 +121,38 @@ func (s *Server) routes() { r.Get("/jobs/events", s.handleJobEvents) r.Get("/jobs/{jobID}", s.handleGetJob) r.Post("/jobs/{jobID}/cancel", s.handleCancelJob) + + // Across every converted manual, because "which manual says X" is the + // question. Narrowed to one with ?documentId=. + r.Get("/search", s.handleSearch) + + r.Get("/locations", s.handleListLocations) + r.Post("/locations", s.handleCreateLocation) + + r.Get("/devices", s.handleListDevices) + r.Post("/devices", s.handleCreateDevice) + r.Get("/devices/{deviceID}", s.handleGetDevice) + r.Patch("/devices/{deviceID}", s.handleUpdateDevice) + r.Delete("/devices/{deviceID}", s.handleDeleteDevice) + + r.Get("/devices/{deviceID}/documents", s.handleListDocuments) + r.Post("/devices/{deviceID}/documents", s.handleUploadDocument) + + r.Get("/documents/{documentID}", s.handleGetDocument) + // The gate is the decision point: what is in the document, what would + // be processed, and what that would cost. + r.Get("/documents/{documentID}/gate", s.handleDocumentGate) + r.Get("/documents/{documentID}/languages", s.handleDocumentLanguages) + r.Get("/documents/{documentID}/content", s.handleDocumentContent) + r.Post("/documents/{documentID}/decline", s.handleDeclineDocument) + // The other half of the decision. Approving is what authorises the + // first work in the pipeline that is not free. + r.Post("/documents/{documentID}/approve", s.handleApproveDocument) + + // What the conversion produced. Deliberately not served from + // /content, which is the original bytes and stays that way. + r.Get("/documents/{documentID}/conversion", s.handleDocumentConversion) + r.Get("/documents/{documentID}/figures/{sha256}", s.handleDocumentFigure) }) }) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 1f52bd6..ed6cb06 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -16,18 +16,21 @@ import ( "github.com/gordon2/manualbox/internal/auth" "github.com/gordon2/manualbox/internal/config" "github.com/gordon2/manualbox/internal/db" + "github.com/gordon2/manualbox/internal/ingest" "github.com/gordon2/manualbox/internal/jobs" "github.com/gordon2/manualbox/internal/logging" + "github.com/gordon2/manualbox/internal/registry" "github.com/gordon2/manualbox/internal/store" ) const testPassword = "a perfectly fine passphrase" type harness struct { - server *httptest.Server - queue *jobs.Queue - auth *auth.Service - client *http.Client + server *httptest.Server + queue *jobs.Queue + auth *auth.Service + registry *registry.Service + client *http.Client } // newHarness starts a real server over a real database, so these tests exercise @@ -55,9 +58,15 @@ func newHarness(t *testing.T) *harness { cfg := config.Default() cfg.Content.Languages = []string{"de", "en"} + registryService := registry.New(database, registry.Options{}) + ingestService := ingest.New(ingest.Deps{ + Config: cfg, Registry: registryService, Store: blobs, Jobs: queue, + }) + srv := New(Deps{ Config: cfg, DB: database, Store: blobs, Auth: authService, - Jobs: queue, Logger: logging.Discard(), Version: "test", + Jobs: queue, Registry: registryService, Ingest: ingestService, + Logger: logging.Discard(), Version: "test", }) ts := httptest.NewServer(srv.Handler()) @@ -66,10 +75,11 @@ func newHarness(t *testing.T) *harness { // A cookie jar so the session behaves as it would in a browser. jar := &cookieJar{} return &harness{ - server: ts, - queue: queue, - auth: authService, - client: &http.Client{Jar: jar, Timeout: 10 * time.Second}, + server: ts, + queue: queue, + auth: authService, + registry: registryService, + client: &http.Client{Jar: jar, Timeout: 10 * time.Second}, } } @@ -239,6 +249,21 @@ func TestProtectedRoutesRequireASession(t *testing.T) { "/api/v1/jobs", "/api/v1/jobs/job_123", "/api/v1/jobs/events", + // The registry and document routes. This list went stale once already — + // eleven routes were added without being added here — so the guarantee in + // the comment above was not actually being enforced for any of them. + "/api/v1/locations", + "/api/v1/devices", + "/api/v1/devices/dev_123", + "/api/v1/devices/dev_123/documents", + "/api/v1/documents/doc_123", + "/api/v1/documents/doc_123/gate", + "/api/v1/documents/doc_123/languages", + "/api/v1/documents/doc_123/content", + "/api/v1/documents/doc_123/conversion", + // Search reads every converted manual in the household, so it is the last + // route that may answer an anonymous caller. + "/api/v1/search", } { resp := h.do(t, http.MethodGet, path, nil) if resp.StatusCode != http.StatusUnauthorized { diff --git a/internal/api/handlers_conversion.go b/internal/api/handlers_conversion.go new file mode 100644 index 0000000..7e3bc77 --- /dev/null +++ b/internal/api/handlers_conversion.go @@ -0,0 +1,237 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/gordon2/manualbox/internal/ingest" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// maxApproveBodyBytes bounds the approve body. It holds one boolean, so anything +// larger is a mistake or an attack and reading it costs nothing to refuse. +const maxApproveBodyBytes = 4 << 10 + +// approveRequest is the whole of what a caller may say about scope. +// +// One boolean, answering the one question the gate asked as `neutral`: convert the +// pages no language owns as well. There is still no language list and there must +// not be one — the languages are configuration, and the gate rendered them from it. +// [ingest.Service.Approve] carries the argument for why answering an offer is +// different in kind from composing a scope, and why this may not grow into a list +// of pages. +type approveRequest struct { + IncludeNeutralPages bool `json:"includeNeutralPages"` +} + +// handleApproveDocument authorises the work the gate reported and queues it. +// +// The body is optional and holds at most one flag: the scope is the household's +// configured languages, which is what the gate showed, plus the pages the gate +// offered as `neutral` if the user asked for them. Accepting a language list here +// would let a caller approve something other than what the user was told about — +// see [ingest.Service.Approve]. +func (s *Server) handleApproveDocument(w http.ResponseWriter, r *http.Request) { + documentID := chi.URLParam(r, "documentID") + + // An absent or empty body is today's request and must keep working unchanged: it + // means the languages and nothing else. Only a malformed body is an error, because + // a client that meant to ask for the extra pages and mistyped it deserves to be + // told rather than silently given the smaller scope. + var req approveRequest + if r.Body != nil { + body, err := io.ReadAll(io.LimitReader(r.Body, maxApproveBodyBytes)) + if err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_body", + "The request body could not be read.") + return + } + if len(bytes.TrimSpace(body)) > 0 { + if err := json.Unmarshal(body, &req); err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_body", + "The request body must be a JSON object.") + return + } + } + } + + job, err := s.deps.Ingest.Approve(r.Context(), documentID, ingest.ApproveScope{ + IncludeNeutralPages: req.IncludeNeutralPages, + }) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + + document, err := s.deps.Registry.GetDocument(r.Context(), documentID) + if err != nil { + s.internalError(w, r, err) + return + } + + body := map[string]any{"document": document} + if job != nil { + body["jobId"] = job.ID + } + // 202: the conversion is queued, not done. Progress arrives on the existing job + // event stream, the same way an upload's probe does. + writeJSON(w, http.StatusAccepted, body) +} + +// handleDocumentConversion serves what the conversion produced: the readable +// blocks and the pictures. +// +// # Why this is not /content +// +// /documents/{id}/content is the stored original, byte for byte, and +// docs/design/privacy.md is explicit that the original is kept whole. That path is +// the "own your data" promise at its most literal, so it keeps serving the PDF and +// nothing else. Overloading it would mean either content negotiation — which is +// invisible in a URL, so the derived view could not be linked or bookmarked — or a +// query parameter that silently changes the response from bytes to JSON. A +// separate path costs one route and keeps both answers unambiguous. +// +// `?lang=de` is the funnel's own query and returns one language's blocks together +// with the pictures belonging to it, which includes every picture belonging to no +// language. Omitting the parameter returns everything stored, which is already only +// what the household's scope charged for. `?lang=` with an empty value is a third, +// real question: the blocks and pictures nothing could name, which no other +// language's answer contains. +func (s *Server) handleDocumentConversion(w http.ResponseWriter, r *http.Request) { + documentID := chi.URLParam(r, "documentID") + document, err := s.deps.Registry.GetDocument(r.Context(), documentID) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + + var ( + blocks []registry.Block + figures []registry.Figure + ) + lang := r.URL.Query().Get("lang") + // Has, not a non-empty value: "" is a question about the unnamed content rather + // than the absence of a filter. + if r.URL.Query().Has("lang") { + blocks, err = s.deps.Registry.BlocksByLang(r.Context(), documentID, lang) + if err == nil { + figures, err = s.deps.Registry.FiguresByLang(r.Context(), documentID, lang) + } + } else { + blocks, err = s.deps.Registry.Blocks(r.Context(), documentID) + if err == nil { + figures, err = s.deps.Registry.Figures(r.Context(), documentID) + } + } + if err != nil { + s.internalError(w, r, err) + return + } + + body := map[string]any{ + "documentId": document.ID, + // The state is what distinguishes "converted and empty" from "not converted", + // which no count can: a document that has not been through the gate has no + // blocks, and that is not the claim that it has no content. + "state": document.State, + "blocks": blocks, + "figures": figures, + } + if r.URL.Query().Has("lang") { + body["lang"] = lang + } + // The printed page a contents entry names, mapped onto a PDF page. Served once + // for the document rather than resolved per entry, because it is one constant + // per document -- registry.FolioOffset carries the measurement -- and because + // the reader has to decide per entry whether the target is a page this + // language's conversion actually holds, which only it knows. + // + // Omitted entirely when the folios do not agree on one offset. It must not + // default to 0: the columns manual's real offset IS 0, and a reader that could + // not tell "no mapping" from "the mapping is identity" would either refuse a + // link that works or offer one that does not. + folio, err := s.deps.Registry.FolioOffset(r.Context(), documentID) + if err != nil { + s.internalError(w, r, err) + return + } + if folio != nil { + body["folioOffset"] = folio.Offset + } + if document.LastError != "" { + body["lastError"] = document.LastError + } + writeJSON(w, http.StatusOK, body) +} + +// handleDocumentFigure serves one rendered figure's PNG. +// +// Addressed by digest rather than by page and index, because the digest is what +// the conversion response already carries, and because the content is the name: the +// ETag is exact and the bytes are immutable for ever. +// +// The digest is checked against this document's own figures rather than being +// handed to the blob store directly. The store is content addressed and holds every +// original anyone has uploaded, so a route that opened any digest a caller named +// would serve another household's manual to whoever could guess a hash. +func (s *Server) handleDocumentFigure(w http.ResponseWriter, r *http.Request) { + documentID := chi.URLParam(r, "documentID") + digest := chi.URLParam(r, "sha256") + if err := store.ValidDigest(digest); err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_digest", + "A figure is addressed by its SHA-256.") + return + } + + figures, err := s.deps.Registry.Figures(r.Context(), documentID) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + found := false + for i := range figures { + if figures[i].SHA256 == digest { + found = true + break + } + } + if !found { + s.writeError(w, r, http.StatusNotFound, "not_found", + "This document has no such figure.") + return + } + + content, err := s.deps.Store.Open(digest) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + s.writeError(w, r, http.StatusNotFound, "content_missing", + "The stored bytes for this figure are missing.") + return + } + s.internalError(w, r, err) + return + } + defer func() { _ = content.Close() }() + + // image/png is checked rather than assumed: internal/doc renders with pdftoppm + // -png and verifies the signature and IHDR before returning the bytes, and + // registry.SaveConversion records that type against the blob. + w.Header().Set("Content-Type", "image/png") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("ETag", `"`+digest+`"`) + w.Header().Set("Cache-Control", "private, max-age=31536000, immutable") + + seeker, ok := content.(io.ReadSeeker) + if !ok { + s.internalError(w, r, errors.New("api: stored figure is not seekable")) + return + } + http.ServeContent(w, r, digest+".png", time.Time{}, seeker) +} diff --git a/internal/api/handlers_conversion_approve_test.go b/internal/api/handlers_conversion_approve_test.go new file mode 100644 index 0000000..1dc60cd --- /dev/null +++ b/internal/api/handlers_conversion_approve_test.go @@ -0,0 +1,138 @@ +package api + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// gateDoc stores a probed document whose page 1 belongs to no language and carries +// pictures, and whose page 2 is German. That is the shape the second scope exists +// for, reduced to two pages. +// +// Stored directly rather than probed, because what is under test here is the HTTP +// seam — whether the body reaches the scope — and running poppler to reach it would +// make this a slow test of something else. +func (h *harness) gateDoc(t *testing.T, digest string) string { + t.Helper() + ctx := context.Background() + + device, err := h.registry.CreateDevice(ctx, registry.NewDevice{Name: "Vacuum " + digest}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat(digest, 32), Size: 10} + if err := h.registry.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := h.registry.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, Filename: "manual.pdf", + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + + plate := 4 + res := &doc.Result{ + Info: doc.Info{Pages: 2}, + HasTextLayer: true, + Pages: []doc.Page{ + {No: 1, Chars: 7, Script: "Latin", Figures: &plate}, + {No: 2, Chars: 900, Script: "Latin", Lang: "de", LangSource: "reconciled"}, + }, + Runs: []doc.Run{{Lang: "de", Code: "DE", Start: 2, End: 2, Source: doc.SourceReconciled}}, + Regions: []doc.Region{ + {Page: 1, X0: 0, X1: 918, Chars: 7}, + {Page: 2, X0: 0, X1: 918, Lang: "de", Code: "DE", Source: doc.SourceReconciled, Chars: 900}, + }, + } + if err := h.registry.SaveProbe(ctx, document.ID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + return document.ID +} + +// TestApproveCarriesTheOneScopeFlagAndNothingElse is the HTTP contract on the seam +// the design argument turns on. +// +// The body may say exactly one thing — include the pages the gate offered as +// `neutral`, or do not — and that answer has to reach the stored scope. What it may +// NOT do is name a page or a language: the funnel's promise is that what is approved +// is what the gate showed, and a request that could describe work rather than accept +// an offer would break it. See ingest.Service.Approve. +func TestApproveCarriesTheOneScopeFlagAndNothingElse(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + + for _, tc := range []struct { + name string + body any + want bool + }{ + // The request today, which must keep meaning the languages and nothing else. + {"no body at all", nil, false}, + {"an empty object", map[string]any{}, false}, + {"an explicit no", map[string]any{"includeNeutralPages": false}, false}, + {"an explicit yes", map[string]any{"includeNeutralPages": true}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + // A digest per case: each needs its own document, and the store keys on it. + id := h.gateDoc(t, string(rune('a'+len(tc.name)%16))) + + if code := h.status(t, http.MethodPost, + "/api/v1/documents/"+id+"/approve", tc.body); code != http.StatusAccepted { + t.Fatalf("approve returned %d, want 202", code) + } + + document, err := h.registry.GetDocument(context.Background(), id) + if err != nil { + t.Fatalf("get document: %v", err) + } + if document.IncludeNeutralPages != tc.want { + t.Errorf("includeNeutralPages stored as %t, want %t", + document.IncludeNeutralPages, tc.want) + } + if document.State != registry.StateConverting { + t.Errorf("state = %q, want %q", document.State, registry.StateConverting) + } + }) + } +} + +// TestApproveRefusesABodyItCannotRead is the small kindness in the parsing rule: an +// absent body is today's request and is fine, but a malformed one is not silently +// downgraded to the smaller scope. A user who ticked the box deserves to be told +// their request was not understood rather than to find the pages missing later. +func TestApproveRefusesABodyItCannotRead(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + id := h.gateDoc(t, "f") + + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + h.server.URL+"/api/v1/documents/"+id+"/approve", strings.NewReader("{not json")) + if err != nil { + t.Fatalf("build request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := h.client.Do(req) + if err != nil { + t.Fatalf("approve: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("a malformed body returned %d, want 400", resp.StatusCode) + } + document, err := h.registry.GetDocument(context.Background(), id) + if err != nil { + t.Fatalf("get document: %v", err) + } + if document.State == registry.StateConverting { + t.Error("a request that could not be read still started a conversion") + } +} diff --git a/internal/api/handlers_conversion_folio_test.go b/internal/api/handlers_conversion_folio_test.go new file mode 100644 index 0000000..76c9a17 --- /dev/null +++ b/internal/api/handlers_conversion_folio_test.go @@ -0,0 +1,128 @@ +package api + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// folioDoc stores a document whose pages print the given folios, converted and +// ready, and returns its id. A nil entry is a page that prints no folio. +func (h *harness) folioDoc(t *testing.T, digest string, folios []*int) string { + t.Helper() + ctx := context.Background() + + device, err := h.registry.CreateDevice(ctx, registry.NewDevice{Name: "Vacuum " + digest}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat(digest, 32), Size: 10} + if err := h.registry.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := h.registry.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, Filename: "manual.pdf", + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + + pages := make([]doc.Page, 0, len(folios)) + for i, folio := range folios { + pages = append(pages, doc.Page{No: i + 1, Chars: 100, Script: "Latin", Folio: folio}) + } + res := &doc.Result{Info: doc.Info{Pages: len(folios)}, Pages: pages} + if err := h.registry.SaveProbe(ctx, document.ID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + if err := h.registry.SaveConversion(ctx, document.ID, + []doc.Block{para(1, "de", "Den Ausblasfilter tauschen.")}, nil, nil, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + return document.ID +} + +func folioPtr(n int) *int { return &n } + +// TestConversionServesFolioOffsetOrOmitsIt is the one contract on this field that a +// client cannot recover from getting wrong: absent and zero are different answers. +// +// A manual whose page 1 is its cover has a real offset of 0, and the columns manual +// measured here is exactly that. If the response defaulted to 0 where the folios +// agreed on nothing, every contents entry of an unmappable document would become a +// link to the wrong page -- so this asserts the key's PRESENCE, not just its value, +// which is why the body is decoded into a map rather than into the response struct. +func TestConversionServesFolioOffsetOrOmitsIt(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + + // Offset 6 on 8 of 8 pages: printed 1 is PDF page 7. + offset6 := make([]*int, 0, 8) + for i := range 8 { + offset6 = append(offset6, folioPtr(i+1-6)) + } + // Offset 0 on 8 of 8. The value that must not be confusable with absence. + offset0 := make([]*int, 0, 8) + for i := range 8 { + offset0 = append(offset0, folioPtr(i+1)) + } + // Folios restarting halfway: four at offset 0 and four at offset 4. Neither + // holds a majority, so there is no answer to give. + restarting := []*int{ + folioPtr(1), folioPtr(2), folioPtr(3), folioPtr(4), + folioPtr(1), folioPtr(2), folioPtr(3), folioPtr(4), + } + // Nothing prints a folio at all. + none := []*int{nil, nil, nil, nil, nil, nil, nil, nil} + + tests := []struct { + name string + digest string + folios []*int + wantKey bool + wantValue float64 + }{ + {"a document whose front matter is six pages", "a", offset6, true, 6}, + {"a document whose page 1 is its cover", "b", offset0, true, 0}, + {"folios that restart halfway", "c", restarting, false, 0}, + {"no page prints a folio", "d", none, false, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id := h.folioDoc(t, tt.digest, tt.folios) + res := h.do(t, "GET", "/api/v1/documents/"+id+"/conversion?lang=de", nil) + defer func() { _ = res.Body.Close() }() + + var body map[string]any + if err := json.NewDecoder(res.Body).Decode(&body); err != nil { + t.Fatalf("decode: %v", err) + } + got, present := body["folioOffset"] + if present != tt.wantKey { + t.Fatalf("folioOffset present = %v, want %v (body keys: %v).\n"+ + "Absent and zero are different answers here: absent means the folios "+ + "agreed on no offset, and a client that reads a missing field as 0 "+ + "links every contents entry to the wrong page.", + present, tt.wantKey, keysOf(body)) + } + if tt.wantKey && got != tt.wantValue { + t.Fatalf("folioOffset = %v, want %v", got, tt.wantValue) + } + }) + } +} + +func keysOf(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/api/handlers_registry.go b/internal/api/handlers_registry.go new file mode 100644 index 0000000..05ba2f5 --- /dev/null +++ b/internal/api/handlers_registry.go @@ -0,0 +1,489 @@ +package api + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// --- locations --- + +func (s *Server) handleListLocations(w http.ResponseWriter, r *http.Request) { + locations, err := s.deps.Registry.ListLocations(r.Context()) + if err != nil { + s.internalError(w, r, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"locations": locations}) +} + +func (s *Server) handleCreateLocation(w http.ResponseWriter, r *http.Request) { + var body struct { + Name string `json:"name"` + ParentID string `json:"parentId"` + Notes string `json:"notes"` + } + if err := decodeJSON(w, r, &body); err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_body", err.Error()) + return + } + + location, err := s.deps.Registry.CreateLocation(r.Context(), strings.TrimSpace(body.Name), body.ParentID, body.Notes) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusCreated, location) +} + +// --- devices --- + +func (s *Server) handleListDevices(w http.ResponseWriter, r *http.Request) { + devices, err := s.deps.Registry.ListDevices(r.Context()) + if err != nil { + s.internalError(w, r, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"devices": devices}) +} + +// deviceBody is the shared shape for creating and updating a device. +// +// Serial number and purchase price are deliberately absent. They are the +// highest-harm fields manualbox would hold and must be encrypted with a key kept +// outside the data directory; accepting them before the keyring is wired in would +// store them in the clear. See docs/design/privacy.md. +type deviceBody struct { + Name string `json:"name"` + Brand string `json:"brand"` + Model string `json:"model"` + Category string `json:"category"` + LocationID string `json:"locationId"` + Notes string `json:"notes"` + PurchasedAt string `json:"purchasedAt"` +} + +func (b deviceBody) toNewDevice() (registry.NewDevice, error) { + in := registry.NewDevice{ + Name: strings.TrimSpace(b.Name), + Brand: strings.TrimSpace(b.Brand), + Model: strings.TrimSpace(b.Model), + Category: strings.TrimSpace(b.Category), + LocationID: b.LocationID, + Notes: b.Notes, + } + if b.PurchasedAt != "" { + // Date only: a purchase has a date, not a time of day, and accepting a + // timestamp would invite a timezone shifting it to the previous day. + t, err := time.Parse(time.DateOnly, b.PurchasedAt) + if err != nil { + return in, fmt.Errorf("purchasedAt must be a date like 2026-07-25") + } + in.PurchasedAt = &t + } + return in, nil +} + +func (s *Server) handleCreateDevice(w http.ResponseWriter, r *http.Request) { + var body deviceBody + if err := decodeJSON(w, r, &body); err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_body", err.Error()) + return + } + in, err := body.toNewDevice() + if err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_body", err.Error()) + return + } + + device, err := s.deps.Registry.CreateDevice(r.Context(), in) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusCreated, device) +} + +func (s *Server) handleGetDevice(w http.ResponseWriter, r *http.Request) { + device, err := s.deps.Registry.GetDevice(r.Context(), chi.URLParam(r, "deviceID")) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusOK, device) +} + +func (s *Server) handleUpdateDevice(w http.ResponseWriter, r *http.Request) { + var body deviceBody + if err := decodeJSON(w, r, &body); err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_body", err.Error()) + return + } + in, err := body.toNewDevice() + if err != nil { + s.writeError(w, r, http.StatusBadRequest, "invalid_body", err.Error()) + return + } + + device, err := s.deps.Registry.UpdateDevice(r.Context(), chi.URLParam(r, "deviceID"), in) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusOK, device) +} + +func (s *Server) handleDeleteDevice(w http.ResponseWriter, r *http.Request) { + if err := s.deps.Registry.DeleteDevice(r.Context(), chi.URLParam(r, "deviceID")); err != nil { + s.writeRegistryError(w, r, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// --- documents --- + +func (s *Server) handleListDocuments(w http.ResponseWriter, r *http.Request) { + deviceID := chi.URLParam(r, "deviceID") + if _, err := s.deps.Registry.GetDevice(r.Context(), deviceID); err != nil { + s.writeRegistryError(w, r, err) + return + } + documents, err := s.deps.Registry.ListDocumentsForDevice(r.Context(), deviceID) + if err != nil { + s.internalError(w, r, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"documents": documents}) +} + +// uploadFormField is the multipart field name for the file itself. +const uploadFormField = "file" + +// handleUploadDocument stores an uploaded file and queues the free probe. +// +// The response is deliberately returned before the document has been read: the +// probe takes a couple of seconds on a large manual, and blocking an HTTP request +// on it would mean a user who closes the tab loses the upload. Progress arrives +// over the existing job event stream. +func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) { + deviceID := chi.URLParam(r, "deviceID") + if _, err := s.deps.Registry.GetDevice(r.Context(), deviceID); err != nil { + s.writeRegistryError(w, r, err) + return + } + + // Cap the request body before reading any of it, so an oversized upload is + // refused rather than filling the disk on the way to being rejected. + r.Body = http.MaxBytesReader(w, r.Body, s.deps.Config.Server.MaxUploadBytes) + + file, header, err := r.FormFile(uploadFormField) + if err != nil { + if errors.Is(err, http.ErrMissingFile) { + s.writeError(w, r, http.StatusBadRequest, "missing_file", + fmt.Sprintf("Attach the document as a multipart field named %q.", uploadFormField)) + return + } + // A body that exceeded the cap surfaces here, as does malformed multipart. + s.writeError(w, r, http.StatusRequestEntityTooLarge, "upload_failed", + "The upload could not be read. It may be larger than this instance allows.") + return + } + defer func() { _ = file.Close() }() + + kind := r.FormValue("kind") + if kind == "" { + kind = registry.KindManual + } + if !validDocumentKind(kind) { + s.writeError(w, r, http.StatusBadRequest, "invalid_kind", + "kind must be one of manual, receipt, warranty, photo, other.") + return + } + + ref, err := s.deps.Store.Put(r.Context(), file) + if err != nil { + s.internalError(w, r, err) + return + } + + // Determine the media type from the bytes, never from the upload. + // + // The client's Content-Type is attacker-controlled — any HTTP client sets it + // freely, and a browser derives it from the file extension. Storing it and + // later echoing it back turns "upload a manual you found on the web" into + // script execution at this instance's own origin, with the session cookie + // riding along. Sniffing is the only version of this that cannot be lied to. + mediaType, err := s.sniffMediaType(ref.SHA256) + if err != nil { + s.internalError(w, r, err) + return + } + if err := s.deps.Registry.RecordBlob(r.Context(), ref, mediaType); err != nil { + s.internalError(w, r, err) + return + } + + document, created, err := s.deps.Registry.CreateDocument(r.Context(), registry.NewDocument{ + DeviceID: deviceID, + BlobSHA256: ref.SHA256, + Filename: baseFilename(header.Filename), + MediaType: mediaType, + Kind: kind, + }) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + + // Queue the probe whether or not the row is new: an earlier attempt may have + // been interrupted before the job was created, and the probe is idempotent. + job, err := s.deps.Ingest.EnqueueProbe(r.Context(), document.ID) + if err != nil { + s.internalError(w, r, err) + return + } + + status := http.StatusCreated + if !created { + // The same bytes against the same device is the same document, not a + // conflict: say so with 200 rather than inventing a duplicate. + status = http.StatusOK + } + body := map[string]any{"document": document, "duplicate": !created} + if job != nil { + body["jobId"] = job.ID + } + writeJSON(w, status, body) +} + +func validDocumentKind(kind string) bool { + switch kind { + case registry.KindManual, registry.KindReceipt, registry.KindWarranty, + registry.KindPhoto, registry.KindOther: + return true + default: + return false + } +} + +func (s *Server) handleGetDocument(w http.ResponseWriter, r *http.Request) { + document, err := s.deps.Registry.GetDocument(r.Context(), chi.URLParam(r, "documentID")) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusOK, document) +} + +// handleDocumentGate answers the pre-flight question: what is in this document, +// what would be processed, and what that would cost. +func (s *Server) handleDocumentGate(w http.ResponseWriter, r *http.Request) { + gate, err := s.deps.Ingest.Gate(r.Context(), chi.URLParam(r, "documentID")) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusOK, gate) +} + +// handleDocumentLanguages returns one signal's view of the language map. +// +// The default is the reconciled view. Asking for a specific signal is what makes +// a disagreement inspectable rather than merely flagged — "the tag says DA, the +// index says FI" is answerable after the fact. +func (s *Server) handleDocumentLanguages(w http.ResponseWriter, r *http.Request) { + source := doc.Source(r.URL.Query().Get("source")) + if source == "" { + source = doc.SourceReconciled + } + switch source { + case doc.SourcePageTag, doc.SourceIndex, doc.SourceScript, doc.SourceDetector, doc.SourceReconciled: + default: + s.writeError(w, r, http.StatusBadRequest, "invalid_source", + "source must be one of page-tag, index, script, detector, reconciled.") + return + } + + runs, err := s.deps.Registry.LanguageRuns(r.Context(), chi.URLParam(r, "documentID"), source) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"source": source, "runs": runs}) +} + +// handleDeclineDocument records that the user does not want the document +// processed. The original is kept regardless. +func (s *Server) handleDeclineDocument(w http.ResponseWriter, r *http.Request) { + documentID := chi.URLParam(r, "documentID") + if _, err := s.deps.Registry.GetDocument(r.Context(), documentID); err != nil { + s.writeRegistryError(w, r, err) + return + } + if err := s.deps.Ingest.Decline(r.Context(), documentID); err != nil { + s.internalError(w, r, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleDocumentContent serves the stored original, byte for byte. +// +// This is the "own your data" promise at its most literal: whatever manualbox +// derives, the file you uploaded is retrievable unchanged. Served with +// ServeContent so range requests work, which is what lets a PDF viewer fetch one +// page at a time instead of a 15 MB download. +func (s *Server) handleDocumentContent(w http.ResponseWriter, r *http.Request) { + document, err := s.deps.Registry.GetDocument(r.Context(), chi.URLParam(r, "documentID")) + if err != nil { + s.writeRegistryError(w, r, err) + return + } + + content, err := s.deps.Store.Open(document.BlobSHA256) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + s.writeError(w, r, http.StatusNotFound, "content_missing", + "The stored content for this document is missing.") + return + } + s.internalError(w, r, err) + return + } + defer func() { _ = content.Close() }() + + // Only a short list of types may render in the browser. Anything else is + // downloaded as opaque bytes, because rendering it would run it at this + // instance's origin. SVG is deliberately absent from the safe list: it is an + // XML document that can carry script. + served, inline := safeInlineType(document.MediaType) + w.Header().Set("Content-Type", served) + // Without this a browser may sniff the body and render it as HTML regardless + // of the type we sent, which would undo the allowlist above. + w.Header().Set("X-Content-Type-Options", "nosniff") + + disposition := "attachment" + if inline { + disposition = "inline" + } + if name := document.Filename; name != "" { + w.Header().Set("Content-Disposition", disposition+"; filename*=UTF-8''"+urlPathEscape(name)) + } else { + w.Header().Set("Content-Disposition", disposition) + } + + // The digest is the content, so the ETag is exact and the content immutable. + w.Header().Set("ETag", `"`+document.BlobSHA256+`"`) + w.Header().Set("Cache-Control", "private, max-age=31536000, immutable") + + seeker, ok := content.(io.ReadSeeker) + if !ok { + s.internalError(w, r, errors.New("api: stored content is not seekable")) + return + } + http.ServeContent(w, r, document.Filename, document.UpdatedAt, seeker) +} + +// urlPathEscape escapes a filename for a Content-Disposition header, so a name +// with a space or a non-ASCII character does not corrupt the header. +func urlPathEscape(name string) string { return url.PathEscape(name) } + +// sniffBytes is how much of a file [Server.sniffMediaType] inspects. +// http.DetectContentType never looks at more than this. +const sniffBytes = 512 + +// sniffMediaType determines a stored blob's type from its own bytes. +func (s *Server) sniffMediaType(digest string) (string, error) { + content, err := s.deps.Store.Open(digest) + if err != nil { + return "", err + } + defer func() { _ = content.Close() }() + + head := make([]byte, sniffBytes) + n, err := io.ReadFull(content, head) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return "", fmt.Errorf("api: read blob head: %w", err) + } + return http.DetectContentType(head[:n]), nil +} + +// inlineSafeTypes are the media types a browser may render directly from this +// origin. Everything else is served as an attachment. +// +// The list is short on purpose. A document served inline shares the origin of +// the app and its session cookie, so anything that can execute — HTML, SVG, XML +// with a stylesheet — must not be on it, whatever the uploader called the file. +// +// text/plain is deliberately absent, and that is not an oversight. +// http.DetectContentType has no SVG signature and returns text/plain for any +// textual content it does not recognise, so the bucket contains SVG, XML and +// anything else script-bearing that dodges the HTML signatures. A current +// browser honouring nosniff renders it as text and nothing executes — but that +// would make one response header the only thing preventing it. The cost of +// leaving it out is that a genuine .txt downloads instead of displaying, which +// is a poor trade to reverse. +var inlineSafeTypes = map[string]bool{ + "application/pdf": true, + "image/png": true, + "image/jpeg": true, + "image/gif": true, + "image/webp": true, +} + +// safeInlineType maps a stored media type onto what will actually be sent, and +// whether it may be displayed rather than downloaded. +func safeInlineType(stored string) (served string, inline bool) { + // DetectContentType returns parameters such as "; charset=utf-8"; match on + // the bare type. + base := stored + if i := strings.IndexByte(base, ';'); i >= 0 { + base = base[:i] + } + base = strings.ToLower(strings.TrimSpace(base)) + + if inlineSafeTypes[base] { + // Re-send the bare type rather than the stored string, so no parameter + // from the stored value is echoed back into the header. + return base, true + } + return "application/octet-stream", false +} + +// baseFilename reduces an uploaded name to its last path component. +// +// filepath.Base alone is not enough: it is platform-specific, so on the Linux +// server this targets it leaves a Windows path such as +// `C:\Users\alice\Downloads\manual.pdf` entirely intact — carrying the +// uploader's directory layout, and their username, into the database. Cut on +// both separators. +func baseFilename(name string) string { + if i := strings.LastIndexAny(name, `\/`); i >= 0 { + name = name[i+1:] + } + return filepath.Base(name) +} + +// writeRegistryError maps registry errors onto status codes. +func (s *Server) writeRegistryError(w http.ResponseWriter, r *http.Request, err error) { + switch { + case errors.Is(err, registry.ErrNotFound): + s.writeError(w, r, http.StatusNotFound, "not_found", "No such item.") + case errors.Is(err, registry.ErrInvalid): + s.writeError(w, r, http.StatusBadRequest, "invalid", err.Error()) + default: + s.internalError(w, r, err) + } +} diff --git a/internal/api/handlers_registry_test.go b/internal/api/handlers_registry_test.go new file mode 100644 index 0000000..82e3b23 --- /dev/null +++ b/internal/api/handlers_registry_test.go @@ -0,0 +1,212 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "mime/multipart" + "net/http" + "net/textproto" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/testpdf" +) + +// upload posts a file to a device the way a browser would, including a +// client-supplied Content-Type for the file part — which is exactly the value +// that must not be trusted. +func (h *harness) upload(t *testing.T, deviceID, filename, clientType string, content []byte) *http.Response { + t.Helper() + + var body bytes.Buffer + form := multipart.NewWriter(&body) + + partHeader := make(textproto.MIMEHeader) + partHeader.Set("Content-Disposition", + `form-data; name="file"; filename="`+filename+`"`) + if clientType != "" { + partHeader.Set("Content-Type", clientType) + } + part, err := form.CreatePart(partHeader) + if err != nil { + t.Fatalf("create part: %v", err) + } + if _, err := part.Write(content); err != nil { + t.Fatalf("write part: %v", err) + } + if err := form.Close(); err != nil { + t.Fatalf("close form: %v", err) + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + h.server.URL+"/api/v1/devices/"+deviceID+"/documents", &body) + if err != nil { + t.Fatalf("build request: %v", err) + } + req.Header.Set("Content-Type", form.FormDataContentType()) + req.Header.Set("Origin", h.server.URL) + + resp, err := h.client.Do(req) + if err != nil { + t.Fatalf("upload: %v", err) + } + return resp +} + +func (h *harness) createDevice(t *testing.T) string { + t.Helper() + resp := h.do(t, http.MethodPost, "/api/v1/devices", + map[string]string{"name": "Kettle"}, [2]string{"Origin", h.server.URL}) + if resp.StatusCode != http.StatusCreated { + defer resp.Body.Close() + t.Fatalf("create device returned %d, want 201", resp.StatusCode) + } + body := decode(t, resp) + id, _ := body["id"].(string) + if id == "" { + t.Fatal("device has no id") + } + return id +} + +func uploadedDocumentID(t *testing.T, resp *http.Response) string { + t.Helper() + defer resp.Body.Close() + var body struct { + Document struct { + ID string `json:"id"` + MediaType string `json:"mediaType"` + Filename string `json:"filename"` + } `json:"document"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode upload response: %v", err) + } + return body.Document.ID +} + +// TestUploadedHTMLIsNeverServedAsHTML is the regression test for a stored XSS. +// +// The client sets the file part's Content-Type, and it used to be stored verbatim +// and echoed back with `Content-Disposition: inline`. Fetching the "manual" then +// executed its script at this instance's own origin, next to the session cookie — +// reachable by uploading an .html file found on the web, with no deliberate +// attacker. Same-origin also means checkOrigin cannot help. +func TestUploadedHTMLIsNeverServedAsHTML(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + deviceID := h.createDevice(t) + + const payload = `` + documentID := uploadedDocumentID(t, h.upload(t, deviceID, "manual.html", "text/html", []byte(payload))) //nolint:bodyclose // uploadedDocumentID closes it + + resp := h.do(t, http.MethodGet, "/api/v1/documents/"+documentID+"/content", nil) + defer resp.Body.Close() + + if got := resp.Header.Get("Content-Type"); strings.Contains(strings.ToLower(got), "html") { + t.Errorf("Content-Type = %q — HTML must never be served back from this origin", got) + } + if got := resp.Header.Get("Content-Disposition"); !strings.HasPrefix(got, "attachment") { + t.Errorf("Content-Disposition = %q, want attachment", got) + } + if got := resp.Header.Get("X-Content-Type-Options"); got != "nosniff" { + t.Errorf("X-Content-Type-Options = %q, want nosniff — without it the browser may sniff the body as HTML anyway", got) + } +} + +// TestUploadedSVGIsNotInline covers the other executable format. SVG is an XML +// document that can carry script, so it must not be rendered inline even though +// it is nominally an image. +func TestUploadedSVGIsNotInline(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + deviceID := h.createDevice(t) + + const payload = `` + documentID := uploadedDocumentID(t, h.upload(t, deviceID, "diagram.svg", "image/svg+xml", []byte(payload))) //nolint:bodyclose // uploadedDocumentID closes it + + resp := h.do(t, http.MethodGet, "/api/v1/documents/"+documentID+"/content", nil) + defer resp.Body.Close() + + if got := resp.Header.Get("Content-Disposition"); !strings.HasPrefix(got, "attachment") { + t.Errorf("Content-Disposition = %q, want attachment for SVG", got) + } + if got := resp.Header.Get("Content-Type"); strings.Contains(got, "svg") { + t.Errorf("Content-Type = %q, want the type not to be honoured", got) + } +} + +// TestAGenuinePDFIsStillServedInline guards against over-correcting: the whole +// point of the route is to let a browser display a manual. +func TestAGenuinePDFIsStillServedInline(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + deviceID := h.createDevice(t) + + pdf := testpdf.TaggedSections([]string{"EN"}, 2, false).Build() + // Note the lie: the client claims plain text, the bytes are a PDF. Sniffing + // must win in both directions. + documentID := uploadedDocumentID(t, h.upload(t, deviceID, "manual.pdf", "text/plain", pdf)) //nolint:bodyclose // uploadedDocumentID closes it + + resp := h.do(t, http.MethodGet, "/api/v1/documents/"+documentID+"/content", nil) + defer resp.Body.Close() + + if got := resp.Header.Get("Content-Type"); got != "application/pdf" { + t.Errorf("Content-Type = %q, want application/pdf — the bytes decide, not the upload", got) + } + if got := resp.Header.Get("Content-Disposition"); !strings.HasPrefix(got, "inline") { + t.Errorf("Content-Disposition = %q, want inline for a real PDF", got) + } +} + +func TestSafeInlineType(t *testing.T) { + tests := []struct { + stored string + wantServed string + wantInline bool + }{ + {"application/pdf", "application/pdf", true}, + {"image/png", "image/png", true}, + {"image/jpeg", "image/jpeg", true}, + {"text/html; charset=utf-8", "application/octet-stream", false}, + // text/plain is the sniffer's catch-all for unrecognised text, which + // includes SVG and anything else script-bearing, so it is not inline-safe. + {"text/plain; charset=utf-8", "application/octet-stream", false}, + {"image/svg+xml", "application/octet-stream", false}, + {"application/xhtml+xml", "application/octet-stream", false}, + {"", "application/octet-stream", false}, + // Parameters on a safe type are dropped rather than echoed back. + {"application/pdf; charset=binary", "application/pdf", true}, + {"APPLICATION/PDF", "application/pdf", true}, + } + for _, tc := range tests { + served, inline := safeInlineType(tc.stored) + if served != tc.wantServed || inline != tc.wantInline { + t.Errorf("safeInlineType(%q) = %q,%t; want %q,%t", + tc.stored, served, inline, tc.wantServed, tc.wantInline) + } + } +} + +func TestBaseFilenameStripsWindowsPaths(t *testing.T) { + // filepath.Base is platform-specific, so on the Linux server this targets it + // leaves a Windows path completely intact — storing the uploader's directory + // layout, and their username, in the database. + tests := map[string]string{ + `C:\Users\alice\Downloads\manual.pdf`: "manual.pdf", + // Not a /home/... path: CI's hygiene job greps for those to catch a real + // developer's directory leaking into the repo, and it cannot tell a + // fictional name from a real one. Any absolute POSIX path tests the same + // thing. + `/srv/uploads/manual.pdf`: "manual.pdf", + `manual.pdf`: "manual.pdf", + `../../etc/passwd`: "passwd", + ``: ".", + } + for in, want := range tests { + if got := baseFilename(in); got != want { + t.Errorf("baseFilename(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/api/handlers_search.go b/internal/api/handlers_search.go new file mode 100644 index 0000000..b594d48 --- /dev/null +++ b/internal/api/handlers_search.go @@ -0,0 +1,61 @@ +package api + +import ( + "net/http" + "strconv" + + "github.com/gordon2/manualbox/internal/registry" +) + +// handleSearch answers the question README puts first: which manual says X, and +// where. +// +// # Why the query is a parameter and not a body +// +// It is a GET with `?q=`, so a search is a URL: linkable, bookmarkable, and back in +// the browser history where a user expects it. A POST with a JSON body would hide +// the query from all three. +// +// # What the response says beyond the hits +// +// `mode` is which path answered -- the FTS5 index, or the substring scan that +// covers the queries a trigram index cannot represent. `truncated` says the limit +// cut the list off. `indexed` appears only when nothing matched, and it is the +// difference between "no manual says that" and "no manual has been converted yet", +// which are the same empty list otherwise. Each hit carries `bm25` and `score` +// because the gap between them is a judgement about headings, and a number in a +// response can be argued with in a way that one buried in an ORDER BY cannot. +// +// `?documentId=` narrows to one manual, which is what a reader already inside a +// document asks. It is not required: search spans documents by default, because a +// household looking for the descaling interval does not know which manual to open. +// An unknown id is not an error here -- it is a search of nothing, which returns no +// hits, and reporting 404 would turn a scoping parameter into an existence oracle +// on a different route's behalf. +func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + + limit := 0 + if raw := query.Get("limit"); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > registry.MaxSearchLimit { + s.writeError(w, r, http.StatusBadRequest, "invalid_limit", + "limit must be a number between 1 and "+strconv.Itoa(registry.MaxSearchLimit)+".") + return + } + limit = n + } + + results, err := s.deps.Registry.Search(r.Context(), registry.SearchQuery{ + Text: query.Get("q"), + DocumentID: query.Get("documentId"), + Limit: limit, + }) + if err != nil { + // An empty q is registry.ErrInvalid and becomes a 400 with the service's own + // message, rather than an empty result set that reads as "nothing matched". + s.writeRegistryError(w, r, err) + return + } + writeJSON(w, http.StatusOK, results) +} diff --git a/internal/api/handlers_search_test.go b/internal/api/handlers_search_test.go new file mode 100644 index 0000000..9cd87ce --- /dev/null +++ b/internal/api/handlers_search_test.go @@ -0,0 +1,200 @@ +package api + +import ( + "context" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// searchable stores a converted manual through the real registry, and returns the +// document's id. The API's job is to shape the answer; the index is internal/db's +// and internal/registry's business and is tested there. +func (h *harness) searchable(t *testing.T, deviceName, filename, digest string, blocks ...doc.Block) string { + t.Helper() + ctx := context.Background() + + device, err := h.registry.CreateDevice(ctx, registry.NewDevice{Name: deviceName}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat(digest, 32), Size: 10} + if err := h.registry.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := h.registry.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, Filename: filename, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + if err := h.registry.SaveConversion(ctx, document.ID, blocks, nil, nil, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + return document.ID +} + +func para(page int, lang, text string) doc.Block { + return doc.Block{ + Page: page, RegionX0: 43, Index: 0, Kind: doc.BlockParagraph, + Text: text, Lang: lang, X0: 43, X1: 300, Y0: 100, Y1: 118, + Lines: 1, Chars: len([]rune(text)), + } +} + +// TestSearchEndpointSaysWhichManualAndWhere is the endpoint's whole job: a GET with +// a query string, answering across the household. +func TestSearchEndpointSaysWhichManualAndWhere(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + + vacuum := h.searchable(t, "Vacuum cleaner", "thomas-drybox.pdf", "a", + para(48, "de", "Den Ausblasfilter alle zwei Jahre tauschen.")) + h.searchable(t, "Washing machine", "washer.pdf", "b", + para(7, "de", "Den Flusenfilter nach jedem Waschgang reinigen.")) + + body := decode(t, h.do(t, http.MethodGet, "/api/v1/search?q=Filter", nil)) //nolint:bodyclose // decode closes it + if body["mode"] != "index" { + t.Errorf("mode = %v, want index", body["mode"]) + } + if body["query"] != "Filter" { + t.Errorf("query = %v, want it echoed", body["query"]) + } + if body["truncated"] != false { + t.Errorf("truncated = %v, want false", body["truncated"]) + } + if _, ok := body["indexed"]; ok { + t.Errorf("indexed = %v on a search that matched; it is only for an empty result", + body["indexed"]) + } + hits, ok := body["hits"].([]any) + if !ok || len(hits) != 2 { + t.Fatalf("hits = %v, want 2 across both manuals", body["hits"]) + } + first, ok := hits[0].(map[string]any) + if !ok { + t.Fatalf("hit is not an object: %v", hits[0]) + } + for _, key := range []string{ + "documentId", "filename", "deviceId", "deviceName", "state", + "page", "regionX0", "index", "kind", "lang", "name", "snippet", "chars", + "bm25", "score", + } { + if _, ok := first[key]; !ok { + t.Errorf("hit is missing %q: %v", key, first) + } + } + + // Narrowed to one manual. + one := decode(t, h.do(t, http.MethodGet, //nolint:bodyclose // decode closes it + "/api/v1/search?q=Filter&documentId="+vacuum, nil)) + narrowed, ok := one["hits"].([]any) + if !ok || len(narrowed) != 1 { + t.Fatalf("narrowed hits = %v, want 1", one["hits"]) + } + if got := narrowed[0].(map[string]any)["documentId"]; got != vacuum { + t.Errorf("narrowed hit came from %v, want %s", got, vacuum) + } +} + +// TestSearchEndpointTakesTheQueryLiterally. A search box has no query language, so +// FTS5 syntax in a URL parameter must be text rather than an expression -- and a +// 500 from a background parser is the worst possible answer to a typo. +func TestSearchEndpointTakesTheQueryLiterally(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + h.searchable(t, "Vacuum cleaner", "manual.pdf", "a", + para(1, "de", `Fehler: "Motor laeuft NICHT" - Filter pruefen.`)) + + for _, q := range []string{ + `Fehler:`, `"Motor`, `Motor NOT laeuft`, `Filter*`, `NEAR(a b)`, `{x}`, `^a`, + // Percent and underscore would be wildcards on the scan path, which is the + // path a two-character query takes. + `%`, `_`, `%%`, + } { + code := h.status(t, http.MethodGet, "/api/v1/search?q="+url.QueryEscape(q), nil) + if code != http.StatusOK { + t.Errorf("GET /search?q=%q returned %d, want 200", q, code) + } + } +} + +func TestSearchEndpointValidation(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + + // No q at all, and whitespace-only q: both are 400 rather than an empty result + // that reads as "nothing in the house says that". + for _, path := range []string{ + "/api/v1/search", + "/api/v1/search?q=", + "/api/v1/search?q=%20%20", + } { + if code := h.status(t, http.MethodGet, path, nil); code != http.StatusBadRequest { + t.Errorf("GET %s returned %d, want 400", path, code) + } + } + + for _, path := range []string{ + "/api/v1/search?q=Filter&limit=0", + "/api/v1/search?q=Filter&limit=-1", + "/api/v1/search?q=Filter&limit=nope", + "/api/v1/search?q=Filter&limit=101", + } { + if code := h.status(t, http.MethodGet, path, nil); code != http.StatusBadRequest { + t.Errorf("GET %s returned %d, want 400", path, code) + } + } + if code := h.status(t, http.MethodGet, "/api/v1/search?q=Filter&limit=100", nil); code != http.StatusOK { + t.Errorf("the maximum limit was refused, %d", code) + } +} + +// TestSearchEndpointOnAnEmptyLibrarySaysSo. Nothing converted yet is not a search +// failure, and it is not the same answer as "no manual says that". +func TestSearchEndpointOnAnEmptyLibrarySaysSo(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + + body := decode(t, h.do(t, http.MethodGet, "/api/v1/search?q=Saugkraft", nil)) //nolint:bodyclose // decode closes it + hits, ok := body["hits"].([]any) + if !ok || len(hits) != 0 { + t.Errorf("hits = %v, want an empty array rather than null", body["hits"]) + } + indexed, ok := body["indexed"].(float64) + if !ok || indexed != 0 { + t.Errorf("indexed = %v, want 0", body["indexed"]) + } +} + +// TestSearchEndpointReportsTheScanPath. A two-character query cannot be in a +// trigram index, so the response says which path answered it rather than leaving a +// client to wonder why the ranking is flat. +func TestSearchEndpointReportsTheScanPath(t *testing.T) { + h := newHarness(t) + h.completeSetup(t) + h.searchable(t, "Robot vacuum", "dreame-l40.pdf", "a", + para(541, "ja", "電源を入れる前に取扱説明書をお読みください。")) + + body := decode(t, h.do(t, http.MethodGet, //nolint:bodyclose // decode closes it + "/api/v1/search?q="+url.QueryEscape("電源"), nil)) + if body["mode"] != "substring" { + t.Errorf("mode = %v, want substring", body["mode"]) + } + hits, ok := body["hits"].([]any) + if !ok || len(hits) != 1 { + t.Fatalf("hits = %v, want the one Japanese block", body["hits"]) + } + + indexed := decode(t, h.do(t, http.MethodGet, //nolint:bodyclose // decode closes it + "/api/v1/search?q="+url.QueryEscape("取扱説明書"), nil)) + if indexed["mode"] != "index" { + t.Errorf("a five-character Japanese query ran as %v, want index", indexed["mode"]) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 83f7b15..5a9cb44 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -28,6 +28,7 @@ const EnvPrefix = "MANUALBOX_" type Config struct { Server Server `yaml:"server"` Content Content `yaml:"content"` + Ingest Ingest `yaml:"ingest" envPrefix:"INGEST_"` Providers Providers `yaml:"providers"` Jobs Jobs `yaml:"jobs"` // LOG_ prefix: bare MANUALBOX_LEVEL and MANUALBOX_FORMAT would be ambiguous @@ -70,6 +71,19 @@ type Content struct { OCRLanguages []string `yaml:"ocr_languages" env:"OCR_LANGUAGES" envSeparator:","` } +// Ingest bounds what the document pipeline will do without being asked. +type Ingest struct { + // MaxPagesAuto is the largest document that may be processed automatically + // after probing. Above it, the document is stored and probed — both free — + // and the user is asked before anything is converted or sent to a provider. + // + // The default is deliberately low. A real appliance manual runs to 560 pages + // in 34 languages, of which a household reads perhaps 16; converting the whole + // thing costs two orders of magnitude more than converting the part that + // matters. A large upload must never silently become a bill. + MaxPagesAuto int `yaml:"max_pages_auto" env:"MAX_PAGES_AUTO"` +} + // Providers configures the pluggable adapters. Every slot may be empty; an // empty slot disables the corresponding feature rather than failing. type Providers struct { @@ -134,6 +148,9 @@ func Default() Config { Content: Content{ Languages: []string{"en"}, }, + Ingest: Ingest{ + MaxPagesAuto: 64, + }, Providers: Providers{ // Local and free by default. Convert: Provider{Kind: "poppler"}, @@ -237,6 +254,10 @@ func (c Config) Validate() error { } } + if c.Ingest.MaxPagesAuto < 1 { + errs = append(errs, errors.New("ingest.max_pages_auto must be at least 1")) + } + if c.Jobs.Workers < 1 { errs = append(errs, errors.New("jobs.workers must be at least 1")) } diff --git a/internal/db/docblocks_generated_test.go b/internal/db/docblocks_generated_test.go new file mode 100644 index 0000000..53a07b3 --- /dev/null +++ b/internal/db/docblocks_generated_test.go @@ -0,0 +1,258 @@ +package db + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/id" +) + +// TestDocBlockQueriesExecute is a SMOKE TEST FOR THE GENERATOR, not a behaviour +// test, and it is the twin of TestDocRegionQueriesExecute. It asserts only that +// every generated doc_blocks and doc_figures statement prepares and runs against a +// real migrated database and returns something coherent; what the pipeline does +// with them is internal/registry's business. +// +// It exists for the reason set out at length above TestDocRegionQueriesExecute and +// in the header of queries/docblocks.sql: sqlc v1.31.1 silently truncates the tail +// of a generated statement when a query file contains a non-ASCII character. `make +// sqlc` exits 0, the Go compiles, the linter passes, and the statement fails at +// PREPARE time inside a background job against a user's database. Executing each +// statement once is the cheapest thing that turns that into a build failure. +// +// The ORDER BY clauses are asserted rather than assumed, because a truncated tail +// is exactly what the bug eats and an unordered result is what a reader would see +// as scrambled prose. +func TestDocBlockQueriesExecute(t *testing.T) { + ctx := context.Background() + + database, err := Open(ctx, Options{Path: filepath.Join(t.TempDir(), "blocks.db")}) + if err != nil { + t.Fatalf("open database: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + + // Parents first: both tables cascade from documents, and doc_figures also + // references blobs. + w := gen.New(database.Write()) + docID, deviceID := id.New(id.Document), id.New(id.Device) + sha := strings.Repeat("a", 64) + figureSHA := strings.Repeat("b", 64) + for _, digest := range []string{sha, figureSHA} { + if err := w.UpsertBlob(ctx, gen.UpsertBlobParams{ + Sha256: digest, SizeBytes: 1, MediaType: "application/pdf", CreatedAt: Now(), + }); err != nil { + t.Fatalf("blob %s: %v", digest[:4], err) + } + } + if _, err := database.Write().ExecContext(ctx, + `INSERT INTO devices (id, name, created_at, updated_at) VALUES (?, 'Dryer', ?, ?)`, + deviceID, Now(), Now()); err != nil { + t.Fatalf("device: %v", err) + } + if _, err := w.CreateDocument(ctx, gen.CreateDocumentParams{ + ID: docID, DeviceID: deviceID, BlobSha256: sha, Filename: "manual.pdf", + Kind: "manual", State: "uploaded", CreatedAt: Now(), UpdatedAt: Now(), + }); err != nil { + t.Fatalf("document: %v", err) + } + + // Two regions of different languages on one page, plus a whole-page region on + // another, plus a block nothing could name a language for. Between them these + // exercise every column, every kind that is produced today, and the two states + // the schema calls out: region_x0 = 0 for a whole page, and lang = ''. + blocks := []gen.UpsertDocBlockParams{ + {Page: 2, RegionX0: 43, Idx: 0, Kind: "heading", Level: 1, Text: "Sicherheitshinweise", + Lang: "de", X0: 43.2, X1: 300.8, Y0: 100.5, Y1: 118.5, Lines: 1, Chars: 19, + Note: "18pt bold at 19 characters"}, + {Page: 2, RegionX0: 43, Idx: 1, Kind: "paragraph", Text: "Lesen Sie diese Anleitung.", + Lang: "de", X0: 43.2, X1: 304.9, Y0: 122.0, Y1: 170.0, Lines: 3, Chars: 26}, + {Page: 2, RegionX0: 323, Idx: 0, Kind: "list-item", Text: "Przeczytaj instrukcje.", + Lang: "pl", X0: 323.4, X1: 585.1, Y0: 122.0, Y1: 138.0, Lines: 1, Chars: 22}, + {Page: 3, RegionX0: 0, Idx: 0, Kind: "table", Text: "230 V, 50 Hz", Lang: "uk", + X0: 30.0, X1: 400.0, Y0: 90.0, Y1: 106.0, Lines: 1, Chars: 12, + Note: "row 2, column 1 of 2"}, + {Page: 3, RegionX0: 0, Idx: 1, Kind: "paragraph", Text: "62", Lang: "", + X0: 430.0, X1: 445.0, Y0: 1150.0, Y1: 1166.0, Lines: 1, Chars: 2, + Note: "no language established for this region"}, + } + for i := range blocks { + blocks[i].DocumentID = docID + blocks[i].CreatedAt = Now() + if err := w.UpsertDocBlock(ctx, blocks[i]); err != nil { + t.Fatalf("UpsertDocBlock %d: %v", i, err) + } + } + + figures := []gen.UpsertDocFigureParams{ + {Page: 2, Idx: 0, X0: 43, Y0: 200, X1: 300, Y1: 460, Ink: 42, TextFraction: 0.02, + Dpi: 216, PixelWidth: 514, PixelHeight: 520, BlobSha256: figureSHA}, + {Page: 3, Idx: 0, X0: 60, Y0: 300, X1: 500, Y1: 700, Ink: 17, TextFraction: 0.11, + Dpi: 216, PixelWidth: 880, PixelHeight: 800, BlobSha256: figureSHA}, + } + for i := range figures { + figures[i].DocumentID = docID + figures[i].CreatedAt = Now() + if err := w.UpsertDocFigure(ctx, figures[i]); err != nil { + t.Fatalf("UpsertDocFigure %d: %v", i, err) + } + } + + // THE UPSERT ITSELF, which nothing else covers. registry.SaveConversion deletes + // before it inserts, so its idempotency test passes with the ON CONFLICT clause + // removed entirely -- measured, not assumed. What the clause is actually for is a + // retry WITHIN one conversion, where the delete has already happened and the same + // key is written twice, and this is the only place that case is exercised. + // + // The re-write changes kind, which is deliberately NOT in the key: a paragraph + // that a better heading rule promotes to a heading must update in place rather + // than become a second row at the same index. + reWrite := blocks[1] + reWrite.Kind = "heading" + reWrite.Level = 2 + reWrite.Text = "Lesen Sie diese Anleitung" + reWrite.Chars = 25 + if err := w.UpsertDocBlock(ctx, reWrite); err != nil { + t.Fatalf("UpsertDocBlock over an existing key: %v", err) + } + if err := w.UpsertDocFigure(ctx, figures[0]); err != nil { + t.Fatalf("UpsertDocFigure over an existing key: %v", err) + } + + r := gen.New(database.Read()) + + rewritten, err := r.ListDocBlocksForPage(ctx, gen.ListDocBlocksForPageParams{ + DocumentID: docID, Page: 2, + }) + if err != nil { + t.Fatalf("ListDocBlocksForPage after re-upsert: %v", err) + } + if len(rewritten) != 3 { + t.Errorf("re-upserting one block left %d rows on page 2, want 3: the ON CONFLICT "+ + "clause did not match and the block was duplicated", len(rewritten)) + } + for i := range rewritten { + b := &rewritten[i] + if b.RegionX0 == 43 && b.Idx == 1 && (b.Kind != "heading" || b.Level != 2 || b.Chars != 25) { + t.Errorf("the re-upserted block did not take the new values: %+v", b) + } + } + if figs, err := r.ListDocFigures(ctx, docID); err != nil || len(figs) != 2 { + t.Errorf("re-upserting a figure left %d rows, want 2 (err %v)", len(figs), err) + } + + all, err := r.ListDocBlocks(ctx, docID) + if err != nil { + t.Fatalf("ListDocBlocks: %v", err) + } + if len(all) != len(blocks) { + t.Errorf("ListDocBlocks returned %d rows, want %d", len(all), len(blocks)) + } + // ORDER BY page, region_x0, idx: three sort keys, and the tail of that clause is + // precisely what the truncation bug eats. + for i := 1; i < len(all); i++ { + prev, cur := &all[i-1], &all[i] + before := prev.Page < cur.Page || + (prev.Page == cur.Page && prev.RegionX0 < cur.RegionX0) || + (prev.Page == cur.Page && prev.RegionX0 == cur.RegionX0 && prev.Idx < cur.Idx) + if !before { + t.Errorf("ListDocBlocks is not ordered by page, region_x0, idx: row %d is "+ + "(%d, %d, %d), row %d is (%d, %d, %d)", + i-1, prev.Page, prev.RegionX0, prev.Idx, + i, cur.Page, cur.RegionX0, cur.Idx) + } + } + + byLang, err := r.ListDocBlocksByLang(ctx, gen.ListDocBlocksByLangParams{ + DocumentID: docID, Lang: "de", + }) + if err != nil { + t.Fatalf("ListDocBlocksByLang: %v", err) + } + if len(byLang) != 2 { + t.Errorf("de has %d blocks, want 2", len(byLang)) + } + // And the unnamed content stays reachable by asking for it. + unnamed, err := r.ListDocBlocksByLang(ctx, gen.ListDocBlocksByLangParams{ + DocumentID: docID, Lang: "", + }) + if err != nil { + t.Fatalf("ListDocBlocksByLang(''): %v", err) + } + if len(unnamed) != 1 { + t.Errorf("%d blocks have no language, want 1", len(unnamed)) + } + + onPage, err := r.ListDocBlocksForPage(ctx, gen.ListDocBlocksForPageParams{ + DocumentID: docID, Page: 2, + }) + if err != nil { + t.Fatalf("ListDocBlocksForPage: %v", err) + } + if len(onPage) != 3 { + t.Errorf("page 2 has %d blocks, want 3", len(onPage)) + } + + summary, err := r.SummarizeDocBlocks(ctx, docID) + if err != nil { + t.Fatalf("SummarizeDocBlocks: %v", err) + } + // Four labels: de, pl, uk and the unnamed one. The aggregates must be int64 and + // not interface{} -- that is what the CASTs buy, and it is a compile-time + // assertion as much as a runtime one. + if len(summary) != 4 { + t.Errorf("SummarizeDocBlocks returned %d rows, want 4: %+v", len(summary), summary) + } + for i := range summary { + s := &summary[i] + if s.Lang != "de" { + continue + } + if s.Blocks != 2 || s.Chars != 44 || s.Lines != 4 || s.Pages != 1 || s.FirstPage != 2 { + t.Errorf("de summary = %+v; want blocks 2, chars 44, lines 4, pages 1, first_page 2", s) + } + } + for i := 1; i < len(summary); i++ { + if summary[i-1].FirstPage > summary[i].FirstPage { + t.Errorf("SummarizeDocBlocks is not ordered by first_page: %+v", summary) + } + } + + figs, err := r.ListDocFigures(ctx, docID) + if err != nil { + t.Fatalf("ListDocFigures: %v", err) + } + if len(figs) != 2 { + t.Errorf("ListDocFigures returned %d rows, want 2", len(figs)) + } + for i := 1; i < len(figs); i++ { + if figs[i-1].Page > figs[i].Page { + t.Errorf("ListDocFigures is not ordered by page: %+v", figs) + } + } + figsOnPage, err := r.ListDocFiguresForPage(ctx, gen.ListDocFiguresForPageParams{ + DocumentID: docID, Page: 3, + }) + if err != nil { + t.Fatalf("ListDocFiguresForPage: %v", err) + } + if len(figsOnPage) != 1 { + t.Errorf("page 3 has %d figures, want 1", len(figsOnPage)) + } + + if err := r.DeleteDocBlocks(ctx, docID); err != nil { + t.Fatalf("DeleteDocBlocks: %v", err) + } + if left, err := r.ListDocBlocks(ctx, docID); err != nil || len(left) != 0 { + t.Errorf("%d blocks survived DeleteDocBlocks (err %v)", len(left), err) + } + if err := r.DeleteDocFigures(ctx, docID); err != nil { + t.Fatalf("DeleteDocFigures: %v", err) + } + if left, err := r.ListDocFigures(ctx, docID); err != nil || len(left) != 0 { + t.Errorf("%d figures survived DeleteDocFigures (err %v)", len(left), err) + } +} diff --git a/internal/db/docregions_generated_test.go b/internal/db/docregions_generated_test.go new file mode 100644 index 0000000..0e9718f --- /dev/null +++ b/internal/db/docregions_generated_test.go @@ -0,0 +1,194 @@ +package db + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf8" + + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/id" +) + +// TestDocRegionQueriesExecute is a SMOKE TEST FOR THE GENERATOR, not a behaviour +// test. It asserts only that every generated doc_regions query prepares and runs +// against a real migrated database and returns something coherent; what the +// pipeline does with them is internal/registry's business. +// +// It exists because sqlc v1.31.1 (pinned in tools/go.mod, built to ./bin/sqlc) +// silently TRUNCATES the tail of a generated statement when a queries/*.sql file +// contains a non-ASCII character. It confuses character and byte offsets when +// cutting statements out of the file; the measured rule is that a statement loses +// one character of SQL per extra byte those characters occupy, and every statement +// after the character is affected. Measured while writing docregions.sql: one +// em-dash in a comment turned "ORDER BY first_page, code" into +// "ORDER BY first_page, co", and four em-dashes turned it into "ORDER BY first_pa". +// Placed elsewhere in a file the same corruption instead breaks sqlc's own parser +// and it exits loudly, so neither outcome is a reliable signal. See the header of +// queries/docregions.sql. +// +// Nothing upstream catches that. `make sqlc` exits 0, the generated Go compiles, +// the linter passes, and the statement fails at PREPARE time inside a background +// job against a user's database. Executing each statement once is the cheapest +// thing that turns it into a build failure, and a mangled statement names itself in +// the error. +func TestDocRegionQueriesExecute(t *testing.T) { + ctx := context.Background() + + database, err := Open(ctx, Options{Path: filepath.Join(t.TempDir(), "regions.db")}) + if err != nil { + t.Fatalf("open database: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + + // Parents first: doc_regions cascades from documents. + w := gen.New(database.Write()) + docID, deviceID := id.New(id.Document), id.New(id.Device) + sha := strings.Repeat("a", 64) + if err := w.UpsertBlob(ctx, gen.UpsertBlobParams{ + Sha256: sha, SizeBytes: 1, MediaType: "application/pdf", CreatedAt: Now(), + }); err != nil { + t.Fatalf("blob: %v", err) + } + if _, err := database.Write().ExecContext(ctx, + `INSERT INTO devices (id, name, created_at, updated_at) VALUES (?, 'Dryer', ?, ?)`, + deviceID, Now(), Now()); err != nil { + t.Fatalf("device: %v", err) + } + if _, err := w.CreateDocument(ctx, gen.CreateDocumentParams{ + ID: docID, DeviceID: deviceID, BlobSha256: sha, Filename: "manual.pdf", + Kind: "manual", State: "uploaded", CreatedAt: Now(), UpdatedAt: Now(), + }); err != nil { + t.Fatalf("document: %v", err) + } + + // Two boxed regions of one language on one page, plus a whole-page region on + // another, plus one region no signal could name. Between them these exercise + // every column and both of the states the schema calls out: source = '' and a + // page holding more than one region. + regions := []gen.UpsertDocRegionParams{ + {Source: "repertoire", Page: 2, X0: 43, X1: 305, Code: "D", Lang: "de", Chars: 900, Runs: 50}, + {Source: "repertoire", Page: 2, X0: 323, X1: 585, Code: "D", Lang: "de", Chars: 880, Runs: 47}, + {Source: "page-tag", Page: 3, X0: 0, X1: 892, Code: "UA", Lang: "uk", Chars: 1700, Runs: 96, + Conflict: 1, Note: "the page reads as Ukrainian, but 1 of its 2 columns read as Kazakh"}, + {Source: "", Page: 4, X0: 0, X1: 892, Chars: 120, Runs: 12, + Note: "no language established for this page"}, + } + for i := range regions { + regions[i].DocumentID = docID + regions[i].CreatedAt = Now() + if err := w.UpsertDocRegion(ctx, regions[i]); err != nil { + t.Fatalf("UpsertDocRegion %d: %v", i, err) + } + } + + r := gen.New(database.Read()) + + all, err := r.ListDocRegions(ctx, docID) + if err != nil { + t.Fatalf("ListDocRegions: %v", err) + } + if len(all) != len(regions) { + t.Errorf("ListDocRegions returned %d rows, want %d", len(all), len(regions)) + } + // The ORDER BY is the clause the truncation bug ate, so it is asserted rather + // than assumed: page ascending, then x0 ascending within a page. + for i := 1; i < len(all); i++ { + prev, cur := &all[i-1], &all[i] + if prev.Page > cur.Page || (prev.Page == cur.Page && prev.X0 >= cur.X0) { + t.Errorf("ListDocRegions is not ordered by page, x0: row %d is (%d, %d), row %d is (%d, %d)", + i-1, prev.Page, prev.X0, i, cur.Page, cur.X0) + } + } + + onPage, err := r.ListDocRegionsForPage(ctx, gen.ListDocRegionsForPageParams{ + DocumentID: docID, Page: 2, + }) + if err != nil { + t.Fatalf("ListDocRegionsForPage: %v", err) + } + if len(onPage) != 2 { + t.Errorf("page 2 has %d regions, want 2", len(onPage)) + } + + summary, err := r.SummarizeDocRegions(ctx, docID) + if err != nil { + t.Fatalf("SummarizeDocRegions: %v", err) + } + // Three labels: D/de, UA/uk, and the unnamed one. + if len(summary) != 3 { + t.Errorf("SummarizeDocRegions returned %d rows, want 3: %+v", len(summary), summary) + } + // The aggregates must be int64, not interface{} — that is what the CASTs buy, + // and it is a compile-time assertion as much as a runtime one. Also confirms the + // two same-language columns were summed rather than one of them being lost. + for i := range summary { + s := &summary[i] + if s.Lang != "de" { + continue + } + if s.Chars != 1780 || s.Runs != 97 || s.Pages != 1 || s.FirstPage != 2 { + t.Errorf("de summary = %+v; want chars 1780, runs 97, pages 1, first_page 2", s) + } + } + // ORDER BY first_page, code: the second sort key is the other half of the clause + // the bug truncated. + for i := 1; i < len(summary); i++ { + if summary[i-1].FirstPage > summary[i].FirstPage { + t.Errorf("SummarizeDocRegions is not ordered by first_page: %+v", summary) + } + } + + if err := r.DeleteDocRegions(ctx, docID); err != nil { + t.Fatalf("DeleteDocRegions: %v", err) + } + left, err := r.ListDocRegions(ctx, docID) + if err != nil { + t.Fatalf("ListDocRegions after delete: %v", err) + } + if len(left) != 0 { + t.Errorf("%d regions survived DeleteDocRegions", len(left)) + } +} + +// TestQueryFilesAreASCII is the cause-side guard for the generator bug that +// TestDocRegionQueriesExecute catches symptomatically. +// +// sqlc v1.31.1 corrupts generated statements when a queries/*.sql file contains a +// non-ASCII character, losing one character of SQL for every extra byte such +// characters occupy, in every statement after them. All the query files were pure +// ASCII when this was written — 0 non-ASCII bytes across all ten, measured rather +// than assumed — which is the only reason the bug had never fired here. The +// codebase's prose comments elsewhere use em-dashes freely, so the first person to +// write one in a query comment would have shipped invalid SQL that generates and +// compiles cleanly. +// +// Restricting these files to ASCII costs nothing — they are SQL and identifiers — +// and it removes the whole failure mode rather than one instance of it. +func TestQueryFilesAreASCII(t *testing.T) { + files, err := filepath.Glob(filepath.Join("queries", "*.sql")) + if err != nil { + t.Fatalf("glob: %v", err) + } + if len(files) == 0 { + t.Fatal("found no query files; this guard would pass vacuously") + } + for _, f := range files { + raw, err := os.ReadFile(f) + if err != nil { + t.Fatalf("read %s: %v", f, err) + } + body := string(raw) + for i, r := range body { + if r >= utf8.RuneSelf { + line := 1 + strings.Count(body[:i], "\n") + t.Errorf("%s:%d contains the non-ASCII character %q. sqlc v1.31.1 will "+ + "silently truncate the tail of every statement after it; use plain "+ + "ASCII in query files.", f, line, r) + break + } + } + } +} diff --git a/internal/db/gen/devices.sql.go b/internal/db/gen/devices.sql.go new file mode 100644 index 0000000..a95565c --- /dev/null +++ b/internal/db/gen/devices.sql.go @@ -0,0 +1,227 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: devices.sql + +package gen + +import ( + "context" +) + +const countDevices = `-- name: CountDevices :one +SELECT CAST(count(*) AS INTEGER) AS total FROM devices +` + +func (q *Queries) CountDevices(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countDevices) + var total int64 + err := row.Scan(&total) + return total, err +} + +const createDevice = `-- name: CreateDevice :one +INSERT INTO devices (id, name, brand, model, category, location_id, notes, purchased_at, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +RETURNING id, name, brand, model, category, location_id, notes, purchased_at, created_at, updated_at +` + +type CreateDeviceParams struct { + ID string + Name string + Brand string + Model string + Category string + LocationID *string + Notes string + PurchasedAt *int64 + CreatedAt int64 + UpdatedAt int64 +} + +func (q *Queries) CreateDevice(ctx context.Context, arg CreateDeviceParams) (Device, error) { + row := q.db.QueryRowContext(ctx, createDevice, + arg.ID, + arg.Name, + arg.Brand, + arg.Model, + arg.Category, + arg.LocationID, + arg.Notes, + arg.PurchasedAt, + arg.CreatedAt, + arg.UpdatedAt, + ) + var i Device + err := row.Scan( + &i.ID, + &i.Name, + &i.Brand, + &i.Model, + &i.Category, + &i.LocationID, + &i.Notes, + &i.PurchasedAt, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const deleteDevice = `-- name: DeleteDevice :exec +DELETE FROM devices WHERE id = ? +` + +func (q *Queries) DeleteDevice(ctx context.Context, id string) error { + _, err := q.db.ExecContext(ctx, deleteDevice, id) + return err +} + +const getDevice = `-- name: GetDevice :one +SELECT id, name, brand, model, category, location_id, notes, purchased_at, created_at, updated_at FROM devices WHERE id = ? +` + +func (q *Queries) GetDevice(ctx context.Context, id string) (Device, error) { + row := q.db.QueryRowContext(ctx, getDevice, id) + var i Device + err := row.Scan( + &i.ID, + &i.Name, + &i.Brand, + &i.Model, + &i.Category, + &i.LocationID, + &i.Notes, + &i.PurchasedAt, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const listDevices = `-- name: ListDevices :many +SELECT id, name, brand, model, category, location_id, notes, purchased_at, created_at, updated_at FROM devices ORDER BY name +` + +func (q *Queries) ListDevices(ctx context.Context) ([]Device, error) { + rows, err := q.db.QueryContext(ctx, listDevices) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Device{} + for rows.Next() { + var i Device + if err := rows.Scan( + &i.ID, + &i.Name, + &i.Brand, + &i.Model, + &i.Category, + &i.LocationID, + &i.Notes, + &i.PurchasedAt, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDevicesByLocation = `-- name: ListDevicesByLocation :many +SELECT id, name, brand, model, category, location_id, notes, purchased_at, created_at, updated_at FROM devices WHERE location_id = ? ORDER BY name +` + +// Filtering by location is a separate query rather than a nullable parameter on +// ListDevices. CONTRIBUTING.md: an "IS NULL OR =" filter defeats sqlc's type +// inference and reads worse than two explicit queries. +func (q *Queries) ListDevicesByLocation(ctx context.Context, locationID *string) ([]Device, error) { + rows, err := q.db.QueryContext(ctx, listDevicesByLocation, locationID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Device{} + for rows.Next() { + var i Device + if err := rows.Scan( + &i.ID, + &i.Name, + &i.Brand, + &i.Model, + &i.Category, + &i.LocationID, + &i.Notes, + &i.PurchasedAt, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateDevice = `-- name: UpdateDevice :one +UPDATE devices +SET name = ?, brand = ?, model = ?, category = ?, location_id = ?, notes = ?, + purchased_at = ?, updated_at = ? +WHERE id = ? +RETURNING id, name, brand, model, category, location_id, notes, purchased_at, created_at, updated_at +` + +type UpdateDeviceParams struct { + Name string + Brand string + Model string + Category string + LocationID *string + Notes string + PurchasedAt *int64 + UpdatedAt int64 + ID string +} + +func (q *Queries) UpdateDevice(ctx context.Context, arg UpdateDeviceParams) (Device, error) { + row := q.db.QueryRowContext(ctx, updateDevice, + arg.Name, + arg.Brand, + arg.Model, + arg.Category, + arg.LocationID, + arg.Notes, + arg.PurchasedAt, + arg.UpdatedAt, + arg.ID, + ) + var i Device + err := row.Scan( + &i.ID, + &i.Name, + &i.Brand, + &i.Model, + &i.Category, + &i.LocationID, + &i.Notes, + &i.PurchasedAt, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/db/gen/docblocks.sql.go b/internal/db/gen/docblocks.sql.go new file mode 100644 index 0000000..dcb1cde --- /dev/null +++ b/internal/db/gen/docblocks.sql.go @@ -0,0 +1,626 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: docblocks.sql + +package gen + +import ( + "context" +) + +const deleteDocBlocks = `-- name: DeleteDocBlocks :exec +DELETE FROM doc_blocks WHERE document_id = ? +` + +// Replacing a document's blocks wholesale is how a re-conversion stays honest, +// and it is required rather than merely tidy: a region that converted to 12 +// blocks and now converts to 9 would otherwise keep rows at idx 9, 10 and 11, +// which a reader renders as three paragraphs of the previous run's text. +func (q *Queries) DeleteDocBlocks(ctx context.Context, documentID string) error { + _, err := q.db.ExecContext(ctx, deleteDocBlocks, documentID) + return err +} + +const deleteDocFigureLabels = `-- name: DeleteDocFigureLabels :exec +DELETE FROM doc_figure_labels WHERE document_id = ? +` + +// Deleting a document's figures already cascades to its labels, so this exists for +// the one case the cascade does not cover: rewriting the labels of figures that are +// themselves unchanged. Cheaper and clearer than reasoning about which rows the +// figure delete happened to take with it. +func (q *Queries) DeleteDocFigureLabels(ctx context.Context, documentID string) error { + _, err := q.db.ExecContext(ctx, deleteDocFigureLabels, documentID) + return err +} + +const deleteDocFigures = `-- name: DeleteDocFigures :exec +DELETE FROM doc_figures WHERE document_id = ? +` + +func (q *Queries) DeleteDocFigures(ctx context.Context, documentID string) error { + _, err := q.db.ExecContext(ctx, deleteDocFigures, documentID) + return err +} + +const listDocBlocks = `-- name: ListDocBlocks :many +SELECT document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at +FROM doc_blocks +WHERE document_id = ? +ORDER BY page, region_x0, idx +` + +// Reading order across the whole document: down the pages, then left to right +// across each, then in order within a region. A whole-page region sorts first on +// its page because its region_x0 is 0. +func (q *Queries) ListDocBlocks(ctx context.Context, documentID string) ([]DocBlock, error) { + rows, err := q.db.QueryContext(ctx, listDocBlocks, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocBlock{} + for rows.Next() { + var i DocBlock + if err := rows.Scan( + &i.DocumentID, + &i.Page, + &i.RegionX0, + &i.Idx, + &i.Kind, + &i.Level, + &i.Text, + &i.Lang, + &i.X0, + &i.X1, + &i.Y0, + &i.Y1, + &i.Lines, + &i.Chars, + &i.Note, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocBlocksByLang = `-- name: ListDocBlocksByLang :many +SELECT document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at +FROM doc_blocks +WHERE document_id = ? AND lang = ? +ORDER BY page, region_x0, idx +` + +type ListDocBlocksByLangParams struct { + DocumentID string + Lang string +} + +// The funnel's own query: one household's language, and nothing else. A German +// reader of the columns manual gets the German column of each page rather than +// the page, which conversion.md measures as a fifth of the work. +// +// Blocks whose language was never established have lang = ” and are therefore +// NOT returned by any language's query. That is deliberate rather than an +// oversight: passing ” asks for exactly those, which is how the unnamed content +// of a document stays reachable instead of becoming invisible. +func (q *Queries) ListDocBlocksByLang(ctx context.Context, arg ListDocBlocksByLangParams) ([]DocBlock, error) { + rows, err := q.db.QueryContext(ctx, listDocBlocksByLang, arg.DocumentID, arg.Lang) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocBlock{} + for rows.Next() { + var i DocBlock + if err := rows.Scan( + &i.DocumentID, + &i.Page, + &i.RegionX0, + &i.Idx, + &i.Kind, + &i.Level, + &i.Text, + &i.Lang, + &i.X0, + &i.X1, + &i.Y0, + &i.Y1, + &i.Lines, + &i.Chars, + &i.Note, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocBlocksForPage = `-- name: ListDocBlocksForPage :many +SELECT document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at +FROM doc_blocks +WHERE document_id = ? AND page = ? +ORDER BY region_x0, idx +` + +type ListDocBlocksForPageParams struct { + DocumentID string + Page int64 +} + +func (q *Queries) ListDocBlocksForPage(ctx context.Context, arg ListDocBlocksForPageParams) ([]DocBlock, error) { + rows, err := q.db.QueryContext(ctx, listDocBlocksForPage, arg.DocumentID, arg.Page) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocBlock{} + for rows.Next() { + var i DocBlock + if err := rows.Scan( + &i.DocumentID, + &i.Page, + &i.RegionX0, + &i.Idx, + &i.Kind, + &i.Level, + &i.Text, + &i.Lang, + &i.X0, + &i.X1, + &i.Y0, + &i.Y1, + &i.Lines, + &i.Chars, + &i.Note, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocFigureLabels = `-- name: ListDocFigureLabels :many +SELECT document_id, page, figure_idx, idx, text, created_at +FROM doc_figure_labels +WHERE document_id = ? +ORDER BY page, figure_idx, idx +` + +// ORDER BY idx is the label's own reading order, down then across, and it is the +// order the description is read in. Both list queries carry it. +func (q *Queries) ListDocFigureLabels(ctx context.Context, documentID string) ([]DocFigureLabel, error) { + rows, err := q.db.QueryContext(ctx, listDocFigureLabels, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocFigureLabel{} + for rows.Next() { + var i DocFigureLabel + if err := rows.Scan( + &i.DocumentID, + &i.Page, + &i.FigureIdx, + &i.Idx, + &i.Text, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocFigureLabelsForPage = `-- name: ListDocFigureLabelsForPage :many +SELECT document_id, page, figure_idx, idx, text, created_at +FROM doc_figure_labels +WHERE document_id = ? AND page = ? +ORDER BY figure_idx, idx +` + +type ListDocFigureLabelsForPageParams struct { + DocumentID string + Page int64 +} + +func (q *Queries) ListDocFigureLabelsForPage(ctx context.Context, arg ListDocFigureLabelsForPageParams) ([]DocFigureLabel, error) { + rows, err := q.db.QueryContext(ctx, listDocFigureLabelsForPage, arg.DocumentID, arg.Page) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocFigureLabel{} + for rows.Next() { + var i DocFigureLabel + if err := rows.Scan( + &i.DocumentID, + &i.Page, + &i.FigureIdx, + &i.Idx, + &i.Text, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocFigures = `-- name: ListDocFigures :many +SELECT document_id, page, idx, x0, y0, x1, y1, ink, text_fraction, + dpi, pixel_width, pixel_height, blob_sha256, created_at +FROM doc_figures +WHERE document_id = ? +ORDER BY page, idx +` + +func (q *Queries) ListDocFigures(ctx context.Context, documentID string) ([]DocFigure, error) { + rows, err := q.db.QueryContext(ctx, listDocFigures, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocFigure{} + for rows.Next() { + var i DocFigure + if err := rows.Scan( + &i.DocumentID, + &i.Page, + &i.Idx, + &i.X0, + &i.Y0, + &i.X1, + &i.Y1, + &i.Ink, + &i.TextFraction, + &i.Dpi, + &i.PixelWidth, + &i.PixelHeight, + &i.BlobSha256, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocFiguresForPage = `-- name: ListDocFiguresForPage :many +SELECT document_id, page, idx, x0, y0, x1, y1, ink, text_fraction, + dpi, pixel_width, pixel_height, blob_sha256, created_at +FROM doc_figures +WHERE document_id = ? AND page = ? +ORDER BY idx +` + +type ListDocFiguresForPageParams struct { + DocumentID string + Page int64 +} + +func (q *Queries) ListDocFiguresForPage(ctx context.Context, arg ListDocFiguresForPageParams) ([]DocFigure, error) { + rows, err := q.db.QueryContext(ctx, listDocFiguresForPage, arg.DocumentID, arg.Page) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocFigure{} + for rows.Next() { + var i DocFigure + if err := rows.Scan( + &i.DocumentID, + &i.Page, + &i.Idx, + &i.X0, + &i.Y0, + &i.X1, + &i.Y1, + &i.Ink, + &i.TextFraction, + &i.Dpi, + &i.PixelWidth, + &i.PixelHeight, + &i.BlobSha256, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const summarizeDocBlocks = `-- name: SummarizeDocBlocks :many +SELECT lang, + CAST(count(*) AS INTEGER) AS blocks, + CAST(sum(chars) AS INTEGER) AS chars, + CAST(sum(lines) AS INTEGER) AS lines, + CAST(count(DISTINCT page) AS INTEGER) AS pages, + CAST(min(page) AS INTEGER) AS first_page +FROM doc_blocks +WHERE document_id = ? +GROUP BY lang +ORDER BY first_page, lang +` + +type SummarizeDocBlocksRow struct { + Lang string + Blocks int64 + Chars int64 + Lines int64 + Pages int64 + FirstPage int64 +} + +// What a conversion cost and covered, for the pipeline to report without reading +// every block back. Every aggregate is wrapped in CAST(... AS INTEGER): without +// it sqlc cannot infer an aggregate's type in SQLite and emits interface{}, +// pushing a type assertion onto every caller. +func (q *Queries) SummarizeDocBlocks(ctx context.Context, documentID string) ([]SummarizeDocBlocksRow, error) { + rows, err := q.db.QueryContext(ctx, summarizeDocBlocks, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SummarizeDocBlocksRow{} + for rows.Next() { + var i SummarizeDocBlocksRow + if err := rows.Scan( + &i.Lang, + &i.Blocks, + &i.Chars, + &i.Lines, + &i.Pages, + &i.FirstPage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const upsertDocBlock = `-- name: UpsertDocBlock :exec + +INSERT INTO doc_blocks (document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page, region_x0, idx) DO UPDATE SET + kind = excluded.kind, + level = excluded.level, + text = excluded.text, + lang = excluded.lang, + x0 = excluded.x0, + x1 = excluded.x1, + y0 = excluded.y0, + y1 = excluded.y1, + lines = excluded.lines, + chars = excluded.chars, + note = excluded.note, + created_at = excluded.created_at +` + +type UpsertDocBlockParams struct { + DocumentID string + Page int64 + RegionX0 int64 + Idx int64 + Kind string + Level int64 + Text string + Lang string + X0 float64 + X1 float64 + Y0 float64 + Y1 float64 + Lines int64 + Chars int64 + Note string + CreatedAt int64 +} + +// Queries over doc_blocks and doc_figures: what a conversion produced. See +// 00005_doc_blocks.sql for the schema's reasoning and docs/design/conversion.md +// for the contract. +// +// THIS FILE MUST STAY PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 +// (pinned in tools/go.mod) mixes up character and byte offsets when it cuts +// statements out of a file, so one non-ASCII character anywhere above corrupts +// every statement after it -- silently, in the dangerous case: `make sqlc` exits +// 0, the Go compiles, the linter passes, and the statement fails at PREPARE time +// inside a background job against a user's database. The full measurement is in +// the header of docregions.sql; TestQueryFilesAreASCII is the cause-side guard +// and TestDocBlockQueriesExecute the symptom-side one. +// +// Columns are listed explicitly rather than with SELECT *, so that adding a +// column later cannot silently change every caller's row shape. +// Upsert on the natural key (document_id, page, region_x0, idx), because a +// conversion job may run twice and must converge on the same rows rather than +// duplicating them. +// +// Every non-key column is updated, kind and lang included. Nothing about a +// block's classification is in the key, so a paragraph that a better heading rule +// promotes to a heading is the same block updated in place -- see the note above +// the primary key in 00005_doc_blocks.sql. +// +// This is belt and braces beside the delete SaveConversion does first, and it +// cannot be the whole story: a re-conversion that produces FEWER blocks in a +// region would otherwise leave the tail of the previous run behind at higher +// indices, where it reads as content. +func (q *Queries) UpsertDocBlock(ctx context.Context, arg UpsertDocBlockParams) error { + _, err := q.db.ExecContext(ctx, upsertDocBlock, + arg.DocumentID, + arg.Page, + arg.RegionX0, + arg.Idx, + arg.Kind, + arg.Level, + arg.Text, + arg.Lang, + arg.X0, + arg.X1, + arg.Y0, + arg.Y1, + arg.Lines, + arg.Chars, + arg.Note, + arg.CreatedAt, + ) + return err +} + +const upsertDocFigure = `-- name: UpsertDocFigure :exec +INSERT INTO doc_figures (document_id, page, idx, x0, y0, x1, y1, ink, text_fraction, + dpi, pixel_width, pixel_height, blob_sha256, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page, idx) DO UPDATE SET + x0 = excluded.x0, + y0 = excluded.y0, + x1 = excluded.x1, + y1 = excluded.y1, + ink = excluded.ink, + text_fraction = excluded.text_fraction, + dpi = excluded.dpi, + pixel_width = excluded.pixel_width, + pixel_height = excluded.pixel_height, + blob_sha256 = excluded.blob_sha256, + created_at = excluded.created_at +` + +type UpsertDocFigureParams struct { + DocumentID string + Page int64 + Idx int64 + X0 float64 + Y0 float64 + X1 float64 + Y1 float64 + Ink int64 + TextFraction float64 + Dpi int64 + PixelWidth int64 + PixelHeight int64 + BlobSha256 string + CreatedAt int64 +} + +// Upsert on (document_id, page, idx). A figure has no region and no language in +// its key, because conversion.md settles that a picture belonging to no language +// belongs to every language. +func (q *Queries) UpsertDocFigure(ctx context.Context, arg UpsertDocFigureParams) error { + _, err := q.db.ExecContext(ctx, upsertDocFigure, + arg.DocumentID, + arg.Page, + arg.Idx, + arg.X0, + arg.Y0, + arg.X1, + arg.Y1, + arg.Ink, + arg.TextFraction, + arg.Dpi, + arg.PixelWidth, + arg.PixelHeight, + arg.BlobSha256, + arg.CreatedAt, + ) + return err +} + +const upsertDocFigureLabel = `-- name: UpsertDocFigureLabel :exec +INSERT INTO doc_figure_labels (document_id, page, figure_idx, idx, text, created_at) +VALUES (?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page, figure_idx, idx) DO UPDATE SET + text = excluded.text, + created_at = excluded.created_at +` + +type UpsertDocFigureLabelParams struct { + DocumentID string + Page int64 + FigureIdx int64 + Idx int64 + Text string + CreatedAt int64 +} + +// A figure's callout labels: the text a leader points at. No geometry -- 00009 +// removed it, because the crop is a band that already prints its own labels and the +// stored text is now the picture's accessible description rather than something a +// reader re-lays out. Kept out of doc_blocks on purpose -- see 00008's header for why +// a sixth block kind was the wrong shape and what it would have cost. +// Upsert on (document_id, page, figure_idx, idx), so a re-conversion converges. +func (q *Queries) UpsertDocFigureLabel(ctx context.Context, arg UpsertDocFigureLabelParams) error { + _, err := q.db.ExecContext(ctx, upsertDocFigureLabel, + arg.DocumentID, + arg.Page, + arg.FigureIdx, + arg.Idx, + arg.Text, + arg.CreatedAt, + ) + return err +} diff --git a/internal/db/gen/doclangs.sql.go b/internal/db/gen/doclangs.sql.go new file mode 100644 index 0000000..da16c4e --- /dev/null +++ b/internal/db/gen/doclangs.sql.go @@ -0,0 +1,250 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: doclangs.sql + +package gen + +import ( + "context" +) + +const countDocLangConflicts = `-- name: CountDocLangConflicts :one +SELECT CAST(count(*) AS INTEGER) AS total +FROM doc_langs +WHERE document_id = ? AND source = ? AND conflict = 1 +` + +type CountDocLangConflictsParams struct { + DocumentID string + Source string +} + +func (q *Queries) CountDocLangConflicts(ctx context.Context, arg CountDocLangConflictsParams) (int64, error) { + row := q.db.QueryRowContext(ctx, countDocLangConflicts, arg.DocumentID, arg.Source) + var total int64 + err := row.Scan(&total) + return total, err +} + +const deleteDocLangs = `-- name: DeleteDocLangs :exec +DELETE FROM doc_langs WHERE document_id = ? +` + +func (q *Queries) DeleteDocLangs(ctx context.Context, documentID string) error { + _, err := q.db.ExecContext(ctx, deleteDocLangs, documentID) + return err +} + +const deleteDocLangsBySource = `-- name: DeleteDocLangsBySource :exec +DELETE FROM doc_langs WHERE document_id = ? AND source = ? +` + +type DeleteDocLangsBySourceParams struct { + DocumentID string + Source string +} + +// Replacing one signal's view wholesale is how a re-probe stays honest: a run +// that no longer exists must disappear rather than linger from the previous +// attempt. Scoped to one source so the other signals' rows survive. +func (q *Queries) DeleteDocLangsBySource(ctx context.Context, arg DeleteDocLangsBySourceParams) error { + _, err := q.db.ExecContext(ctx, deleteDocLangsBySource, arg.DocumentID, arg.Source) + return err +} + +const listDocLangs = `-- name: ListDocLangs :many +SELECT document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, confidence, "conflict", note, created_at FROM doc_langs WHERE document_id = ? ORDER BY source, pdf_start +` + +func (q *Queries) ListDocLangs(ctx context.Context, documentID string) ([]DocLang, error) { + rows, err := q.db.QueryContext(ctx, listDocLangs, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocLang{} + for rows.Next() { + var i DocLang + if err := rows.Scan( + &i.DocumentID, + &i.Source, + &i.PdfStart, + &i.PdfEnd, + &i.Code, + &i.Lang, + &i.Title, + &i.PrintedPage, + &i.Confidence, + &i.Conflict, + &i.Note, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocLangsBySource = `-- name: ListDocLangsBySource :many +SELECT document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, confidence, "conflict", note, created_at FROM doc_langs WHERE document_id = ? AND source = ? ORDER BY pdf_start +` + +type ListDocLangsBySourceParams struct { + DocumentID string + Source string +} + +func (q *Queries) ListDocLangsBySource(ctx context.Context, arg ListDocLangsBySourceParams) ([]DocLang, error) { + rows, err := q.db.QueryContext(ctx, listDocLangsBySource, arg.DocumentID, arg.Source) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocLang{} + for rows.Next() { + var i DocLang + if err := rows.Scan( + &i.DocumentID, + &i.Source, + &i.PdfStart, + &i.PdfEnd, + &i.Code, + &i.Lang, + &i.Title, + &i.PrintedPage, + &i.Confidence, + &i.Conflict, + &i.Note, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const summarizeDocLangs = `-- name: SummarizeDocLangs :many +SELECT code, + lang, + CAST(sum(CASE WHEN pdf_start = 0 THEN 0 ELSE pdf_end - pdf_start + 1 END) + AS INTEGER) AS pages, + CAST(count(*) AS INTEGER) AS runs, + CAST(max(conflict) AS INTEGER) AS disputed, + CAST(min(pdf_start) AS INTEGER) AS first_page +FROM doc_langs +WHERE document_id = ? AND source = ? +GROUP BY code, lang +ORDER BY first_page +` + +type SummarizeDocLangsParams struct { + DocumentID string + Source string +} + +type SummarizeDocLangsRow struct { + Code string + Lang string + Pages int64 + Runs int64 + Disputed int64 + FirstPage int64 +} + +// The language map as shown to the user: one row per language in the reconciled +// view, with its page total and whether any of its runs are disputed. +// +// A run with pdf_start = 0 named a language it could not place, so it covers no +// pages at all. Counting its span reported a language the printed index merely +// mentioned as a one-page section. +func (q *Queries) SummarizeDocLangs(ctx context.Context, arg SummarizeDocLangsParams) ([]SummarizeDocLangsRow, error) { + rows, err := q.db.QueryContext(ctx, summarizeDocLangs, arg.DocumentID, arg.Source) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SummarizeDocLangsRow{} + for rows.Next() { + var i SummarizeDocLangsRow + if err := rows.Scan( + &i.Code, + &i.Lang, + &i.Pages, + &i.Runs, + &i.Disputed, + &i.FirstPage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const upsertDocLang = `-- name: UpsertDocLang :exec +INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, lang, title, + printed_page, confidence, conflict, note, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, source, code, pdf_start) DO UPDATE SET + pdf_end = excluded.pdf_end, + lang = excluded.lang, + title = excluded.title, + printed_page = excluded.printed_page, + confidence = excluded.confidence, + conflict = excluded.conflict, + note = excluded.note +` + +type UpsertDocLangParams struct { + DocumentID string + Source string + PdfStart int64 + PdfEnd int64 + Code string + Lang string + Title string + PrintedPage *int64 + Confidence float64 + Conflict int64 + Note string + CreatedAt int64 +} + +func (q *Queries) UpsertDocLang(ctx context.Context, arg UpsertDocLangParams) error { + _, err := q.db.ExecContext(ctx, upsertDocLang, + arg.DocumentID, + arg.Source, + arg.PdfStart, + arg.PdfEnd, + arg.Code, + arg.Lang, + arg.Title, + arg.PrintedPage, + arg.Confidence, + arg.Conflict, + arg.Note, + arg.CreatedAt, + ) + return err +} diff --git a/internal/db/gen/docpages.sql.go b/internal/db/gen/docpages.sql.go new file mode 100644 index 0000000..8eaa68c --- /dev/null +++ b/internal/db/gen/docpages.sql.go @@ -0,0 +1,225 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: docpages.sql + +package gen + +import ( + "context" +) + +const countDocPagesByLang = `-- name: CountDocPagesByLang :many +SELECT lang, CAST(count(*) AS INTEGER) AS pages +FROM doc_pages +WHERE document_id = ? AND lang <> '' +GROUP BY lang +ORDER BY pages DESC, lang +` + +type CountDocPagesByLangRow struct { + Lang string + Pages int64 +} + +// How many pages the document holds in each resolved language. The CAST is +// required: without it sqlc infers interface{} for the aggregate. +func (q *Queries) CountDocPagesByLang(ctx context.Context, documentID string) ([]CountDocPagesByLangRow, error) { + rows, err := q.db.QueryContext(ctx, countDocPagesByLang, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []CountDocPagesByLangRow{} + for rows.Next() { + var i CountDocPagesByLangRow + if err := rows.Scan(&i.Lang, &i.Pages); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const countDocPagesWithText = `-- name: CountDocPagesWithText :one +SELECT CAST(count(*) AS INTEGER) AS pages +FROM doc_pages +WHERE document_id = ? AND chars > 0 +` + +func (q *Queries) CountDocPagesWithText(ctx context.Context, documentID string) (int64, error) { + row := q.db.QueryRowContext(ctx, countDocPagesWithText, documentID) + var pages int64 + err := row.Scan(&pages) + return pages, err +} + +const deleteDocPages = `-- name: DeleteDocPages :exec +DELETE FROM doc_pages WHERE document_id = ? +` + +func (q *Queries) DeleteDocPages(ctx context.Context, documentID string) error { + _, err := q.db.ExecContext(ctx, deleteDocPages, documentID) + return err +} + +const docPageFolioOffsets = `-- name: DocPageFolioOffsets :many +SELECT CAST(page_no - printed_folio AS INTEGER) AS folio_offset, + CAST(count(*) AS INTEGER) AS pages +FROM doc_pages +WHERE document_id = ? AND printed_folio IS NOT NULL +GROUP BY folio_offset +ORDER BY pages DESC, folio_offset +` + +type DocPageFolioOffsetsRow struct { + FolioOffset int64 + Pages int64 +} + +// How far each page's PDF number runs ahead of the number printed on the paper, +// as a histogram over the pages that print one at all. +// +// This is derived on read rather than stored, because doc_pages already holds the +// whole answer and a stored copy could only go stale against it: the folio is +// re-read on every probe, so a change to how it is read must move this number in +// the same breath. It is one small grouped scan per document over rows the probe +// already wrote, asked once when a conversion is served, not per block or per page. +// +// The caller decides which row to believe -- see registry.FolioOffset -- so the +// whole histogram comes back rather than just its first row. The CASTs are +// required: without them sqlc infers interface{} for both columns. "offset" is a +// SQL keyword, hence the name. +func (q *Queries) DocPageFolioOffsets(ctx context.Context, documentID string) ([]DocPageFolioOffsetsRow, error) { + rows, err := q.db.QueryContext(ctx, docPageFolioOffsets, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocPageFolioOffsetsRow{} + for rows.Next() { + var i DocPageFolioOffsetsRow + if err := rows.Scan(&i.FolioOffset, &i.Pages); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getDocPage = `-- name: GetDocPage :one +SELECT document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source, figures FROM doc_pages WHERE document_id = ? AND page_no = ? +` + +type GetDocPageParams struct { + DocumentID string + PageNo int64 +} + +func (q *Queries) GetDocPage(ctx context.Context, arg GetDocPageParams) (DocPage, error) { + row := q.db.QueryRowContext(ctx, getDocPage, arg.DocumentID, arg.PageNo) + var i DocPage + err := row.Scan( + &i.DocumentID, + &i.PageNo, + &i.Chars, + &i.Script, + &i.PageTag, + &i.PrintedFolio, + &i.Lang, + &i.LangSource, + &i.Figures, + ) + return i, err +} + +const listDocPages = `-- name: ListDocPages :many +SELECT document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source, figures FROM doc_pages WHERE document_id = ? ORDER BY page_no +` + +func (q *Queries) ListDocPages(ctx context.Context, documentID string) ([]DocPage, error) { + rows, err := q.db.QueryContext(ctx, listDocPages, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocPage{} + for rows.Next() { + var i DocPage + if err := rows.Scan( + &i.DocumentID, + &i.PageNo, + &i.Chars, + &i.Script, + &i.PageTag, + &i.PrintedFolio, + &i.Lang, + &i.LangSource, + &i.Figures, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const upsertDocPage = `-- name: UpsertDocPage :exec +INSERT INTO doc_pages (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source, figures) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page_no) DO UPDATE SET + chars = excluded.chars, + script = excluded.script, + page_tag = excluded.page_tag, + printed_folio = excluded.printed_folio, + lang = excluded.lang, + lang_source = excluded.lang_source, + figures = excluded.figures +` + +type UpsertDocPageParams struct { + DocumentID string + PageNo int64 + Chars int64 + Script string + PageTag string + PrintedFolio *int64 + Lang string + LangSource string + Figures *int64 +} + +// Upsert on the natural key, because a probe job may run twice and must converge +// on the same rows rather than duplicating them. +func (q *Queries) UpsertDocPage(ctx context.Context, arg UpsertDocPageParams) error { + _, err := q.db.ExecContext(ctx, upsertDocPage, + arg.DocumentID, + arg.PageNo, + arg.Chars, + arg.Script, + arg.PageTag, + arg.PrintedFolio, + arg.Lang, + arg.LangSource, + arg.Figures, + ) + return err +} diff --git a/internal/db/gen/docregions.sql.go b/internal/db/gen/docregions.sql.go new file mode 100644 index 0000000..a255e03 --- /dev/null +++ b/internal/db/gen/docregions.sql.go @@ -0,0 +1,276 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: docregions.sql + +package gen + +import ( + "context" +) + +const deleteDocRegions = `-- name: DeleteDocRegions :exec +DELETE FROM doc_regions WHERE document_id = ? +` + +// Replacing a document's regions wholesale is how a re-probe stays honest, and it +// is required rather than merely tidy: a region whose attribution changed is a new +// row under this key, so without the delete the superseded one lingers and the +// page reports itself twice. +func (q *Queries) DeleteDocRegions(ctx context.Context, documentID string) error { + _, err := q.db.ExecContext(ctx, deleteDocRegions, documentID) + return err +} + +const listDocRegions = `-- name: ListDocRegions :many +SELECT document_id, source, page, x0, x1, code, lang, chars, runs, conflict, note, created_at +FROM doc_regions +WHERE document_id = ? +ORDER BY page, x0 +` + +// Reading order: down the page, then left to right across it. A whole-page region +// sorts first on its page because it begins at x0 = 0. +func (q *Queries) ListDocRegions(ctx context.Context, documentID string) ([]DocRegion, error) { + rows, err := q.db.QueryContext(ctx, listDocRegions, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocRegion{} + for rows.Next() { + var i DocRegion + if err := rows.Scan( + &i.DocumentID, + &i.Source, + &i.Page, + &i.X0, + &i.X1, + &i.Code, + &i.Lang, + &i.Chars, + &i.Runs, + &i.Conflict, + &i.Note, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocRegionsForPage = `-- name: ListDocRegionsForPage :many +SELECT document_id, source, page, x0, x1, code, lang, chars, runs, conflict, note, created_at +FROM doc_regions +WHERE document_id = ? AND page = ? +ORDER BY x0 +` + +type ListDocRegionsForPageParams struct { + DocumentID string + Page int64 +} + +func (q *Queries) ListDocRegionsForPage(ctx context.Context, arg ListDocRegionsForPageParams) ([]DocRegion, error) { + rows, err := q.db.QueryContext(ctx, listDocRegionsForPage, arg.DocumentID, arg.Page) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DocRegion{} + for rows.Next() { + var i DocRegion + if err := rows.Scan( + &i.DocumentID, + &i.Source, + &i.Page, + &i.X0, + &i.X1, + &i.Code, + &i.Lang, + &i.Chars, + &i.Runs, + &i.Conflict, + &i.Note, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const summarizeDocRegions = `-- name: SummarizeDocRegions :many +SELECT code, + lang, + CAST(sum(chars) AS INTEGER) AS chars, + CAST(sum(runs) AS INTEGER) AS runs, + CAST(count(DISTINCT page) AS INTEGER) AS pages, + CAST(min(page) AS INTEGER) AS first_page, + CAST(max(conflict) AS INTEGER) AS disputed +FROM doc_regions +WHERE document_id = ? +GROUP BY code, lang +ORDER BY first_page, code +` + +type SummarizeDocRegionsRow struct { + Code string + Lang string + Chars int64 + Runs int64 + Pages int64 + FirstPage int64 + Disputed int64 +} + +// The region map as shown to the user: one row per language label, with the +// characters and runs it holds, how many pages it appears on, and whether any of +// its regions are disputed. +// +// Characters rather than pages is the point, because a page holding three +// languages is not a unit of size; pages are still what a reader is shown, so both +// are reported. Every aggregate is wrapped in CAST(... AS INTEGER): without it +// sqlc cannot infer the type and emits interface{}, pushing a type assertion onto +// every caller. +func (q *Queries) SummarizeDocRegions(ctx context.Context, documentID string) ([]SummarizeDocRegionsRow, error) { + rows, err := q.db.QueryContext(ctx, summarizeDocRegions, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SummarizeDocRegionsRow{} + for rows.Next() { + var i SummarizeDocRegionsRow + if err := rows.Scan( + &i.Code, + &i.Lang, + &i.Chars, + &i.Runs, + &i.Pages, + &i.FirstPage, + &i.Disputed, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const upsertDocRegion = `-- name: UpsertDocRegion :exec + +INSERT INTO doc_regions (document_id, source, page, x0, x1, code, lang, chars, runs, + conflict, note, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, source, page, x0) DO UPDATE SET + x1 = excluded.x1, + code = excluded.code, + lang = excluded.lang, + chars = excluded.chars, + runs = excluded.runs, + conflict = excluded.conflict, + note = excluded.note, + created_at = excluded.created_at +` + +type UpsertDocRegionParams struct { + DocumentID string + Source string + Page int64 + X0 int64 + X1 int64 + Code string + Lang string + Chars int64 + Runs int64 + Conflict int64 + Note string + CreatedAt int64 +} + +// Queries over doc_regions: one language's territory on a page. See +// 00004_doc_regions.sql for the schema's reasoning and docs/design/regions.md for +// the contract. +// +// TWO RULES FOR THIS FILE, BOTH LEARNED THE HARD WAY WHILE WRITING IT. +// +// 1. KEEP THIS FILE PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 +// (pinned in tools/go.mod) mixes up character and byte offsets when it cuts +// statements out of a file, so a single non-ASCII character anywhere earlier +// corrupts every statement after it. What is measured is the rule, not the +// internals: the damage equals the extra bytes those characters occupy, one +// character of SQL lost per extra byte. +// +// Two shapes were observed, and the quiet one is the dangerous one. With +// em-dashes in a comment above, "ORDER BY first_page, code" generated as +// "ORDER BY first_page, co" for one and "ORDER BY first_pa" for four -- clean +// Go, broken SQL. With em-dashes placed differently, sqlc instead garbled a +// statement badly enough to fail its own parser, printing tokens like +// "SELdocument_id" and exiting noisily. Which of the two you get depends on +// where the character sits, so neither a clean run nor a loud failure tells +// you the file is safe. Only ASCII does. +// +// The direction of sqlc's own mismatch is deliberately not asserted here. It +// was not read out of sqlc's source, and the two published guesses point +// opposite ways -- byte offsets applied to characters would overshoot a +// statement's end rather than cut it short, which is not what happens. The +// rule above is what was measured and is what protects this file. +// +// This is the worst failure shape available: `make sqlc` exits 0, the generated +// Go compiles, the linter is happy, and the statement fails at PREPARE time +// inside a background job against a user's database. All ten pre-existing query +// files happen to be pure ASCII, which is the only reason this had not bitten +// anyone yet. That was checked rather than assumed: 0 non-ASCII bytes across +// every one of them. TestDocRegionQueriesExecute in internal/db is the guard; +// it runs every statement below against a real migrated database, so a mangled +// one cannot reach a user. +// +// 2. Columns are listed explicitly rather than with SELECT *, so that adding a +// column to doc_regions later cannot silently change every caller's row shape. +// +// Upsert on the natural key (document_id, source, page, x0), because a probe job +// may run twice and must converge on the same rows rather than duplicating them. +// +// This is belt and braces beside the delete that SaveProbe does first, and it +// cannot be the whole story: source is part of the key and a region's source can +// change between probes, so an upsert alone would leave the superseded row behind +// at the same x0. The note at the foot of 00004_doc_regions.sql explains why. +func (q *Queries) UpsertDocRegion(ctx context.Context, arg UpsertDocRegionParams) error { + _, err := q.db.ExecContext(ctx, upsertDocRegion, + arg.DocumentID, + arg.Source, + arg.Page, + arg.X0, + arg.X1, + arg.Code, + arg.Lang, + arg.Chars, + arg.Runs, + arg.Conflict, + arg.Note, + arg.CreatedAt, + ) + return err +} diff --git a/internal/db/gen/documents.sql.go b/internal/db/gen/documents.sql.go new file mode 100644 index 0000000..156d69e --- /dev/null +++ b/internal/db/gen/documents.sql.go @@ -0,0 +1,349 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: documents.sql + +package gen + +import ( + "context" +) + +const approveDocumentScope = `-- name: ApproveDocumentScope :exec +UPDATE documents +SET include_neutral_pages = ?, + state = ?, + last_error = '', + updated_at = ? +WHERE id = ? +` + +type ApproveDocumentScopeParams struct { + IncludeNeutralPages int64 + State string + UpdatedAt int64 + ID string +} + +// Records the scope the user approved at the gate, together with the state that +// says the work is authorised. One statement rather than two, for +// RecordDocumentProbe's reason: a crash between them would leave a document that +// is converting under a scope nobody chose. +// +// include_neutral_pages is the whole of the extra scope, and it is a flag and not a +// page list on purpose -- see 00007's header. The server recomputes the set of pages +// from the stored region map, so a stale client cannot name a page the gate never +// offered it. +func (q *Queries) ApproveDocumentScope(ctx context.Context, arg ApproveDocumentScopeParams) error { + _, err := q.db.ExecContext(ctx, approveDocumentScope, + arg.IncludeNeutralPages, + arg.State, + arg.UpdatedAt, + arg.ID, + ) + return err +} + +const countDocuments = `-- name: CountDocuments :one +SELECT CAST(count(*) AS INTEGER) AS total FROM documents +` + +func (q *Queries) CountDocuments(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countDocuments) + var total int64 + err := row.Scan(&total) + return total, err +} + +const countDocumentsForBlob = `-- name: CountDocumentsForBlob :one +SELECT CAST(count(*) AS INTEGER) AS total FROM documents WHERE blob_sha256 = ? +` + +// Used to decide whether a blob is still referenced before deleting it, since +// two devices can legitimately share one uploaded file. +func (q *Queries) CountDocumentsForBlob(ctx context.Context, blobSha256 string) (int64, error) { + row := q.db.QueryRowContext(ctx, countDocumentsForBlob, blobSha256) + var total int64 + err := row.Scan(&total) + return total, err +} + +const createDocument = `-- name: CreateDocument :execrows +INSERT INTO documents (id, device_id, blob_sha256, filename, media_type, kind, state, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(device_id, blob_sha256) DO NOTHING +` + +type CreateDocumentParams struct { + ID string + DeviceID string + BlobSha256 string + Filename string + MediaType string + Kind string + State string + CreatedAt int64 + UpdatedAt int64 +} + +// Uploading the same bytes against the same device twice is the same document, +// enforced by documents_device_blob_idx. DO NOTHING plus a follow-up lookup makes +// the upload handler idempotent without the caller having to check first. +func (q *Queries) CreateDocument(ctx context.Context, arg CreateDocumentParams) (int64, error) { + result, err := q.db.ExecContext(ctx, createDocument, + arg.ID, + arg.DeviceID, + arg.BlobSha256, + arg.Filename, + arg.MediaType, + arg.Kind, + arg.State, + arg.CreatedAt, + arg.UpdatedAt, + ) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const deleteDocument = `-- name: DeleteDocument :exec +DELETE FROM documents WHERE id = ? +` + +func (q *Queries) DeleteDocument(ctx context.Context, id string) error { + _, err := q.db.ExecContext(ctx, deleteDocument, id) + return err +} + +const getDocument = `-- name: GetDocument :one +SELECT id, device_id, blob_sha256, filename, media_type, kind, state, last_error, page_count, encrypted, tagged, has_text_layer, median_chars_per_page, content_start_page, content_end_page, created_at, updated_at, probed_at, include_neutral_pages FROM documents WHERE id = ? +` + +func (q *Queries) GetDocument(ctx context.Context, id string) (Document, error) { + row := q.db.QueryRowContext(ctx, getDocument, id) + var i Document + err := row.Scan( + &i.ID, + &i.DeviceID, + &i.BlobSha256, + &i.Filename, + &i.MediaType, + &i.Kind, + &i.State, + &i.LastError, + &i.PageCount, + &i.Encrypted, + &i.Tagged, + &i.HasTextLayer, + &i.MedianCharsPerPage, + &i.ContentStartPage, + &i.ContentEndPage, + &i.CreatedAt, + &i.UpdatedAt, + &i.ProbedAt, + &i.IncludeNeutralPages, + ) + return i, err +} + +const getDocumentByDeviceAndBlob = `-- name: GetDocumentByDeviceAndBlob :one +SELECT id, device_id, blob_sha256, filename, media_type, kind, state, last_error, page_count, encrypted, tagged, has_text_layer, median_chars_per_page, content_start_page, content_end_page, created_at, updated_at, probed_at, include_neutral_pages FROM documents WHERE device_id = ? AND blob_sha256 = ? +` + +type GetDocumentByDeviceAndBlobParams struct { + DeviceID string + BlobSha256 string +} + +func (q *Queries) GetDocumentByDeviceAndBlob(ctx context.Context, arg GetDocumentByDeviceAndBlobParams) (Document, error) { + row := q.db.QueryRowContext(ctx, getDocumentByDeviceAndBlob, arg.DeviceID, arg.BlobSha256) + var i Document + err := row.Scan( + &i.ID, + &i.DeviceID, + &i.BlobSha256, + &i.Filename, + &i.MediaType, + &i.Kind, + &i.State, + &i.LastError, + &i.PageCount, + &i.Encrypted, + &i.Tagged, + &i.HasTextLayer, + &i.MedianCharsPerPage, + &i.ContentStartPage, + &i.ContentEndPage, + &i.CreatedAt, + &i.UpdatedAt, + &i.ProbedAt, + &i.IncludeNeutralPages, + ) + return i, err +} + +const listDocumentsByState = `-- name: ListDocumentsByState :many +SELECT id, device_id, blob_sha256, filename, media_type, kind, state, last_error, page_count, encrypted, tagged, has_text_layer, median_chars_per_page, content_start_page, content_end_page, created_at, updated_at, probed_at, include_neutral_pages FROM documents WHERE state = ? ORDER BY created_at DESC +` + +func (q *Queries) ListDocumentsByState(ctx context.Context, state string) ([]Document, error) { + rows, err := q.db.QueryContext(ctx, listDocumentsByState, state) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Document{} + for rows.Next() { + var i Document + if err := rows.Scan( + &i.ID, + &i.DeviceID, + &i.BlobSha256, + &i.Filename, + &i.MediaType, + &i.Kind, + &i.State, + &i.LastError, + &i.PageCount, + &i.Encrypted, + &i.Tagged, + &i.HasTextLayer, + &i.MedianCharsPerPage, + &i.ContentStartPage, + &i.ContentEndPage, + &i.CreatedAt, + &i.UpdatedAt, + &i.ProbedAt, + &i.IncludeNeutralPages, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocumentsForDevice = `-- name: ListDocumentsForDevice :many +SELECT id, device_id, blob_sha256, filename, media_type, kind, state, last_error, page_count, encrypted, tagged, has_text_layer, median_chars_per_page, content_start_page, content_end_page, created_at, updated_at, probed_at, include_neutral_pages FROM documents WHERE device_id = ? ORDER BY created_at DESC +` + +func (q *Queries) ListDocumentsForDevice(ctx context.Context, deviceID string) ([]Document, error) { + rows, err := q.db.QueryContext(ctx, listDocumentsForDevice, deviceID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Document{} + for rows.Next() { + var i Document + if err := rows.Scan( + &i.ID, + &i.DeviceID, + &i.BlobSha256, + &i.Filename, + &i.MediaType, + &i.Kind, + &i.State, + &i.LastError, + &i.PageCount, + &i.Encrypted, + &i.Tagged, + &i.HasTextLayer, + &i.MedianCharsPerPage, + &i.ContentStartPage, + &i.ContentEndPage, + &i.CreatedAt, + &i.UpdatedAt, + &i.ProbedAt, + &i.IncludeNeutralPages, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const recordDocumentProbe = `-- name: RecordDocumentProbe :exec +UPDATE documents +SET page_count = ?, + encrypted = ?, + tagged = ?, + has_text_layer = ?, + median_chars_per_page = ?, + content_start_page = ?, + content_end_page = ?, + state = ?, + last_error = '', + probed_at = ?, + updated_at = ? +WHERE id = ? +` + +type RecordDocumentProbeParams struct { + PageCount *int64 + Encrypted *int64 + Tagged *int64 + HasTextLayer *int64 + MedianCharsPerPage *int64 + ContentStartPage *int64 + ContentEndPage *int64 + State string + ProbedAt *int64 + UpdatedAt int64 + ID string +} + +// Records everything stages 0 and 1 discovered, in one statement. Writing the +// probe result and the new state together keeps a crash from leaving a document +// that claims to be probed but has no page count. +func (q *Queries) RecordDocumentProbe(ctx context.Context, arg RecordDocumentProbeParams) error { + _, err := q.db.ExecContext(ctx, recordDocumentProbe, + arg.PageCount, + arg.Encrypted, + arg.Tagged, + arg.HasTextLayer, + arg.MedianCharsPerPage, + arg.ContentStartPage, + arg.ContentEndPage, + arg.State, + arg.ProbedAt, + arg.UpdatedAt, + arg.ID, + ) + return err +} + +const setDocumentState = `-- name: SetDocumentState :exec +UPDATE documents SET state = ?, last_error = ?, updated_at = ? WHERE id = ? +` + +type SetDocumentStateParams struct { + State string + LastError string + UpdatedAt int64 + ID string +} + +func (q *Queries) SetDocumentState(ctx context.Context, arg SetDocumentStateParams) error { + _, err := q.db.ExecContext(ctx, setDocumentState, + arg.State, + arg.LastError, + arg.UpdatedAt, + arg.ID, + ) + return err +} diff --git a/internal/db/gen/locations.sql.go b/internal/db/gen/locations.sql.go new file mode 100644 index 0000000..1af2815 --- /dev/null +++ b/internal/db/gen/locations.sql.go @@ -0,0 +1,153 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: locations.sql + +package gen + +import ( + "context" +) + +const countLocations = `-- name: CountLocations :one +SELECT CAST(count(*) AS INTEGER) AS total FROM locations +` + +func (q *Queries) CountLocations(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countLocations) + var total int64 + err := row.Scan(&total) + return total, err +} + +const createLocation = `-- name: CreateLocation :one +INSERT INTO locations (id, name, parent_id, notes, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?) +RETURNING id, name, parent_id, notes, created_at, updated_at +` + +type CreateLocationParams struct { + ID string + Name string + ParentID *string + Notes string + CreatedAt int64 + UpdatedAt int64 +} + +func (q *Queries) CreateLocation(ctx context.Context, arg CreateLocationParams) (Location, error) { + row := q.db.QueryRowContext(ctx, createLocation, + arg.ID, + arg.Name, + arg.ParentID, + arg.Notes, + arg.CreatedAt, + arg.UpdatedAt, + ) + var i Location + err := row.Scan( + &i.ID, + &i.Name, + &i.ParentID, + &i.Notes, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const deleteLocation = `-- name: DeleteLocation :exec +DELETE FROM locations WHERE id = ? +` + +func (q *Queries) DeleteLocation(ctx context.Context, id string) error { + _, err := q.db.ExecContext(ctx, deleteLocation, id) + return err +} + +const getLocation = `-- name: GetLocation :one +SELECT id, name, parent_id, notes, created_at, updated_at FROM locations WHERE id = ? +` + +func (q *Queries) GetLocation(ctx context.Context, id string) (Location, error) { + row := q.db.QueryRowContext(ctx, getLocation, id) + var i Location + err := row.Scan( + &i.ID, + &i.Name, + &i.ParentID, + &i.Notes, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const listLocations = `-- name: ListLocations :many +SELECT id, name, parent_id, notes, created_at, updated_at FROM locations ORDER BY name +` + +func (q *Queries) ListLocations(ctx context.Context) ([]Location, error) { + rows, err := q.db.QueryContext(ctx, listLocations) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Location{} + for rows.Next() { + var i Location + if err := rows.Scan( + &i.ID, + &i.Name, + &i.ParentID, + &i.Notes, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateLocation = `-- name: UpdateLocation :one +UPDATE locations +SET name = ?, parent_id = ?, notes = ?, updated_at = ? +WHERE id = ? +RETURNING id, name, parent_id, notes, created_at, updated_at +` + +type UpdateLocationParams struct { + Name string + ParentID *string + Notes string + UpdatedAt int64 + ID string +} + +func (q *Queries) UpdateLocation(ctx context.Context, arg UpdateLocationParams) (Location, error) { + row := q.db.QueryRowContext(ctx, updateLocation, + arg.Name, + arg.ParentID, + arg.Notes, + arg.UpdatedAt, + arg.ID, + ) + var i Location + err := row.Scan( + &i.ID, + &i.Name, + &i.ParentID, + &i.Notes, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/db/gen/models.go b/internal/db/gen/models.go index 150371c..a42009d 100644 --- a/internal/db/gen/models.go +++ b/internal/db/gen/models.go @@ -11,6 +11,132 @@ type Blob struct { CreatedAt int64 } +type Device struct { + ID string + Name string + Brand string + Model string + Category string + LocationID *string + Notes string + PurchasedAt *int64 + CreatedAt int64 + UpdatedAt int64 +} + +type DocBlock struct { + DocumentID string + Page int64 + RegionX0 int64 + Idx int64 + Kind string + Level int64 + Text string + Lang string + X0 float64 + X1 float64 + Y0 float64 + Y1 float64 + Lines int64 + Chars int64 + Note string + CreatedAt int64 +} + +type DocBlocksFt struct { + Text string +} + +type DocFigure struct { + DocumentID string + Page int64 + Idx int64 + X0 float64 + Y0 float64 + X1 float64 + Y1 float64 + Ink int64 + TextFraction float64 + Dpi int64 + PixelWidth int64 + PixelHeight int64 + BlobSha256 string + CreatedAt int64 +} + +type DocFigureLabel struct { + DocumentID string + Page int64 + FigureIdx int64 + Idx int64 + Text string + CreatedAt int64 +} + +type DocLang struct { + DocumentID string + Source string + PdfStart int64 + PdfEnd int64 + Code string + Lang string + Title string + PrintedPage *int64 + Confidence float64 + Conflict int64 + Note string + CreatedAt int64 +} + +type DocPage struct { + DocumentID string + PageNo int64 + Chars int64 + Script string + PageTag string + PrintedFolio *int64 + Lang string + LangSource string + Figures *int64 +} + +type DocRegion struct { + DocumentID string + Source string + Page int64 + X0 int64 + X1 int64 + Code string + Lang string + Chars int64 + Runs int64 + Conflict int64 + Note string + CreatedAt int64 +} + +type Document struct { + ID string + DeviceID string + BlobSha256 string + Filename string + MediaType string + Kind string + State string + LastError string + PageCount *int64 + Encrypted *int64 + Tagged *int64 + HasTextLayer *int64 + MedianCharsPerPage *int64 + ContentStartPage *int64 + ContentEndPage *int64 + CreatedAt int64 + UpdatedAt int64 + ProbedAt *int64 + IncludeNeutralPages int64 +} + type Job struct { ID string Kind string @@ -35,6 +161,15 @@ type Job struct { FinishedAt *int64 } +type Location struct { + ID string + Name string + ParentID *string + Notes string + CreatedAt int64 + UpdatedAt int64 +} + type Session struct { ID string UserID string diff --git a/internal/db/gen/querier.go b/internal/db/gen/querier.go index d46d58e..b9448ff 100644 --- a/internal/db/gen/querier.go +++ b/internal/db/gen/querier.go @@ -9,6 +9,16 @@ import ( ) type Querier interface { + // Records the scope the user approved at the gate, together with the state that + // says the work is authorised. One statement rather than two, for + // RecordDocumentProbe's reason: a crash between them would leave a document that + // is converting under a scope nobody chose. + // + // include_neutral_pages is the whole of the extra scope, and it is a flag and not a + // page list on purpose -- see 00007's header. The server recomputes the set of pages + // from the stored region map, so a stale client cannot name a page the gate never + // offered it. + ApproveDocumentScope(ctx context.Context, arg ApproveDocumentScopeParams) error BlobExists(ctx context.Context, sha256 string) (bool, error) CancelJob(ctx context.Context, arg CancelJobParams) (int64, error) // ClaimNextJob atomically takes the highest-priority runnable job. @@ -23,21 +33,82 @@ type Querier interface { // hard still burns an attempt and a poison job cannot be retried forever. ClaimNextJob(ctx context.Context, arg ClaimNextJobParams) (Job, error) CompleteJob(ctx context.Context, arg CompleteJobParams) error + CountDevices(ctx context.Context) (int64, error) + CountDocLangConflicts(ctx context.Context, arg CountDocLangConflictsParams) (int64, error) + // How many pages the document holds in each resolved language. The CAST is + // required: without it sqlc infers interface{} for the aggregate. + CountDocPagesByLang(ctx context.Context, documentID string) ([]CountDocPagesByLangRow, error) + CountDocPagesWithText(ctx context.Context, documentID string) (int64, error) + CountDocuments(ctx context.Context) (int64, error) + // Used to decide whether a blob is still referenced before deleting it, since + // two devices can legitimately share one uploaded file. + CountDocumentsForBlob(ctx context.Context, blobSha256 string) (int64, error) CountJobsByState(ctx context.Context) ([]CountJobsByStateRow, error) + CountLocations(ctx context.Context) (int64, error) + // How many blocks are indexed at all, so a caller can tell "nothing matched" from + // "nothing has been converted yet". Wrapped in CAST(... AS INTEGER) for the reason + // every other aggregate in these files is: without it sqlc cannot infer an + // aggregate's type in SQLite and emits interface{}. + CountSearchableBlocks(ctx context.Context) (int64, error) // CountUsers backs the first-run check: zero users means setup has not happened. CountUsers(ctx context.Context) (int64, error) + CreateDevice(ctx context.Context, arg CreateDeviceParams) (Device, error) + // Uploading the same bytes against the same device twice is the same document, + // enforced by documents_device_blob_idx. DO NOTHING plus a follow-up lookup makes + // the upload handler idempotent without the caller having to check first. + CreateDocument(ctx context.Context, arg CreateDocumentParams) (int64, error) + CreateLocation(ctx context.Context, arg CreateLocationParams) (Location, error) CreateSession(ctx context.Context, arg CreateSessionParams) (Session, error) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) DeleteBlob(ctx context.Context, sha256 string) error + DeleteDevice(ctx context.Context, id string) error + // Replacing a document's blocks wholesale is how a re-conversion stays honest, + // and it is required rather than merely tidy: a region that converted to 12 + // blocks and now converts to 9 would otherwise keep rows at idx 9, 10 and 11, + // which a reader renders as three paragraphs of the previous run's text. + DeleteDocBlocks(ctx context.Context, documentID string) error + // Deleting a document's figures already cascades to its labels, so this exists for + // the one case the cascade does not cover: rewriting the labels of figures that are + // themselves unchanged. Cheaper and clearer than reasoning about which rows the + // figure delete happened to take with it. + DeleteDocFigureLabels(ctx context.Context, documentID string) error + DeleteDocFigures(ctx context.Context, documentID string) error + DeleteDocLangs(ctx context.Context, documentID string) error + // Replacing one signal's view wholesale is how a re-probe stays honest: a run + // that no longer exists must disappear rather than linger from the previous + // attempt. Scoped to one source so the other signals' rows survive. + DeleteDocLangsBySource(ctx context.Context, arg DeleteDocLangsBySourceParams) error + DeleteDocPages(ctx context.Context, documentID string) error + // Replacing a document's regions wholesale is how a re-probe stays honest, and it + // is required rather than merely tidy: a region whose attribution changed is a new + // row under this key, so without the delete the superseded one lingers and the + // page reports itself twice. + DeleteDocRegions(ctx context.Context, documentID string) error + DeleteDocument(ctx context.Context, id string) error DeleteExpiredSessions(ctx context.Context, expiresAt int64) (int64, error) // DeleteFinishedJobsBefore keeps the activity history from growing without bound. DeleteFinishedJobsBefore(ctx context.Context, finishedAt *int64) (int64, error) + DeleteLocation(ctx context.Context, id string) error DeleteSession(ctx context.Context, id string) error DeleteSessionByToken(ctx context.Context, tokenHash []byte) error DeleteSetting(ctx context.Context, key string) error DeleteUser(ctx context.Context, id string) error // DeleteUserSessions logs a user out everywhere, used after a password change. DeleteUserSessions(ctx context.Context, userID string) error + // How far each page's PDF number runs ahead of the number printed on the paper, + // as a histogram over the pages that print one at all. + // + // This is derived on read rather than stored, because doc_pages already holds the + // whole answer and a stored copy could only go stale against it: the folio is + // re-read on every probe, so a change to how it is read must move this number in + // the same breath. It is one small grouped scan per document over rows the probe + // already wrote, asked once when a conversion is served, not per block or per page. + // + // The caller decides which row to believe -- see registry.FolioOffset -- so the + // whole histogram comes back rather than just its first row. The CASTs are + // required: without them sqlc infers interface{} for both columns. "offset" is a + // SQL keyword, hence the name. + DocPageFolioOffsets(ctx context.Context, documentID string) ([]DocPageFolioOffsetsRow, error) EnqueueJob(ctx context.Context, arg EnqueueJobParams) (Job, error) // ExtendLease is the heartbeat for long-running work: translating an eighty-page // manual can outlast any sensible lease, so a live worker renews it rather than @@ -46,7 +117,12 @@ type Querier interface { ExtendSession(ctx context.Context, arg ExtendSessionParams) error FailJob(ctx context.Context, arg FailJobParams) error GetBlob(ctx context.Context, sha256 string) (Blob, error) + GetDevice(ctx context.Context, id string) (Device, error) + GetDocPage(ctx context.Context, arg GetDocPageParams) (DocPage, error) + GetDocument(ctx context.Context, id string) (Document, error) + GetDocumentByDeviceAndBlob(ctx context.Context, arg GetDocumentByDeviceAndBlobParams) (Document, error) GetJob(ctx context.Context, id string) (Job, error) + GetLocation(ctx context.Context, id string) (Location, error) // GetPendingJobByDedupeKey finds the job currently holding a dedupe key, so a // rejected duplicate insert can return the existing job instead of an error. GetPendingJobByDedupeKey(ctx context.Context, dedupeKey *string) (Job, error) @@ -60,11 +136,46 @@ type Querier interface { GetUserByEmail(ctx context.Context, emailFolded string) (User, error) GetUserByID(ctx context.Context, id string) (User, error) ListActiveJobs(ctx context.Context) ([]Job, error) + ListDevices(ctx context.Context) ([]Device, error) + // Filtering by location is a separate query rather than a nullable parameter on + // ListDevices. CONTRIBUTING.md: an "IS NULL OR =" filter defeats sqlc's type + // inference and reads worse than two explicit queries. + ListDevicesByLocation(ctx context.Context, locationID *string) ([]Device, error) + // Reading order across the whole document: down the pages, then left to right + // across each, then in order within a region. A whole-page region sorts first on + // its page because its region_x0 is 0. + ListDocBlocks(ctx context.Context, documentID string) ([]DocBlock, error) + // The funnel's own query: one household's language, and nothing else. A German + // reader of the columns manual gets the German column of each page rather than + // the page, which conversion.md measures as a fifth of the work. + // + // Blocks whose language was never established have lang = '' and are therefore + // NOT returned by any language's query. That is deliberate rather than an + // oversight: passing '' asks for exactly those, which is how the unnamed content + // of a document stays reachable instead of becoming invisible. + ListDocBlocksByLang(ctx context.Context, arg ListDocBlocksByLangParams) ([]DocBlock, error) + ListDocBlocksForPage(ctx context.Context, arg ListDocBlocksForPageParams) ([]DocBlock, error) + // ORDER BY idx is the label's own reading order, down then across, and it is the + // order the description is read in. Both list queries carry it. + ListDocFigureLabels(ctx context.Context, documentID string) ([]DocFigureLabel, error) + ListDocFigureLabelsForPage(ctx context.Context, arg ListDocFigureLabelsForPageParams) ([]DocFigureLabel, error) + ListDocFigures(ctx context.Context, documentID string) ([]DocFigure, error) + ListDocFiguresForPage(ctx context.Context, arg ListDocFiguresForPageParams) ([]DocFigure, error) + ListDocLangs(ctx context.Context, documentID string) ([]DocLang, error) + ListDocLangsBySource(ctx context.Context, arg ListDocLangsBySourceParams) ([]DocLang, error) + ListDocPages(ctx context.Context, documentID string) ([]DocPage, error) + // Reading order: down the page, then left to right across it. A whole-page region + // sorts first on its page because it begins at x0 = 0. + ListDocRegions(ctx context.Context, documentID string) ([]DocRegion, error) + ListDocRegionsForPage(ctx context.Context, arg ListDocRegionsForPageParams) ([]DocRegion, error) + ListDocumentsByState(ctx context.Context, state string) ([]Document, error) + ListDocumentsForDevice(ctx context.Context, deviceID string) ([]Document, error) // Two separate queries rather than one with an optional filter: sqlc cannot infer // the type of a nullable parameter in an "IS NULL OR =" clause and degrades the // parameter to interface{}, pushing a type assertion onto the caller. ListJobs(ctx context.Context, limit int64) ([]Job, error) ListJobsByState(ctx context.Context, arg ListJobsByStateParams) ([]Job, error) + ListLocations(ctx context.Context) ([]Location, error) ListSettings(ctx context.Context) ([]Setting, error) ListUserSessions(ctx context.Context, userID string) ([]Session, error) ListUsers(ctx context.Context) ([]User, error) @@ -72,6 +183,10 @@ type Querier interface { // makes the queue crash-safe: a killed process loses no work, it is simply // picked up again once the lease lapses. ReclaimExpiredLeases(ctx context.Context, arg ReclaimExpiredLeasesParams) (int64, error) + // Records everything stages 0 and 1 discovered, in one statement. Writing the + // probe result and the new state together keeps a crash from leaving a document + // that claims to be probed but has no page count. + RecordDocumentProbe(ctx context.Context, arg RecordDocumentProbeParams) error RecordJobUsage(ctx context.Context, arg RecordJobUsageParams) error // ReleaseJob returns a job to the queue without counting the attempt, used when a // worker is shutting down rather than failing. Without the decrement, every @@ -80,18 +195,213 @@ type Querier interface { ReleaseJob(ctx context.Context, arg ReleaseJobParams) error // RetryJob returns a failed attempt to the queue with a backoff delay. RetryJob(ctx context.Context, arg RetryJobParams) error + // Queries over doc_blocks_fts: which manual says X, and where. See + // 00006_block_search.sql for the index's reasoning and the measurement behind the + // tokeniser, and docs/design/search.md for the contract. + // + // THIS FILE MUST STAY PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 + // mixes up character and byte offsets when it cuts statements out of a file, so + // one non-ASCII character anywhere above corrupts every statement after it -- + // silently in the dangerous case: `make sqlc` exits 0, the Go compiles, the + // linter passes, and the statement fails at PREPARE time inside a request. The + // full measurement is in the header of docregions.sql; TestQueryFilesAreASCII is + // the cause-side guard and TestSearchQueriesExecute the symptom-side one. + // + // TWO THINGS SQLC CANNOT PARSE, BOTH LEARNED HERE AND BOTH LOAD-BEARING. + // + // 1. `WHERE doc_blocks_fts MATCH ?` -- the documented FTS5 form, where the left + // side is the table's own hidden column -- fails generation with `column + // "doc_blocks_fts" does not exist`, because sqlc models the virtual table as + // its declared columns only. `doc_blocks_fts.text MATCH ?` generates and is + // the same query: text is the only indexed column, so a column-scoped match + // over it covers the whole index. Verified against a real database rather than + // assumed, in TestSearchQueriesExecute. + // + // 2. `AS rank` fails generation with `mismatched input 'rank'`, so the ordering + // column is named `score`. That is a happy accident: `rank` is also FTS5's own + // magic column, and a result column of that name reads as if it were that. + // + // Columns are listed explicitly rather than with SELECT *, so that adding a + // column later cannot silently change every caller's row shape. + // + // WHY EVERY QUERY JOINS documents AND devices. A hit has to say WHICH manual, not + // merely that something matched: README's first problem is that the paper pile is + // unsearchable, and "page 47 of something" does not solve it. The filename and the + // device's name are what a household recognises, and they cost one join each + // against a primary key. + // + // WHY THE HEADING BONUS IS 1.0. bm25 is negative and lower is better, so the + // bonus is subtracted. Measured on both real manuals: within one query bm25 spans + // about -9 to -2, and adjacent hits differ by 0.05 to 0.5, so 1.0 moves a heading + // past hits of comparable quality without overturning a decisively better one. On + // "Filter" in the column manual it lifts the maintenance heading "Ausblasfilter + // austauschen" over the parts-list fragments ("1. Filter", "13. Filter") that + // bm25's short-document bias otherwise puts first; on "Saugkraft" the + // troubleshooting cell "Saugkraft ist zu gering" at -8.5 stays first, which is + // right. Both numbers are returned, so the judgement can be argued with rather + // than merely trusted. + SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) + // The same question narrowed to one manual, which is what a reader already inside + // a document asks. A separate statement rather than an optional parameter, because + // sqlc has no optional parameters and `b.document_id = ? OR ? = ''` would put the + // widest query in the household on the sentinel path. + SearchBlocksInDocument(ctx context.Context, arg SearchBlocksInDocumentParams) ([]SearchBlocksInDocumentRow, error) + // THE HOLE THE TOKENISER LEAVES, AND WHAT FILLS IT. + // + // A trigram index holds no token shorter than three characters, so a query of one + // or two characters matches nothing at all -- not "fewer results", none. That is + // tolerable in German and Russian, where a two-letter query is not a word anyone + // searches for, and it is not tolerable in Chinese or Japanese, where two + // characters is an ordinary word: measured on the sequential manual, the two + // characters for "power" occur in 27 stored blocks and those for "product" in 24, + // and the index finds 0 of each. + // + // So a query the index cannot represent is answered by scanning instead. Measured + // over the 3,122 blocks of both real manuals: 1.9 ms for a two-character Japanese + // query, against 0.2 ms for the same question through the index. A household's + // whole library is a small multiple of that corpus, so the scan stays inside a + // request rather than becoming a job. + // + // instr rather than LIKE, because `%` and `_` in a user's query are LIKE wildcards + // and a search box must not have a pattern language. lower() on both sides is + // SQLite's own, which folds ASCII and nothing else -- exact for the CJK queries + // this path exists for, and case-sensitive for a two-letter Cyrillic one, which is + // the honest limit of a scan that must not build an index to fix. + // + // There is no bm25 here because there is no index term to weigh, so score is 0 on + // every row and the order is the heading rule followed by reading order. A caller + // tells the two paths apart by the mode the API reports, not by inferring it from + // the numbers. + SearchBlocksSubstring(ctx context.Context, arg SearchBlocksSubstringParams) ([]SearchBlocksSubstringRow, error) + SearchBlocksSubstringInDocument(ctx context.Context, arg SearchBlocksSubstringInDocumentParams) ([]SearchBlocksSubstringInDocumentRow, error) + SetDocumentState(ctx context.Context, arg SetDocumentStateParams) error SetSetting(ctx context.Context, arg SetSettingParams) error + // What a conversion cost and covered, for the pipeline to report without reading + // every block back. Every aggregate is wrapped in CAST(... AS INTEGER): without + // it sqlc cannot infer an aggregate's type in SQLite and emits interface{}, + // pushing a type assertion onto every caller. + SummarizeDocBlocks(ctx context.Context, documentID string) ([]SummarizeDocBlocksRow, error) + // The language map as shown to the user: one row per language in the reconciled + // view, with its page total and whether any of its runs are disputed. + // + // A run with pdf_start = 0 named a language it could not place, so it covers no + // pages at all. Counting its span reported a language the printed index merely + // mentioned as a one-page section. + SummarizeDocLangs(ctx context.Context, arg SummarizeDocLangsParams) ([]SummarizeDocLangsRow, error) + // The region map as shown to the user: one row per language label, with the + // characters and runs it holds, how many pages it appears on, and whether any of + // its regions are disputed. + // + // Characters rather than pages is the point, because a page holding three + // languages is not a unit of size; pages are still what a reader is shown, so both + // are reported. Every aggregate is wrapped in CAST(... AS INTEGER): without it + // sqlc cannot infer the type and emits interface{}, pushing a type assertion onto + // every caller. + SummarizeDocRegions(ctx context.Context, documentID string) ([]SummarizeDocRegionsRow, error) // The CAST is load-bearing: without it sqlc cannot infer the type of an // aggregate in SQLite and generates interface{}, pushing a type assertion onto // every caller. Wrap aggregates in CAST(... AS INTEGER) throughout. TotalBlobBytes(ctx context.Context) (int64, error) TouchSession(ctx context.Context, arg TouchSessionParams) error TouchUserLogin(ctx context.Context, arg TouchUserLoginParams) error + UpdateDevice(ctx context.Context, arg UpdateDeviceParams) (Device, error) UpdateJobProgress(ctx context.Context, arg UpdateJobProgressParams) error + UpdateLocation(ctx context.Context, arg UpdateLocationParams) (Location, error) UpdateUserPassword(ctx context.Context, arg UpdateUserPasswordParams) error // Blobs are content-addressed, so re-adding identical bytes is a no-op rather // than a conflict. That is what makes uploading the same manual twice cheap. UpsertBlob(ctx context.Context, arg UpsertBlobParams) error + // Queries over doc_blocks and doc_figures: what a conversion produced. See + // 00005_doc_blocks.sql for the schema's reasoning and docs/design/conversion.md + // for the contract. + // + // THIS FILE MUST STAY PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 + // (pinned in tools/go.mod) mixes up character and byte offsets when it cuts + // statements out of a file, so one non-ASCII character anywhere above corrupts + // every statement after it -- silently, in the dangerous case: `make sqlc` exits + // 0, the Go compiles, the linter passes, and the statement fails at PREPARE time + // inside a background job against a user's database. The full measurement is in + // the header of docregions.sql; TestQueryFilesAreASCII is the cause-side guard + // and TestDocBlockQueriesExecute the symptom-side one. + // + // Columns are listed explicitly rather than with SELECT *, so that adding a + // column later cannot silently change every caller's row shape. + // Upsert on the natural key (document_id, page, region_x0, idx), because a + // conversion job may run twice and must converge on the same rows rather than + // duplicating them. + // + // Every non-key column is updated, kind and lang included. Nothing about a + // block's classification is in the key, so a paragraph that a better heading rule + // promotes to a heading is the same block updated in place -- see the note above + // the primary key in 00005_doc_blocks.sql. + // + // This is belt and braces beside the delete SaveConversion does first, and it + // cannot be the whole story: a re-conversion that produces FEWER blocks in a + // region would otherwise leave the tail of the previous run behind at higher + // indices, where it reads as content. + UpsertDocBlock(ctx context.Context, arg UpsertDocBlockParams) error + // Upsert on (document_id, page, idx). A figure has no region and no language in + // its key, because conversion.md settles that a picture belonging to no language + // belongs to every language. + UpsertDocFigure(ctx context.Context, arg UpsertDocFigureParams) error + // A figure's callout labels: the text a leader points at. No geometry -- 00009 + // removed it, because the crop is a band that already prints its own labels and the + // stored text is now the picture's accessible description rather than something a + // reader re-lays out. Kept out of doc_blocks on purpose -- see 00008's header for why + // a sixth block kind was the wrong shape and what it would have cost. + // Upsert on (document_id, page, figure_idx, idx), so a re-conversion converges. + UpsertDocFigureLabel(ctx context.Context, arg UpsertDocFigureLabelParams) error + UpsertDocLang(ctx context.Context, arg UpsertDocLangParams) error + // Upsert on the natural key, because a probe job may run twice and must converge + // on the same rows rather than duplicating them. + UpsertDocPage(ctx context.Context, arg UpsertDocPageParams) error + // Queries over doc_regions: one language's territory on a page. See + // 00004_doc_regions.sql for the schema's reasoning and docs/design/regions.md for + // the contract. + // + // TWO RULES FOR THIS FILE, BOTH LEARNED THE HARD WAY WHILE WRITING IT. + // + // 1. KEEP THIS FILE PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 + // (pinned in tools/go.mod) mixes up character and byte offsets when it cuts + // statements out of a file, so a single non-ASCII character anywhere earlier + // corrupts every statement after it. What is measured is the rule, not the + // internals: the damage equals the extra bytes those characters occupy, one + // character of SQL lost per extra byte. + // + // Two shapes were observed, and the quiet one is the dangerous one. With + // em-dashes in a comment above, "ORDER BY first_page, code" generated as + // "ORDER BY first_page, co" for one and "ORDER BY first_pa" for four -- clean + // Go, broken SQL. With em-dashes placed differently, sqlc instead garbled a + // statement badly enough to fail its own parser, printing tokens like + // "SELdocument_id" and exiting noisily. Which of the two you get depends on + // where the character sits, so neither a clean run nor a loud failure tells + // you the file is safe. Only ASCII does. + // + // The direction of sqlc's own mismatch is deliberately not asserted here. It + // was not read out of sqlc's source, and the two published guesses point + // opposite ways -- byte offsets applied to characters would overshoot a + // statement's end rather than cut it short, which is not what happens. The + // rule above is what was measured and is what protects this file. + // + // This is the worst failure shape available: `make sqlc` exits 0, the generated + // Go compiles, the linter is happy, and the statement fails at PREPARE time + // inside a background job against a user's database. All ten pre-existing query + // files happen to be pure ASCII, which is the only reason this had not bitten + // anyone yet. That was checked rather than assumed: 0 non-ASCII bytes across + // every one of them. TestDocRegionQueriesExecute in internal/db is the guard; + // it runs every statement below against a real migrated database, so a mangled + // one cannot reach a user. + // + // 2. Columns are listed explicitly rather than with SELECT *, so that adding a + // column to doc_regions later cannot silently change every caller's row shape. + // Upsert on the natural key (document_id, source, page, x0), because a probe job + // may run twice and must converge on the same rows rather than duplicating them. + // + // This is belt and braces beside the delete that SaveProbe does first, and it + // cannot be the whole story: source is part of the key and a region's source can + // change between probes, so an upsert alone would leave the superseded row behind + // at the same x0. The note at the foot of 00004_doc_regions.sql explains why. + UpsertDocRegion(ctx context.Context, arg UpsertDocRegionParams) error } var _ Querier = (*Queries)(nil) diff --git a/internal/db/gen/search.sql.go b/internal/db/gen/search.sql.go new file mode 100644 index 0000000..7310d8d --- /dev/null +++ b/internal/db/gen/search.sql.go @@ -0,0 +1,415 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: search.sql + +package gen + +import ( + "context" +) + +const countSearchableBlocks = `-- name: CountSearchableBlocks :one +SELECT CAST(count(*) AS INTEGER) FROM doc_blocks +` + +// How many blocks are indexed at all, so a caller can tell "nothing matched" from +// "nothing has been converted yet". Wrapped in CAST(... AS INTEGER) for the reason +// every other aggregate in these files is: without it sqlc cannot infer an +// aggregate's type in SQLite and emits interface{}. +func (q *Queries) CountSearchableBlocks(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countSearchableBlocks) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + +const searchBlocks = `-- name: SearchBlocks :many + +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + snippet(doc_blocks_fts, 0, '', '', '...', 64) AS snippet, + CAST(bm25(doc_blocks_fts) AS REAL) AS bm25, + CAST(bm25(doc_blocks_fts) + - (CASE b.kind WHEN 'heading' THEN 1.0 ELSE 0.0 END) AS REAL) AS score +FROM doc_blocks_fts +JOIN doc_blocks b ON b.rowid = doc_blocks_fts.rowid +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE doc_blocks_fts.text MATCH ?1 +ORDER BY score, b.document_id, b.page, b.region_x0, b.idx +LIMIT ?2 +` + +type SearchBlocksParams struct { + Match string + Limit int64 +} + +type SearchBlocksRow struct { + DocumentID string + Filename string + State string + DeviceID string + DeviceName string + Page int64 + RegionX0 int64 + Idx int64 + Kind string + Level int64 + Lang string + Chars int64 + Snippet string + Bm25 float64 + Score float64 +} + +// Queries over doc_blocks_fts: which manual says X, and where. See +// 00006_block_search.sql for the index's reasoning and the measurement behind the +// tokeniser, and docs/design/search.md for the contract. +// +// THIS FILE MUST STAY PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 +// mixes up character and byte offsets when it cuts statements out of a file, so +// one non-ASCII character anywhere above corrupts every statement after it -- +// silently in the dangerous case: `make sqlc` exits 0, the Go compiles, the +// linter passes, and the statement fails at PREPARE time inside a request. The +// full measurement is in the header of docregions.sql; TestQueryFilesAreASCII is +// the cause-side guard and TestSearchQueriesExecute the symptom-side one. +// +// TWO THINGS SQLC CANNOT PARSE, BOTH LEARNED HERE AND BOTH LOAD-BEARING. +// +// 1. `WHERE doc_blocks_fts MATCH ?` -- the documented FTS5 form, where the left +// side is the table's own hidden column -- fails generation with `column +// "doc_blocks_fts" does not exist`, because sqlc models the virtual table as +// its declared columns only. `doc_blocks_fts.text MATCH ?` generates and is +// the same query: text is the only indexed column, so a column-scoped match +// over it covers the whole index. Verified against a real database rather than +// assumed, in TestSearchQueriesExecute. +// +// 2. `AS rank` fails generation with `mismatched input 'rank'`, so the ordering +// column is named `score`. That is a happy accident: `rank` is also FTS5's own +// magic column, and a result column of that name reads as if it were that. +// +// Columns are listed explicitly rather than with SELECT *, so that adding a +// column later cannot silently change every caller's row shape. +// +// WHY EVERY QUERY JOINS documents AND devices. A hit has to say WHICH manual, not +// merely that something matched: README's first problem is that the paper pile is +// unsearchable, and "page 47 of something" does not solve it. The filename and the +// device's name are what a household recognises, and they cost one join each +// against a primary key. +// +// WHY THE HEADING BONUS IS 1.0. bm25 is negative and lower is better, so the +// bonus is subtracted. Measured on both real manuals: within one query bm25 spans +// about -9 to -2, and adjacent hits differ by 0.05 to 0.5, so 1.0 moves a heading +// past hits of comparable quality without overturning a decisively better one. On +// "Filter" in the column manual it lifts the maintenance heading "Ausblasfilter +// austauschen" over the parts-list fragments ("1. Filter", "13. Filter") that +// bm25's short-document bias otherwise puts first; on "Saugkraft" the +// troubleshooting cell "Saugkraft ist zu gering" at -8.5 stays first, which is +// right. Both numbers are returned, so the judgement can be argued with rather +// than merely trusted. +func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) { + rows, err := q.db.QueryContext(ctx, searchBlocks, arg.Match, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SearchBlocksRow{} + for rows.Next() { + var i SearchBlocksRow + if err := rows.Scan( + &i.DocumentID, + &i.Filename, + &i.State, + &i.DeviceID, + &i.DeviceName, + &i.Page, + &i.RegionX0, + &i.Idx, + &i.Kind, + &i.Level, + &i.Lang, + &i.Chars, + &i.Snippet, + &i.Bm25, + &i.Score, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const searchBlocksInDocument = `-- name: SearchBlocksInDocument :many +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + snippet(doc_blocks_fts, 0, '', '', '...', 64) AS snippet, + CAST(bm25(doc_blocks_fts) AS REAL) AS bm25, + CAST(bm25(doc_blocks_fts) + - (CASE b.kind WHEN 'heading' THEN 1.0 ELSE 0.0 END) AS REAL) AS score +FROM doc_blocks_fts +JOIN doc_blocks b ON b.rowid = doc_blocks_fts.rowid +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE doc_blocks_fts.text MATCH ?1 + AND b.document_id = ?2 +ORDER BY score, b.page, b.region_x0, b.idx +LIMIT ?3 +` + +type SearchBlocksInDocumentParams struct { + Match string + DocumentID string + Limit int64 +} + +type SearchBlocksInDocumentRow struct { + DocumentID string + Filename string + State string + DeviceID string + DeviceName string + Page int64 + RegionX0 int64 + Idx int64 + Kind string + Level int64 + Lang string + Chars int64 + Snippet string + Bm25 float64 + Score float64 +} + +// The same question narrowed to one manual, which is what a reader already inside +// a document asks. A separate statement rather than an optional parameter, because +// sqlc has no optional parameters and `b.document_id = ? OR ? = ”` would put the +// widest query in the household on the sentinel path. +func (q *Queries) SearchBlocksInDocument(ctx context.Context, arg SearchBlocksInDocumentParams) ([]SearchBlocksInDocumentRow, error) { + rows, err := q.db.QueryContext(ctx, searchBlocksInDocument, arg.Match, arg.DocumentID, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SearchBlocksInDocumentRow{} + for rows.Next() { + var i SearchBlocksInDocumentRow + if err := rows.Scan( + &i.DocumentID, + &i.Filename, + &i.State, + &i.DeviceID, + &i.DeviceName, + &i.Page, + &i.RegionX0, + &i.Idx, + &i.Kind, + &i.Level, + &i.Lang, + &i.Chars, + &i.Snippet, + &i.Bm25, + &i.Score, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const searchBlocksSubstring = `-- name: SearchBlocksSubstring :many +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + substr(b.text, max(1, instr(lower(b.text), lower(?1)) - 24), 64) AS snippet, + CAST(0.0 AS REAL) AS bm25, + CAST(0.0 AS REAL) AS score +FROM doc_blocks b +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE instr(lower(b.text), lower(?1)) > 0 +ORDER BY (CASE b.kind WHEN 'heading' THEN 0 ELSE 1 END), + b.document_id, b.page, b.region_x0, b.idx +LIMIT ?2 +` + +type SearchBlocksSubstringParams struct { + Needle string + Limit int64 +} + +type SearchBlocksSubstringRow struct { + DocumentID string + Filename string + State string + DeviceID string + DeviceName string + Page int64 + RegionX0 int64 + Idx int64 + Kind string + Level int64 + Lang string + Chars int64 + Snippet string + Bm25 float64 + Score float64 +} + +// THE HOLE THE TOKENISER LEAVES, AND WHAT FILLS IT. +// +// A trigram index holds no token shorter than three characters, so a query of one +// or two characters matches nothing at all -- not "fewer results", none. That is +// tolerable in German and Russian, where a two-letter query is not a word anyone +// searches for, and it is not tolerable in Chinese or Japanese, where two +// characters is an ordinary word: measured on the sequential manual, the two +// characters for "power" occur in 27 stored blocks and those for "product" in 24, +// and the index finds 0 of each. +// +// So a query the index cannot represent is answered by scanning instead. Measured +// over the 3,122 blocks of both real manuals: 1.9 ms for a two-character Japanese +// query, against 0.2 ms for the same question through the index. A household's +// whole library is a small multiple of that corpus, so the scan stays inside a +// request rather than becoming a job. +// +// instr rather than LIKE, because `%` and `_` in a user's query are LIKE wildcards +// and a search box must not have a pattern language. lower() on both sides is +// SQLite's own, which folds ASCII and nothing else -- exact for the CJK queries +// this path exists for, and case-sensitive for a two-letter Cyrillic one, which is +// the honest limit of a scan that must not build an index to fix. +// +// There is no bm25 here because there is no index term to weigh, so score is 0 on +// every row and the order is the heading rule followed by reading order. A caller +// tells the two paths apart by the mode the API reports, not by inferring it from +// the numbers. +func (q *Queries) SearchBlocksSubstring(ctx context.Context, arg SearchBlocksSubstringParams) ([]SearchBlocksSubstringRow, error) { + rows, err := q.db.QueryContext(ctx, searchBlocksSubstring, arg.Needle, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SearchBlocksSubstringRow{} + for rows.Next() { + var i SearchBlocksSubstringRow + if err := rows.Scan( + &i.DocumentID, + &i.Filename, + &i.State, + &i.DeviceID, + &i.DeviceName, + &i.Page, + &i.RegionX0, + &i.Idx, + &i.Kind, + &i.Level, + &i.Lang, + &i.Chars, + &i.Snippet, + &i.Bm25, + &i.Score, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const searchBlocksSubstringInDocument = `-- name: SearchBlocksSubstringInDocument :many +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + substr(b.text, max(1, instr(lower(b.text), lower(?1)) - 24), 64) AS snippet, + CAST(0.0 AS REAL) AS bm25, + CAST(0.0 AS REAL) AS score +FROM doc_blocks b +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE instr(lower(b.text), lower(?1)) > 0 + AND b.document_id = ?2 +ORDER BY (CASE b.kind WHEN 'heading' THEN 0 ELSE 1 END), + b.page, b.region_x0, b.idx +LIMIT ?3 +` + +type SearchBlocksSubstringInDocumentParams struct { + Needle string + DocumentID string + Limit int64 +} + +type SearchBlocksSubstringInDocumentRow struct { + DocumentID string + Filename string + State string + DeviceID string + DeviceName string + Page int64 + RegionX0 int64 + Idx int64 + Kind string + Level int64 + Lang string + Chars int64 + Snippet string + Bm25 float64 + Score float64 +} + +func (q *Queries) SearchBlocksSubstringInDocument(ctx context.Context, arg SearchBlocksSubstringInDocumentParams) ([]SearchBlocksSubstringInDocumentRow, error) { + rows, err := q.db.QueryContext(ctx, searchBlocksSubstringInDocument, arg.Needle, arg.DocumentID, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []SearchBlocksSubstringInDocumentRow{} + for rows.Next() { + var i SearchBlocksSubstringInDocumentRow + if err := rows.Scan( + &i.DocumentID, + &i.Filename, + &i.State, + &i.DeviceID, + &i.DeviceName, + &i.Page, + &i.RegionX0, + &i.Idx, + &i.Kind, + &i.Level, + &i.Lang, + &i.Chars, + &i.Snippet, + &i.Bm25, + &i.Score, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/db/migrate_lang_source_test.go b/internal/db/migrate_lang_source_test.go new file mode 100644 index 0000000..6e534ce --- /dev/null +++ b/internal/db/migrate_lang_source_test.go @@ -0,0 +1,289 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "io/fs" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/pressly/goose/v3" + + "github.com/gordon2/manualbox/internal/id" +) + +// openAtVersion opens a real file-backed database and migrates it to exactly +// version v, so a migration can be exercised against data written by the schema +// that preceded it. Open() always migrates to head, which is the one thing a +// migration test must not do. +func openAtVersion(t *testing.T, path string, v int64) (*sql.DB, *goose.Provider) { + t.Helper() + + pool, err := openPool(Options{Path: path, BusyTimeout: 5 * time.Second}, false, true) + if err != nil { + t.Fatalf("openPool: %v", err) + } + t.Cleanup(func() { _ = pool.Close() }) + + sub, err := fs.Sub(migrationsFS, "migrations") + if err != nil { + t.Fatalf("fs.Sub: %v", err) + } + provider, err := goose.NewProvider(goose.DialectSQLite3, pool, sub) + if err != nil { + t.Fatalf("goose.NewProvider: %v", err) + } + if _, err := provider.UpTo(context.Background(), v); err != nil { + t.Fatalf("UpTo(%d): %v", v, err) + } + + got, err := provider.GetDBVersion(context.Background()) + if err != nil { + t.Fatalf("GetDBVersion: %v", err) + } + if got != v { + t.Fatalf("schema version = %d, want %d", got, v) + } + return pool, provider +} + +// snapshot renders every column of every row as text, ordered deterministically, +// so "the rows survived" can be asserted value by value rather than by counting. +// NULL is rendered distinctly from the empty string and from 0, which is the +// distinction a rebuild is most likely to lose. +func snapshot(t *testing.T, pool *sql.DB, query string) []string { + t.Helper() + + rows, err := pool.QueryContext(context.Background(), query) + if err != nil { + t.Fatalf("snapshot query: %v", err) + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + t.Fatalf("columns: %v", err) + } + + var out []string + for rows.Next() { + cells := make([]any, len(cols)) + for i := range cells { + cells[i] = new(any) + } + if err := rows.Scan(cells...); err != nil { + t.Fatalf("scan: %v", err) + } + var b strings.Builder + for i, c := range cells { + v := *(c.(*any)) + if i > 0 { + b.WriteString(" | ") + } + if v == nil { + fmt.Fprintf(&b, "%s=", cols[i]) + } else { + fmt.Fprintf(&b, "%s=%T(%v)", cols[i], v, v) + } + } + out = append(out, b.String()) + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + return out +} + +const ( + pagesSnapshot = `SELECT document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source + FROM doc_pages ORDER BY document_id, page_no` + langsSnapshot = `SELECT document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, + confidence, conflict, note, created_at + FROM doc_langs ORDER BY document_id, source, code, pdf_start` +) + +// TestMigration3WidensLangSourceChecks is the guard on a schema change to other +// people's data: 00003 rebuilds two shipped tables to widen a CHECK, and a +// rebuild that loses a row, a NULL, or the constraint itself is silent damage. +func TestMigration3WidensLangSourceChecks(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "migrate.db") + + pool, provider := openAtVersion(t, path, 2) + + exec := func(query string, args ...any) { + t.Helper() + if _, err := pool.ExecContext(ctx, query, args...); err != nil { + t.Fatalf("exec %s: %v", query, err) + } + } + mustFail := func(what, query string, args ...any) { + t.Helper() + if _, err := pool.ExecContext(ctx, query, args...); err == nil { + t.Errorf("%s: expected the CHECK constraint to reject this, it was accepted", what) + } + } + + // Parents first: doc_pages and doc_langs both cascade from documents. + docID := id.New(id.Document) + deviceID := id.New(id.Device) + sha := strings.Repeat("a", 64) + exec(`INSERT INTO blobs (sha256, size_bytes, media_type, created_at) VALUES (?, 1, 'application/pdf', ?)`, sha, Now()) + exec(`INSERT INTO devices (id, name, created_at, updated_at) VALUES (?, 'Dishwasher', ?, ?)`, deviceID, Now(), Now()) + exec(`INSERT INTO documents (id, device_id, blob_sha256, filename, media_type, created_at, updated_at) + VALUES (?, ?, ?, 'manual.pdf', 'application/pdf', ?, ?)`, docID, deviceID, sha, Now(), Now()) + + // doc_pages: every column exercised, including printed_folio NULL and a page + // with no text layer at all. + exec(`INSERT INTO doc_pages (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source) + VALUES (?, 1, 1840, 'Latin', 'DE', 3, 'de', 'page-tag')`, docID) + exec(`INSERT INTO doc_pages (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source) + VALUES (?, 2, 0, '', '', NULL, '', '')`, docID) + exec(`INSERT INTO doc_pages (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source) + VALUES (?, 3, 920, 'Cyrillic', 'УКР', NULL, 'uk', 'reconciled')`, docID) + + // doc_langs: one placed run per remaining signal, plus the pdf_start = 0 row — + // "named a language but could not place it", the state 00002 documents at + // length and the one a naive rebuild is most likely to reject. + exec(`INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, + confidence, conflict, note, created_at) + VALUES (?, 'index', 5, 12, 'CZ', 'cs', 'Návod k použití', 4, 0.75, 1, 'index claims an Arabic page', ?)`, docID, Now()) + exec(`INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, + confidence, conflict, note, created_at) + VALUES (?, 'index', 0, 0, 'ZH-HK', '', '', NULL, 0, 0, 'could not be placed', ?)`, docID, Now()) + exec(`INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, + confidence, conflict, note, created_at) + VALUES (?, 'script', 1, 4, 'de', 'de', '', NULL, 1.0, 0, '', ?)`, docID, Now()) + exec(`INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, + confidence, conflict, note, created_at) + VALUES (?, 'reconciled', 1, 4, 'de', 'de', '', NULL, 0.9, 1, 'page-tag disagreed', ?)`, docID, Now()) + + // 00002's shape: repertoire is not yet a legal value in either column. + mustFail("doc_pages.lang_source = repertoire at version 2", + `INSERT INTO doc_pages (document_id, page_no, lang, lang_source) VALUES (?, 90, 'el', 'repertoire')`, docID) + mustFail("doc_langs.source = repertoire at version 2", + `INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, created_at) + VALUES (?, 'repertoire', 20, 24, 'EL', ?)`, docID, Now()) + + pagesBefore := snapshot(t, pool, pagesSnapshot) + langsBefore := snapshot(t, pool, langsSnapshot) + if len(pagesBefore) != 3 || len(langsBefore) != 4 { + t.Fatalf("fixture rows: %d doc_pages, %d doc_langs; want 3 and 4", len(pagesBefore), len(langsBefore)) + } + + // The rebuild. goose wraps a migration in a transaction by default and the + // connection carries _pragma foreign_keys(1); if create/copy/drop/rename could + // not run under both, this is where it would fail. + if _, err := provider.UpTo(ctx, 3); err != nil { + t.Fatalf("UpTo(3): %v", err) + } + + assertSnapshotsEqual(t, "after Up", pagesBefore, snapshot(t, pool, pagesSnapshot), langsBefore, snapshot(t, pool, langsSnapshot)) + + // The indexes belonged to the dropped tables and had to be recreated. + for _, idx := range []string{"doc_pages_lang_idx", "doc_langs_source_idx", "doc_langs_lang_idx"} { + var name string + err := pool.QueryRowContext(ctx, + `SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?`, idx).Scan(&name) + if err != nil { + t.Errorf("index %q missing after the rebuild: %v", idx, err) + } + } + + // Leftover scratch tables would mean the rename did not happen. + var leftovers int + if err := pool.QueryRowContext(ctx, + `SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name LIKE 'doc_%_new'`).Scan(&leftovers); err != nil { + t.Fatalf("count scratch tables: %v", err) + } + if leftovers != 0 { + t.Errorf("%d scratch table(s) survived the migration", leftovers) + } + + // What the migration exists for. + exec(`INSERT INTO doc_pages (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source) + VALUES (?, 4, 300, 'Greek', '', NULL, 'el', 'repertoire')`, docID) + exec(`INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, + confidence, conflict, note, created_at) + VALUES (?, 'repertoire', 20, 24, 'EL', 'el', '', NULL, 0.6, 0, '', ?)`, docID, Now()) + + // Widened, not dropped. A rebuild that lost the constraint would pass every + // assertion above. + mustFail("doc_pages.lang_source = bogus at version 3", + `INSERT INTO doc_pages (document_id, page_no, lang, lang_source) VALUES (?, 91, 'el', 'bogus')`, docID) + mustFail("doc_langs.source = bogus at version 3", + `INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, created_at) + VALUES (?, 'bogus', 30, 34, 'EL', ?)`, docID, Now()) + + // Everything else 00002 constrained must still be constrained. + mustFail("doc_pages.page_no = 0 at version 3", + `INSERT INTO doc_pages (document_id, page_no) VALUES (?, 0)`, docID) + mustFail("doc_langs.pdf_end < pdf_start at version 3", + `INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, created_at) + VALUES (?, 'index', 40, 30, 'EL', ?)`, docID, Now()) + mustFail("doc_langs.confidence out of range at version 3", + `INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, confidence, created_at) + VALUES (?, 'index', 50, 54, 'EL', 1.5, ?)`, docID, Now()) + mustFail("doc_pages row for a nonexistent document at version 3", + `INSERT INTO doc_pages (document_id, page_no) VALUES ('doc_nonexistent', 1)`) + + // STRICT must survive the rebuild too. + mustFail("text in doc_pages.chars at version 3", + `INSERT INTO doc_pages (document_id, page_no, chars) VALUES (?, 92, 'lots')`, docID) + + // The cascade from documents must survive the rebuild: the FK clause is easy + // to copy without ON DELETE CASCADE and nothing else would notice. + for _, table := range []string{"doc_pages", "doc_langs"} { + var onDelete string + if err := pool.QueryRowContext(ctx, + `SELECT "on_delete" FROM pragma_foreign_key_list(?) WHERE "table" = 'documents'`, table).Scan(&onDelete); err != nil { + t.Fatalf("%s: read foreign key to documents: %v", table, err) + } + if onDelete != "CASCADE" { + t.Errorf("%s: ON DELETE to documents = %q, want CASCADE", table, onDelete) + } + } + + // Down must restore 00002's shape, not drop the tables. Remove the rows that + // only version 3 can hold first — there is nowhere for them to go, and the + // migration failing on them is the honest behaviour, not the one under test. + exec(`DELETE FROM doc_pages WHERE lang_source = 'repertoire'`) + exec(`DELETE FROM doc_langs WHERE source = 'repertoire'`) + + if _, err := provider.DownTo(ctx, 2); err != nil { + t.Fatalf("DownTo(2): %v", err) + } + + assertSnapshotsEqual(t, "after Down", pagesBefore, snapshot(t, pool, pagesSnapshot), langsBefore, snapshot(t, pool, langsSnapshot)) + + mustFail("doc_pages.lang_source = repertoire after Down", + `INSERT INTO doc_pages (document_id, page_no, lang, lang_source) VALUES (?, 93, 'el', 'repertoire')`, docID) + mustFail("doc_langs.source = repertoire after Down", + `INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, created_at) + VALUES (?, 'repertoire', 60, 64, 'EL', ?)`, docID, Now()) +} + +func assertSnapshotsEqual(t *testing.T, when string, pagesWant, pagesGot, langsWant, langsGot []string) { + t.Helper() + for _, c := range []struct { + table string + want, got []string + }{ + {"doc_pages", pagesWant, pagesGot}, + {"doc_langs", langsWant, langsGot}, + } { + if len(c.got) != len(c.want) { + t.Errorf("%s: %s has %d rows %s, want %d\n got: %v\nwant: %v", + when, c.table, len(c.got), when, len(c.want), c.got, c.want) + continue + } + for i := range c.want { + if c.got[i] != c.want[i] { + t.Errorf("%s: %s row %d changed\n got: %s\nwant: %s", when, c.table, i, c.got[i], c.want[i]) + } + } + } +} diff --git a/internal/db/migrations/00002_registry_and_documents.sql b/internal/db/migrations/00002_registry_and_documents.sql new file mode 100644 index 0000000..3f2ac60 --- /dev/null +++ b/internal/db/migrations/00002_registry_and_documents.sql @@ -0,0 +1,218 @@ +-- M1: the registry (locations, devices) and the document ingest tables. +-- +-- Conventions are those of 00001_init.sql: prefixed ULID primary keys, INTEGER +-- Unix millisecond timestamps, STRICT tables. +-- +-- One deliberate departure. The derived tables — doc_pages and doc_langs — use +-- COMPOSITE natural primary keys rather than ULIDs. That is not a style +-- preference: a job handler can run twice (a worker may die after doing its work +-- but before recording success), so the probe must be able to write its results +-- again without duplicating them. A natural key turns "run it again" into an +-- upsert over the same rows. With surrogate ULIDs the second run would insert a +-- parallel set of 560 page rows and the reconciliation would silently double. + +-- +goose Up + +-- Where things are. Nestable, so "House > Kitchen > Under the sink" works +-- without a separate hierarchy table. +CREATE TABLE locations ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + -- Self-reference for nesting. ON DELETE SET NULL rather than CASCADE: + -- deleting a room should orphan its shelves, never silently delete the + -- devices filed under them. + parent_id TEXT REFERENCES locations(id) ON DELETE SET NULL, + notes TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +) STRICT; + +CREATE INDEX locations_parent_idx ON locations(parent_id); + +-- The things a household owns. +-- +-- Deliberately absent: serial number and purchase price. Both are high-harm +-- fields that docs/design/privacy.md says must be encrypted with a key held +-- outside the data directory, and the keyring is not wired into the schema yet. +-- Adding them as plaintext columns now would mean either migrating real user +-- data later or quietly storing the most identifying field manualbox holds in +-- the clear. They land with the keyring, in their encrypted form. +CREATE TABLE devices ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + brand TEXT NOT NULL DEFAULT '', + model TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT '', + location_id TEXT REFERENCES locations(id) ON DELETE SET NULL, + notes TEXT NOT NULL DEFAULT '', + -- Date of purchase, millis. Nullable because it is frequently unknown, and + -- an unknown date must not become the epoch. + purchased_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +) STRICT; + +CREATE INDEX devices_location_idx ON devices(location_id); +CREATE INDEX devices_name_idx ON devices(name); + +-- An uploaded file belonging to a device. The bytes live in the blob store; this +-- row is the document's identity, its classification, and the result of probing +-- it. +CREATE TABLE documents ( + id TEXT PRIMARY KEY, + device_id TEXT NOT NULL REFERENCES devices(id) ON DELETE CASCADE, + blob_sha256 TEXT NOT NULL REFERENCES blobs(sha256), + + -- The name the user's file had, for display only. Never used to build a path. + filename TEXT NOT NULL DEFAULT '', + media_type TEXT NOT NULL DEFAULT '', + + -- Classification drives privacy behaviour, not just presentation: receipts + -- and warranties are never sent to a cloud provider (privacy.md), so the + -- class has to be known before any provider is called. + kind TEXT NOT NULL DEFAULT 'manual' + CHECK (kind IN ('manual', 'receipt', 'warranty', 'photo', 'other')), + + -- Pipeline state. 'converting' and 'ready' are listed now although nothing + -- sets them yet: extending a CHECK constraint in SQLite means rebuilding the + -- table, and naming the two states that are certainly coming costs nothing. + -- uploaded stored, probe queued + -- probing a worker is probing it + -- awaiting_scope probed; waiting for the user to approve what to process + -- declined the user said no; the original is kept regardless + -- converting conversion in progress + -- ready nothing further to do automatically + -- failed probing or conversion failed permanently + state TEXT NOT NULL DEFAULT 'uploaded' + CHECK (state IN ('uploaded', 'probing', 'awaiting_scope', + 'declined', 'converting', 'ready', 'failed')), + last_error TEXT NOT NULL DEFAULT '', + + -- Stage 0 and stage 1 results. All nullable: they are unknown until the + -- probe runs, and NULL says "not yet" where 0 would claim "none". + page_count INTEGER, + encrypted INTEGER CHECK (encrypted IN (0, 1)), + tagged INTEGER CHECK (tagged IN (0, 1)), + has_text_layer INTEGER CHECK (has_text_layer IN (0, 1)), + -- Median extracted characters (runes, not bytes) on a content page. A scan + -- yields ~0, which is what selects between the free extraction path and one + -- that costs a vision call per page. + median_chars_per_page INTEGER, + -- Page range holding actual content, excluding front matter and back cover. + content_start_page INTEGER, + content_end_page INTEGER, + + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + probed_at INTEGER +) STRICT; + +-- The same bytes attached to the same device twice is the same document. This is +-- what makes an accidental double upload a no-op instead of a duplicate, and it +-- is the constraint the upload handler relies on to be idempotent. +CREATE UNIQUE INDEX documents_device_blob_idx ON documents(device_id, blob_sha256); +CREATE INDEX documents_device_idx ON documents(device_id); +CREATE INDEX documents_blob_idx ON documents(blob_sha256); +CREATE INDEX documents_state_idx ON documents(state); + +-- Per-page facts recorded by the probe. One row per page of the original. +-- +-- This is the evidence behind the language map: it holds what each individual +-- signal saw on that page, so a disagreement can be shown to the user rather +-- than averaged away. See docs/design/language-detection.md. +CREATE TABLE doc_pages ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + -- 1-based page number in the original PDF. + page_no INTEGER NOT NULL CHECK (page_no >= 1), + + -- Extracted characters (runes). Zero means no text layer on this page. + chars INTEGER NOT NULL DEFAULT 0 CHECK (chars >= 0), + -- Dominant Unicode script, e.g. 'Latin', 'Cyrillic', 'Han', 'Kana'. Empty + -- when the page has no text to judge. + script TEXT NOT NULL DEFAULT '', + -- The language code printed on the page itself, when the manual prints one. + -- Empty when absent, which is common and not an error. + page_tag TEXT NOT NULL DEFAULT '', + -- The page number printed in the page's own footer, which is not the PDF + -- page number. Nullable: some pages print none at all. + printed_folio INTEGER, + + -- The resolved language for this page and which signal decided it. + lang TEXT NOT NULL DEFAULT '', + lang_source TEXT NOT NULL DEFAULT '' + CHECK (lang_source IN ('', 'page-tag', 'index', 'script', 'detector', 'reconciled')), + + PRIMARY KEY (document_id, page_no) +) STRICT; + +CREATE INDEX doc_pages_lang_idx ON doc_pages(document_id, lang); + +-- Language runs within a document: a contiguous span of pages in one language. +-- +-- Every signal writes its own rows for the same document, so the map is not one +-- opinion but several, each attributed. The reconciled view is source +-- 'reconciled'; the others are kept because "this manual also contains FR, IT, +-- ES..." must be answerable without re-probing, and because a conflict has to +-- remain inspectable after the fact. +CREATE TABLE doc_langs ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + + -- Which signal produced this run. Part of the key, so each signal's view + -- coexists with the others. + source TEXT NOT NULL + CHECK (source IN ('page-tag', 'index', 'script', 'detector', 'reconciled')), + + -- Zero means "this signal named a language but could not place it". + -- + -- That is a real and useful state, not a defect to reject. A printed index + -- routinely claims a page that does not exist, or one whose script makes the + -- claim impossible — a real manual lists Czech at a page that is Arabic. The + -- claim is still evidence: it tells the user their manual's contents table is + -- wrong, which is exactly the kind of conflict this schema exists to surface + -- rather than silently discard. So the label is kept and the boundary is not + -- invented. + pdf_start INTEGER NOT NULL CHECK (pdf_start >= 0), + pdf_end INTEGER NOT NULL CHECK (pdf_end >= 0), + + -- code is the language as the document expresses it, which is not always a + -- valid tag: real manuals print 'UA' for Ukrainian, 'CZ' for Czech and + -- 'ZH-HK' for Cantonese. lang is that value normalised to BCP-47, empty when + -- it could not be normalised — keeping both means an unrecognised code is + -- still reportable instead of being dropped. + code TEXT NOT NULL, + lang TEXT NOT NULL DEFAULT '', + + -- The section title as printed in the manual's own contents table, in that + -- language. Only the index signal can supply this. + title TEXT NOT NULL DEFAULT '', + -- The start page the printed index claims, which is frequently 1-2 off from + -- the page actually printed. Nullable; only the index signal sets it. + printed_page INTEGER, + + confidence REAL NOT NULL DEFAULT 0 CHECK (confidence BETWEEN 0 AND 1), + -- Set on a reconciled run when the signals disagreed about it. The note says + -- how. Surfacing the conflict is the requirement; resolving it silently is + -- what the design forbids. + conflict INTEGER NOT NULL DEFAULT 0 CHECK (conflict IN (0, 1)), + note TEXT NOT NULL DEFAULT '', + + created_at INTEGER NOT NULL, + + -- Natural key, so re-probing overwrites rather than duplicating. The code is + -- part of it, not just the starting page: a signal may name several languages + -- it could not place, and those all share a start of 0. Keying on the page + -- alone would silently collapse them into whichever was written last. + PRIMARY KEY (document_id, source, code, pdf_start), + + CHECK (pdf_end >= pdf_start) +) STRICT; + +CREATE INDEX doc_langs_source_idx ON doc_langs(document_id, source); +CREATE INDEX doc_langs_lang_idx ON doc_langs(document_id, lang); + +-- +goose Down +DROP TABLE doc_langs; +DROP TABLE doc_pages; +DROP TABLE documents; +DROP TABLE devices; +DROP TABLE locations; diff --git a/internal/db/migrations/00003_widen_lang_source_checks.sql b/internal/db/migrations/00003_widen_lang_source_checks.sql new file mode 100644 index 0000000..9257153 --- /dev/null +++ b/internal/db/migrations/00003_widen_lang_source_checks.sql @@ -0,0 +1,184 @@ +-- M1: admit 'repertoire' as a language signal in the two shipped tables. +-- +-- `repertoire` already exists in Go as doc.SourceRepertoire — the signal that +-- names a language from the characters a page actually uses, the letters only +-- some alphabets have. 00002 listed the other five signals and missed this one, +-- so writing it would fail a CHECK at runtime rather than at review time. This +-- migration is only that correction. The doc_regions table that motivated it +-- lands separately (docs/design/regions.md), so that a schema change to other +-- people's data stays reviewable and revertible on its own. +-- +-- Why a rebuild and not an ALTER. SQLite has no way to alter a CHECK +-- constraint: the constraint is part of the stored CREATE TABLE text, and only +-- create-copy-drop-rename replaces it. That is the documented procedure, and it +-- is why 00002's own comment pre-listed the two document states it knew were +-- coming — extending a closed set here costs a table rebuild. +-- +-- Why this rebuild is safe. Both tables are foreign-key LEAVES: they reference +-- documents(id), and nothing references them. So the drop cannot orphan a child +-- row and the rename cannot leave a dangling reference. Their parent, documents, +-- is untouched. Every column, type, NOT NULL, DEFAULT, other CHECK, composite +-- PRIMARY KEY, STRICT and cascade below is reproduced verbatim from 00002; the +-- CHECK list is the only difference, and only by appending. Verified empirically +-- that this runs inside goose's transaction with _pragma foreign_keys(1) set +-- (internal/db/db.go), so a failure part-way leaves the old tables intact. +-- +-- Append-only, and both directions rebuild. The Down migration restores 00002's +-- narrower lists rather than dropping the tables, so a downgrade keeps the rows. +-- It will fail, correctly, if any row by then holds 'repertoire'. + +-- +goose Up + +-- doc_pages: one row per page of the original, holding what each signal saw. +CREATE TABLE doc_pages_new ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + -- 1-based page number in the original PDF. + page_no INTEGER NOT NULL CHECK (page_no >= 1), + + -- Extracted characters (runes). Zero means no text layer on this page. + chars INTEGER NOT NULL DEFAULT 0 CHECK (chars >= 0), + -- Dominant Unicode script, e.g. 'Latin', 'Cyrillic', 'Han', 'Kana'. Empty + -- when the page has no text to judge. + script TEXT NOT NULL DEFAULT '', + -- The language code printed on the page itself, when the manual prints one. + -- Empty when absent, which is common and not an error. + page_tag TEXT NOT NULL DEFAULT '', + -- The page number printed in the page's own footer, which is not the PDF + -- page number. Nullable: some pages print none at all. + printed_folio INTEGER, + + -- The resolved language for this page and which signal decided it. + -- 'repertoire' is the addition: see the header. + lang TEXT NOT NULL DEFAULT '', + lang_source TEXT NOT NULL DEFAULT '' + CHECK (lang_source IN ('', 'page-tag', 'index', 'script', 'repertoire', 'detector', 'reconciled')), + + PRIMARY KEY (document_id, page_no) +) STRICT; + +-- Named columns on both sides, so a future column added to one table and not the +-- other fails loudly here instead of shifting values silently. +INSERT INTO doc_pages_new (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source) +SELECT document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source FROM doc_pages; + +DROP TABLE doc_pages; +ALTER TABLE doc_pages_new RENAME TO doc_pages; + +-- Indexes belong to the dropped table, so they are recreated, not renamed. +CREATE INDEX doc_pages_lang_idx ON doc_pages(document_id, lang); + +-- doc_langs: a contiguous span of pages in one language, per signal. +CREATE TABLE doc_langs_new ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + + -- Which signal produced this run. Part of the key, so each signal's view + -- coexists with the others. 'repertoire' is the addition. + source TEXT NOT NULL + CHECK (source IN ('page-tag', 'index', 'script', 'repertoire', 'detector', 'reconciled')), + + -- Zero means "this signal named a language but could not place it". + -- + -- That is a real and useful state, not a defect to reject. A printed index + -- routinely claims a page that does not exist, or one whose script makes the + -- claim impossible — a real manual lists Czech at a page that is Arabic. The + -- claim is still evidence: it tells the user their manual's contents table is + -- wrong, which is exactly the kind of conflict this schema exists to surface + -- rather than silently discard. So the label is kept and the boundary is not + -- invented. + pdf_start INTEGER NOT NULL CHECK (pdf_start >= 0), + pdf_end INTEGER NOT NULL CHECK (pdf_end >= 0), + + -- code is the language as the document expresses it, which is not always a + -- valid tag: real manuals print 'UA' for Ukrainian, 'CZ' for Czech and + -- 'ZH-HK' for Cantonese. lang is that value normalised to BCP-47, empty when + -- it could not be normalised — keeping both means an unrecognised code is + -- still reportable instead of being dropped. + code TEXT NOT NULL, + lang TEXT NOT NULL DEFAULT '', + + -- The section title as printed in the manual's own contents table, in that + -- language. Only the index signal can supply this. + title TEXT NOT NULL DEFAULT '', + -- The start page the printed index claims, which is frequently 1-2 off from + -- the page actually printed. Nullable; only the index signal sets it. + printed_page INTEGER, + + confidence REAL NOT NULL DEFAULT 0 CHECK (confidence BETWEEN 0 AND 1), + -- Set on a reconciled run when the signals disagreed about it. The note says + -- how. Surfacing the conflict is the requirement; resolving it silently is + -- what the design forbids. + conflict INTEGER NOT NULL DEFAULT 0 CHECK (conflict IN (0, 1)), + note TEXT NOT NULL DEFAULT '', + + created_at INTEGER NOT NULL, + + -- Natural key, so re-probing overwrites rather than duplicating. The code is + -- part of it, not just the starting page: a signal may name several languages + -- it could not place, and those all share a start of 0. Keying on the page + -- alone would silently collapse them into whichever was written last. + PRIMARY KEY (document_id, source, code, pdf_start), + + CHECK (pdf_end >= pdf_start) +) STRICT; + +INSERT INTO doc_langs_new (document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, confidence, conflict, note, created_at) +SELECT document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, confidence, conflict, note, created_at FROM doc_langs; + +DROP TABLE doc_langs; +ALTER TABLE doc_langs_new RENAME TO doc_langs; + +CREATE INDEX doc_langs_source_idx ON doc_langs(document_id, source); +CREATE INDEX doc_langs_lang_idx ON doc_langs(document_id, lang); + +-- +goose Down + +-- The same rebuild in reverse, restoring 00002's narrower CHECK lists. Rows are +-- carried across rather than dropped; a row holding 'repertoire' makes this fail, +-- which is the honest outcome — there is nowhere for that value to go. +CREATE TABLE doc_pages_old ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + page_no INTEGER NOT NULL CHECK (page_no >= 1), + chars INTEGER NOT NULL DEFAULT 0 CHECK (chars >= 0), + script TEXT NOT NULL DEFAULT '', + page_tag TEXT NOT NULL DEFAULT '', + printed_folio INTEGER, + lang TEXT NOT NULL DEFAULT '', + lang_source TEXT NOT NULL DEFAULT '' + CHECK (lang_source IN ('', 'page-tag', 'index', 'script', 'detector', 'reconciled')), + PRIMARY KEY (document_id, page_no) +) STRICT; + +INSERT INTO doc_pages_old (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source) +SELECT document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source FROM doc_pages; + +DROP TABLE doc_pages; +ALTER TABLE doc_pages_old RENAME TO doc_pages; + +CREATE INDEX doc_pages_lang_idx ON doc_pages(document_id, lang); + +CREATE TABLE doc_langs_old ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + source TEXT NOT NULL + CHECK (source IN ('page-tag', 'index', 'script', 'detector', 'reconciled')), + pdf_start INTEGER NOT NULL CHECK (pdf_start >= 0), + pdf_end INTEGER NOT NULL CHECK (pdf_end >= 0), + code TEXT NOT NULL, + lang TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + printed_page INTEGER, + confidence REAL NOT NULL DEFAULT 0 CHECK (confidence BETWEEN 0 AND 1), + conflict INTEGER NOT NULL DEFAULT 0 CHECK (conflict IN (0, 1)), + note TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + PRIMARY KEY (document_id, source, code, pdf_start), + CHECK (pdf_end >= pdf_start) +) STRICT; + +INSERT INTO doc_langs_old (document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, confidence, conflict, note, created_at) +SELECT document_id, source, pdf_start, pdf_end, code, lang, title, printed_page, confidence, conflict, note, created_at FROM doc_langs; + +DROP TABLE doc_langs; +ALTER TABLE doc_langs_old RENAME TO doc_langs; + +CREATE INDEX doc_langs_source_idx ON doc_langs(document_id, source); +CREATE INDEX doc_langs_lang_idx ON doc_langs(document_id, lang); diff --git a/internal/db/migrations/00004_doc_regions.sql b/internal/db/migrations/00004_doc_regions.sql new file mode 100644 index 0000000..b633a81 --- /dev/null +++ b/internal/db/migrations/00004_doc_regions.sql @@ -0,0 +1,135 @@ +-- M1: store a language that occupies part of a page. +-- +-- internal/doc already computes regions (internal/doc/regions.go) and nothing +-- stores them, so the parallel-columns manual's five languages do not survive a +-- restart. This is the table that lets them. The contract, and what it +-- deliberately leaves unsolved, is docs/design/regions.md. +-- +-- Additive: a new table only, no rebuild. 00002 and 00003 are committed, and +-- editing either would diverge from any database already created from it — the +-- same reason 00003 exists as its own file rather than as a patch to 00002. +-- doc_pages stays exactly as it is: a page genuinely has one dominant script, one +-- printed folio and one tag position, and those are per page. What is not per page +-- is language, and that is what moves here. Widening doc_pages instead would have +-- made every existing column ambiguous about which part of the page it describes. + +-- +goose Up + +-- One language's territory on a page: the whole page where a manual runs its +-- languages in sequence, a box where it runs them in parallel columns. +-- +-- A whole-page region has no box in the sense that x0 = 0 and x1 = the page +-- width. That is the compatibility stance rather than a shortcut: a caller +-- clipping text to the box gets the whole page, so a page-at-a-time reader needs +-- no special case and no null check for "this one has no box". +CREATE TABLE doc_regions ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + + -- Which signal named this region, and '' when none could. + -- + -- '' is a real, reportable state and not a defect to reject. The column + -- manual's page 68 is a page of service addresses in six languages: no + -- printed tag, no usable alphabet, nothing that can name it honestly. Saying + -- "nothing established" beats guessing, and saying it with '' rather than NULL + -- means no caller has to null-check a column that is never meaningfully + -- absent. The other values are doc.Source, all six of them, including + -- 'repertoire' — which 00003 exists because 00002 omitted. + source TEXT NOT NULL + CHECK (source IN ('', 'page-tag', 'index', 'script', 'repertoire', 'detector', 'reconciled')), + + -- 1-based page number in the original PDF. + page INTEGER NOT NULL CHECK (page >= 1), + + -- The region's horizontal bounds, INTEGER although doc.Region carries + -- float64. Three reasons, in order of how much they cost to get wrong: + -- + -- 1. A float in a primary key requires two probes to produce bit-identical + -- floats before the upsert converges. Anything else inserts a second row a + -- hair to the left of the first and reports the page twice. + -- 2. The coordinate space is poppler's, 1.5x the PDF's points (108 dpi against + -- 72). One unit is therefore exactly one pixel of a `pdftoppm -r 108` + -- raster — which is how a stored box is checked against a render at all — + -- and sub-pixel precision says nothing about where a column begins. + -- 3. Rounding cannot merge two columns. Measured over all 169 columns of + -- testdata/fixtures/thomas-drybox-amfibia.json: the two closest x0 values + -- on any one page are 143 units apart, and the narrowest column in the + -- document is 122 units wide. Both are three orders of magnitude clear of + -- the half-unit that rounding can move an edge. That fixture records its + -- own ground-truth edges as integers for the same reason. + -- + -- Round, do not truncate: truncation biases every edge left by up to a unit, + -- and it biases x1 and x0 in the same direction, so a width stays right by + -- luck rather than by construction. + x0 INTEGER NOT NULL CHECK (x0 >= 0), + x1 INTEGER NOT NULL CHECK (x1 >= 0), + + -- code is the label as the document prints it, which need not be a valid tag: + -- real manuals print D, RUS, UA and KAZ. lang is that normalised to BCP-47, + -- empty when it could not be. Keeping both is what makes an unrecognised label + -- reportable instead of dropped — the same pairing as doc_langs. + code TEXT NOT NULL DEFAULT '', + lang TEXT NOT NULL DEFAULT '', + + -- Characters (runes, not bytes) of the text inside the box, and how many text + -- runs it holds. + -- + -- Characters are the unit of size that replaces pages, because a page holding + -- three languages cannot be a unit of anything — "48 of 560 pages" was always + -- a proxy. Runes rather than bytes because half a real manual is Cyrillic, + -- Greek, Hebrew, Arabic or CJK, where the same amount of writing runs about a + -- third more bytes. Runs are the density evidence: a region of five runs is + -- page furniture, whatever its area. + chars INTEGER NOT NULL DEFAULT 0 CHECK (chars >= 0), + runs INTEGER NOT NULL DEFAULT 0 CHECK (runs >= 0), + + -- Set when the region's printed tag and its alphabet disagreed. The note says + -- how, in checkable terms. Surfacing the disagreement is the requirement; + -- resolving it silently is what the design forbids. + conflict INTEGER NOT NULL DEFAULT 0 CHECK (conflict IN (0, 1)), + note TEXT NOT NULL DEFAULT '', + + created_at INTEGER NOT NULL, + + -- Key on GEOMETRY, not on the label. x0 is what tells the German left column + -- from the German right column; the code cannot, because under doc_langs' key + -- those two collide on the same page with the same code and the same source, + -- which is the concrete breakage this table exists to fix (regions.md, and + -- 00002:205 for the key that breaks). + -- + -- Natural, not surrogate, for the reason 00002's header sets out at length: a + -- probe job can run twice, so a second probe must converge on the same rows. + -- A ULID here would make it insert a parallel set instead. + PRIMARY KEY (document_id, source, page, x0), + + CHECK (x1 >= x0) +) STRICT; + +-- What the reader asks for: a page's regions, and a language's territory across +-- the document. +CREATE INDEX doc_regions_page_idx ON doc_regions(document_id, page); +CREATE INDEX doc_regions_lang_idx ON doc_regions(document_id, lang); + +-- WHY source IS IN THE KEY, AND WHY THAT MAKES THE WHOLESALE REPLACE +-- LOAD-BEARING RATHER THAN INCIDENTAL. Do not "optimise" the delete away. +-- +-- Unlike doc_langs, which stores every signal's separate view of the same +-- document side by side, internal/doc produces ONE resolved set of regions in +-- which source merely records which signal named each one. That attribution is +-- not stable across probes: a column named by its alphabet on one run can be +-- named by its printed tag on the next, because the tag reader's vocabulary comes +-- from the document's own contents table and that parse can improve. Same +-- document, same page, same x0, same column — different source. +-- +-- With source in the key, that region is a DIFFERENT row. An upsert alone would +-- therefore leave the old row behind at the same x0 and the page would report two +-- regions where the document has one. So SaveProbe deletes a document's regions +-- and rewrites them inside one transaction; the upsert stays as belt and braces +-- for a retry within a single probe. +-- +-- The alternative was to drop source from the key and carry it as a plain column. +-- That was rejected because geometry-plus-source is what regions.md specifies and +-- because it would silently discard the case where two signals genuinely describe +-- the same box — but the cost is this delete, and it is not optional. + +-- +goose Down +DROP TABLE doc_regions; diff --git a/internal/db/migrations/00005_doc_blocks.sql b/internal/db/migrations/00005_doc_blocks.sql new file mode 100644 index 0000000..1a67a79 --- /dev/null +++ b/internal/db/migrations/00005_doc_blocks.sql @@ -0,0 +1,284 @@ +-- M1: store what a conversion produced -- the readable blocks of a document, and +-- the pictures printed in it. +-- +-- internal/doc computes both and stores neither: blocks.go turns a region into +-- headings, paragraphs, list items and table cells, figures.go finds the +-- illustrations and renders each one. The contract, and the seven things it +-- deliberately does not solve, is docs/design/conversion.md. +-- +-- Additive: two new tables, no rebuild. 00002, 00003 and 00004 are committed, and +-- editing any of them would diverge from a database already created from it -- +-- the same reason 00003 exists as its own file rather than as a patch to 00002. +-- +-- WHY TWO TABLES AND NOT ONE. BlockFigure is a declared kind in blocks.go and +-- nothing emits it, which invites folding figures into doc_blocks as rows of that +-- kind. They are not the same thing and the difference is in the key. A block +-- belongs to a REGION -- to one language's territory on a page -- and is keyed by +-- the region's left edge. A figure belongs to the PAGE: conversion.md settles that +-- "a picture that belongs to no language belongs to every language", so a figure +-- has no region and no language, and giving it a region_x0 would be inventing the +-- one fact the contract says it does not have. Storing them together would mean +-- either a nullable region_x0 in a primary key, which SQLite treats as never +-- conflicting and so would break the upsert outright, or a sentinel that reads as +-- a real column position. + +-- +goose Up + +-- One piece of readable content: a heading, a paragraph, a list item, a table +-- cell. In reading order within its region. +CREATE TABLE doc_blocks ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + + -- 1-based page number in the original PDF, the same number doc_regions and + -- doc_pages count in. Not the folio the paper prints, which is a different + -- thing and lives in doc_pages.printed_folio. + page INTEGER NOT NULL CHECK (page >= 1), + + -- The left edge of the region this block was read from, and the reason it is + -- INTEGER although doc.Block carries float64 is 00004's reason, unchanged: + -- + -- 1. A float in a primary key requires two conversions to produce + -- bit-identical floats before the upsert converges. Anything else inserts + -- a parallel set of blocks a hair to the left of the first, and the page + -- reads twice. + -- 2. It must be the SAME NUMBER as doc_regions.x0 or a block cannot be traced + -- back to the language territory it came from. doc_regions stores a + -- rounded integer; storing a float here would put the join one rounding + -- apart from the only row it can join to. registry.roundCoord is used for + -- both, so this is one function's output stored in two places rather than + -- two functions that agree today. + -- 3. Rounding cannot merge two regions: measured over all 169 columns of + -- testdata/fixtures/thomas-drybox-amfibia.json, the two closest x0 values + -- on any one page are 143 units apart. Half a unit is three orders of + -- magnitude clear of that. + -- + -- 0 for a whole-page region, which is what regions.md rule 3 stores for a page + -- whose columns are all one language. + region_x0 INTEGER NOT NULL CHECK (region_x0 >= 0), + + -- The block's position within its region, from 0, in reading order. + -- + -- Named idx rather than index because INDEX is a SQLite keyword: it parses + -- when quoted and nowhere else, and every query touching it would have to + -- remember the quotes. + idx INTEGER NOT NULL CHECK (idx >= 0), + + -- What the block is. A string rather than an integer for the reason + -- doc.BlockKind is one: "heading" survives a schema change and 1 does not. + -- + -- All five of blocks.go's kinds are listed, including 'figure', which nothing + -- emits yet -- it is declared there precisely so the vocabulary a database + -- column and a reader see does not have to change when that work lands. + -- Omitting it here would put the schema change back exactly where the Go + -- constant was written to avoid it. + kind TEXT NOT NULL + CHECK (kind IN ('heading', 'paragraph', 'list-item', 'table', 'figure')), + + -- The heading level, and 0 for anything that is not a heading. + -- + -- Only 1 and 2 are reachable today and the CHECK is deliberately wider than + -- that. blocks.go derives the level from one region's own body face and says + -- so: a document-wide outline needs every region's sizes ranked together, and + -- the columns manual has four heading sizes, so level 3 and 4 are real and + -- merely not yet computed. A CHECK of (0, 1, 2) would make that later pass a + -- migration. + level INTEGER NOT NULL DEFAULT 0 CHECK (level >= 0), + + -- The content, with the printed line breaks already removed: a break at the + -- original measure is a property of the paper's column width, not of the text. + text TEXT NOT NULL DEFAULT '', + + -- The region's language, empty where none was established -- '' rather than + -- NULL for doc_regions' reason, that a caller must not have to null-check a + -- column which is never meaningfully absent. Denormalised from doc_regions on + -- purpose: a block is self-describing once it leaves the page, which is the + -- state extraction and search will see it in, and it is what lets the reader + -- set dir="rtl" from the block alone. + lang TEXT NOT NULL DEFAULT '', + + -- The block's OWN bounding box, not the region's, in the space + -- doc.ExtractRuns reports: poppler's, 1.5x the PDF's points, where one unit is + -- one pixel of a pdftoppm -r 108 raster. + -- + -- REAL here where region_x0 above is INTEGER, and the difference is entirely + -- the primary key. Rounding region_x0 buys convergence and a joinable value; + -- rounding these buys nothing, because nothing keys on them and nothing joins + -- to them. What it would cost is the ability to compare a block's box against + -- a doc_figures rect, which is REAL for the same reason, without one side + -- having been quantised first. So they are stored exactly as internal/doc + -- reported them. + -- + -- Not constrained to be non-negative. A run parked off the left edge of the + -- page is furniture rather than an error, and clamping it here would move a + -- box that a caller is about to draw on a render. + x0 REAL NOT NULL, + x1 REAL NOT NULL, + y0 REAL NOT NULL, + y1 REAL NOT NULL, + + -- How many printed lines the block was folded from, and its rune count. + -- + -- Runes rather than bytes for the reason doc_regions.chars gives: half a real + -- manual is Cyrillic, Greek, Hebrew, Arabic or CJK, where the same amount of + -- writing runs about a third more bytes. Lines is the folding evidence -- a + -- 40-line "paragraph" is a paragraph break that was missed, and conversion.md + -- records that some are. + lines INTEGER NOT NULL DEFAULT 0 CHECK (lines >= 0), + chars INTEGER NOT NULL DEFAULT 0 CHECK (chars >= 0), + + -- Why this block is the kind it is, in checkable terms, and for a table cell + -- its place in the grid. Same stance as doc_regions.note: the evidence is + -- countable and a reader can hold it against the page. + note TEXT NOT NULL DEFAULT '', + + created_at INTEGER NOT NULL, + + -- The natural key conversion.md specifies: the document, the page, the + -- region's left edge and the block's index within it. Natural and not a + -- surrogate for 00002's reason -- a job handler can run twice, and a ULID here + -- would make the second conversion insert a parallel set instead of converging + -- on the first. + -- + -- It is also the citation extraction needs. "Paragraph 4 of the German region + -- of page 62" is a reference that survives a re-convert, which is what + -- ingest.md asks for when it says extraction must cite a paragraph rather than + -- a document. + -- + -- NOTE WHAT IS NOT IN THE KEY: kind and lang. Unlike doc_regions, which puts + -- source in its key, nothing about a block's classification identifies it. A + -- paragraph that a better heading rule promotes to a heading is the SAME block + -- at the same index, so it must update in place rather than become a second + -- row. That is the one way this key is simpler than doc_regions', and it is + -- deliberate. + PRIMARY KEY (document_id, page, region_x0, idx), + + CHECK (x1 >= x0), + CHECK (y1 >= y0) +) STRICT; + +-- What the reader asks for: a document's blocks in order, and one language's +-- content across the document. The second is the funnel's own query -- a household +-- that reads German asks for the German, not for the pages. +CREATE INDEX doc_blocks_page_idx ON doc_blocks(document_id, page); +CREATE INDEX doc_blocks_lang_idx ON doc_blocks(document_id, lang); + +-- One illustration printed on a page: where it is, the evidence that it is a +-- picture, and the digest of the PNG it was rendered to. +-- +-- Every illustration in both fixtures is VECTOR, which conversion.md measures: +-- pdfimages -- the obvious tool -- yields zero illustrations across all 628 pages +-- of both manuals, and what it does yield is 1,358 gradient-mesh slivers, a corner +-- logo and some CE marks. So a figure is found from what the page draws and its +-- bytes come from rendering the crop, and that is why this table stores a digest +-- rather than pointing at an embedded image. +CREATE TABLE doc_figures ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + + page INTEGER NOT NULL CHECK (page >= 1), + + -- The figure's position in the page's reading order, from 0, down then across. + -- Not a document-wide figure number: internal/doc has one page in view at a + -- time and numbering across pages is a later caller's. + idx INTEGER NOT NULL CHECK (idx >= 0), + + -- The rectangle, in the same 1.5-scaled space as doc_blocks' box, and REAL for + -- the same reason: it is not in the key, so there is nothing to gain by + -- quantising it and a caller placing a figure in a column's reading order + -- compares it against a block box that was not quantised either. + x0 REAL NOT NULL, + y0 REAL NOT NULL, + x1 REAL NOT NULL, + y1 REAL NOT NULL, + + -- How many drawn shapes the figure holds: the shape guard's evidence, kept + -- rather than reduced to the verdict, so a page that was rejected can be shown + -- to have been rejected for the right reason. + ink INTEGER NOT NULL DEFAULT 0 CHECK (ink >= 0), + + -- How much of the figure's area is covered by text: the text guard's evidence, + -- and the only thing separating a picture from a framed illustration grid or a + -- table. A stored figure has passed the guard at 0.15. + -- + -- Checked as >= 0 and deliberately NOT as <= 1. doc.textFraction sums the area + -- of every run overlapping the box without subtracting the overlaps between + -- runs, so two runs sharing a line can exceed the box's own area. An upper + -- bound of 1 would be asserting something the arithmetic does not promise, and + -- it would fail in a background job rather than anywhere a person is looking. + text_fraction REAL NOT NULL DEFAULT 0 CHECK (text_fraction >= 0), + + -- What the render was: 216 dpi today, twice the 108 the geometry is in. + -- Recorded rather than assumed constant, because a stored figure outlives the + -- constant that produced it and a caller scaling the pixels back onto the page + -- needs to know which one it got. + dpi INTEGER NOT NULL CHECK (dpi > 0), + + -- The PNG's pixel size, read back out of its IHDR rather than computed, so a + -- caller comparing it against the rect is comparing what poppler did against + -- what was asked for. + pixel_width INTEGER NOT NULL CHECK (pixel_width > 0), + pixel_height INTEGER NOT NULL CHECK (pixel_height > 0), + + -- THE PNG BYTES ARE NOT HERE. They go to the content-addressed blob store, + -- whose filename IS the SHA-256, and this column is that name -- referencing + -- blobs(sha256) exactly as documents.blob_sha256 does. + -- + -- Two reasons, and the second is the one that decides it. A row per figure of + -- a few hundred KB would put tens of MB of image into a database that is + -- opened, WAL-checkpointed and backed up as one file; the columns manual's + -- largest single figure is 353 KB. And content addressing already does the + -- deduplication this table would otherwise need: the same diagram printed in + -- five languages' sections renders to the same bytes and is stored once, + -- because doc.renderFigure digests exactly what poppler wrote rather than + -- re-encoding it. + -- + -- No ON DELETE clause, matching documents.blob_sha256: a blob outlives the + -- rows that point at it and is collected by counting references, because two + -- documents can legitimately share one. + -- + -- The length CHECK is what stops an unrendered figure being stored. A figure + -- found by doc.FindFigures carries no bytes and an empty digest, and '' would + -- otherwise have to be a blobs row for the FK to hold. + blob_sha256 TEXT NOT NULL REFERENCES blobs(sha256) + CHECK (length(blob_sha256) = 64), + + created_at INTEGER NOT NULL, + + -- Natural, for doc_blocks' reason. No region and no language in the key: a + -- figure belongs to the page, because conversion.md settles that a picture + -- belonging to no language belongs to every language. A reader scoped to one + -- language selects the figures of the pages that language occupies; it does + -- not select figures BY language, and there is no column here to let it try. + PRIMARY KEY (document_id, page, idx), + + CHECK (x1 >= x0), + CHECK (y1 >= y0) +) STRICT; + +CREATE INDEX doc_figures_page_idx ON doc_figures(document_id, page); +CREATE INDEX doc_figures_blob_idx ON doc_figures(blob_sha256); + +-- WHY THE WHOLESALE REPLACE IS LOAD-BEARING HERE TOO, FOR A DIFFERENT REASON THAN +-- 00004'S. Do not "optimise" the delete away. +-- +-- doc_regions needs its delete because source is in its key and a region's +-- attribution changes between probes, so an upsert leaves a superseded row at the +-- same x0. Neither key here carries anything that unstable -- that is what the +-- note above the doc_blocks primary key is about. The reason is simpler and it is +-- not weaker: A RE-CONVERSION CAN PRODUCE FEWER BLOCKS THAN THE ONE BEFORE IT. +-- +-- It routinely will. Every threshold in blocks.go is one measurement away from +-- moving, and most of them merge: paragraphGapFactor folds two paragraphs into +-- one, the heading share cut turns two headings into one, and conversion.md +-- already names four unresolved cases where the count should fall. Indices are +-- consecutive from 0 within a region, so a region that converted to 12 blocks and +-- now converts to 9 leaves rows at idx 9, 10 and 11 -- a reader shows three +-- paragraphs of the previous run's text, in order, indistinguishable from content. +-- The same holds for a figure the trim rule now rejects. +-- +-- So SaveConversion deletes a document's blocks and figures and rewrites them +-- inside one transaction. The upserts stay as belt and braces for a retry within a +-- single conversion. + +-- +goose Down +DROP TABLE doc_figures; +DROP TABLE doc_blocks; diff --git a/internal/db/migrations/00006_block_search.sql b/internal/db/migrations/00006_block_search.sql new file mode 100644 index 0000000..c4d0d59 --- /dev/null +++ b/internal/db/migrations/00006_block_search.sql @@ -0,0 +1,236 @@ +-- M1: make a household's manuals searchable, which is the first problem README +-- claims this project solves -- "the paper pile is unsearchable, and you need the +-- router manual at exactly the moment the internet is down". +-- +-- 00005 stores the blocks. This indexes them. The contract is +-- docs/design/search.md; every number below was measured against both real +-- manuals, 3,122 blocks converted for de, ru, ja, th and he, and the harness that +-- produced them is in that document's own history. +-- +-- Additive: one virtual table, three triggers, and a rebuild. No existing table is +-- touched, for the reason 00004 and 00005 give -- 00002 through 00005 are +-- committed and editing any of them would diverge from a database already created +-- from it. +-- +-- +-- ONE: EXTERNAL CONTENT, NOT A STANDALONE INDEX. +-- +-- content='doc_blocks' means FTS5 stores only the index and reads the text back out +-- of doc_blocks when a query needs it. A standalone table stores its own copy of +-- every block's text, which is the simpler thing and doubles the text on disk. +-- Measured over the 3,122-block corpus, all with 'optimize' then VACUUM run and +-- the whole database file compared against one holding doc_blocks alone (626,688 +-- bytes): +-- +-- standalone unicode61 1,388,544 total +761,856 index +-- external unicode61 897,024 total +270,336 index +-- standalone trigram 1,998,848 total +1,372,160 index +-- external trigram 1,507,328 total +880,640 index +-- +-- The duplicated text is the 491,520-byte difference in both pairs, which is 56% +-- more index for the trigram pair and nothing gained. So: external content. +-- +-- WHAT EXTERNAL CONTENT COSTS, AND WHY IT IS PAID IN TRIGGERS. An external content +-- table is NOT maintained by SQLite. Nothing updates it when doc_blocks changes, +-- and a delete has to be told the OLD text, because FTS5 no longer has a copy to +-- work out which terms to remove. +-- +-- Triggers rather than Go statements next to each write, and this is the decision +-- the correctness of the whole feature rests on. There are three paths that change +-- doc_blocks and only two of them are visible in Go: +-- +-- 1. registry.saveBlocks deletes a document's blocks and reinserts them -- the +-- wholesale replace 00005 explains at length. +-- 2. The same function's upsert updates a block in place when the key is unchanged. +-- 3. documents ON DELETE CASCADE removes every block of a deleted document, and +-- NO GO CODE RUNS AT ALL. Nothing calls DeleteDocBlocks and no handler is +-- involved: deleting a device removes its documents, which removes their +-- blocks, entirely inside SQLite. A rule that had to be remembered in Go on +-- that path would be a rule nobody can see in the source they are editing. +-- +-- Triggers cover all three by construction, and they run inside whatever +-- transaction the write is already in, which is exactly what SaveConversion needs: +-- the blocks, the index and the document's 'ready' state commit together or not at +-- all. +-- +-- That the cascade fires them was MEASURED rather than assumed, because SQLite's +-- own documentation makes trigger firing on a foreign-key action conditional on +-- the recursive_triggers setting, and manualbox does not set it. With +-- foreign_keys(1) and recursive_triggers off -- the pragmas internal/db actually +-- opens with -- deleting the document removed its rows from the index and FTS5's +-- 'integrity-check' passed. +-- +-- WHAT GOES WRONG WITHOUT THE DELETE TRIGGER IS NOT WHAT IT LOOKS LIKE, and the +-- first version of this comment had it wrong. It is NOT that a deleted manual stays +-- findable: every search joins the index to doc_blocks, so an index entry whose row +-- is gone joins to nothing and vanishes from the results by accident. Measured that +-- way round, and it made the obvious control assertion pass over a corrupt index. +-- +-- The real failure is worse and needs one more step. SQLite gives a new row +-- max(rowid)+1, so deleting the highest block frees a rowid the next insert takes. +-- The stale entry then points at a REAL row of a DIFFERENT document, and searching +-- for a word from the deleted manual returns a confident hit naming another manual, +-- another page, and text that does not contain the word. A wrong citation rather +-- than a missing one, which is the failure this project can least afford, since a +-- citation is what extraction will hang a maintenance schedule on. +-- +-- Measured end to end, including the reuse: dropping the delete trigger makes +-- 'integrity-check' report "database disk image is malformed" immediately, and the +-- next insert makes a search for the deleted manual's word answer with the +-- unrelated document's text. revertCheckTheDeleteTrigger in internal/db is that +-- run, kept as a test. +-- +-- ROWID STABILITY, THE ONE RISK THIS SHAPE CARRIES. An external content index joins +-- on doc_blocks' rowid, and doc_blocks' primary key is composite, so its rowid is +-- not an INTEGER PRIMARY KEY alias -- the kind of rowid SQLite does not promise to +-- preserve across a VACUUM. Nothing in manualbox runs VACUUM (grepped, not +-- assumed), and a VACUUM of a 3,122-block database with holes punched in its rowid +-- sequence was measured to leave max(rowid) and every hit unchanged with +-- 'integrity-check' passing. It is still not a promise: if a database is ever +-- vacuumed by hand and search starts returning the wrong block, the repair is +-- `INSERT INTO doc_blocks_fts(doc_blocks_fts) VALUES ('rebuild')`, which is the +-- same statement this migration ends with. +-- +-- +-- TWO: THE TOKENISER, WHICH IS THE ONE DECISION THAT COULD NOT BE REASONED OUT. +-- +-- unicode61, FTS5's default, splits on whitespace and punctuation. That is right +-- for German, Russian, Ukrainian and Greek and it is USELESS for Chinese, Japanese +-- and Thai, which do not put spaces between words. trigram indexes every run of +-- three characters and therefore matches substrings, which works for those scripts +-- and costs a larger index and a three-character minimum. +-- +-- Measured, not chosen from that description. Real words from each script, against +-- the real corpus: +-- +-- query unicode61 trigram +-- "Filter" (de) 21 69 +-- "Saugkraft" (de) 7 7 +-- "Gerat" (de, folded) 71 96 +-- Russian "filter" 31 96 +-- Japanese "instruction manual" 0 6 +-- Thai "manual" 0 6 +-- +-- unicode61 finds NOTHING in Japanese and NOTHING in Thai. It is not degraded +-- there, it is absent: a whole CJK or Thai run is one token, so it matches only a +-- query that happens to be the entire run. (The two-character Japanese word for +-- "power" scores 2 hits under unicode61 against 27 real occurrences, and those 2 +-- are where punctuation happened to isolate it. That is the shape of the failure.) +-- +-- trigram finds a real word in all five scripts, and it costs 880,640 bytes of +-- index against unicode61's 270,336 -- 3.3x, or 2.40x the size of a database +-- holding the blocks alone, which is 195 bytes of index per stored block. The +-- higher hit counts are substring matches: "Filter" also finds "Luftfilter" and +-- "Filterdeckel", which in German is closer to what a person meant than +-- token-exact matching is. +-- +-- SO: trigram, ONE INDEX FOR EVERY SCRIPT, with one named limitation. +-- +-- THE LIMITATION: A QUERY SHORTER THAN THREE CHARACTERS MATCHES NOTHING. Not fewer +-- results -- none, because there is no such token in the index. Measured: the +-- two-character Japanese words for "power" and "product" occur in 27 and 24 stored +-- blocks and the index finds 0 of each. Two characters is an ordinary word in +-- Chinese and Japanese, so this is a real hole in exactly the scripts trigram was +-- chosen for. queries/search.sql fills it by scanning instead for a query that +-- short, measured at 1.9 ms over this corpus, and the API reports which path +-- answered. +-- +-- WHAT WAS REJECTED, AND WHAT IT WOULD HAVE COST. Two indexes -- unicode61 for the +-- space-separated scripts and trigram for the rest -- would give token-exact +-- precision to the majority of languages and still serve CJK. It was rejected: it +-- costs 1,150,976 bytes of index rather than 880,640, both must be maintained by +-- their own triggers, and every query has to guess from the query's own characters +-- which index can answer it. A query mixing a German word and a Japanese one then +-- has no right answer. One index that is somewhat blunt everywhere beats two that +-- are sharp until a household is multilingual, which every household with this +-- kind of manual already is. +-- +-- +-- THREE: DIACRITICS ARE FOLDED, AND THE STATED COST TURNED OUT NOT TO EXIST. +-- +-- remove_diacritics is what lets a German household on any keyboard find "Gerat" +-- and get "Gerat". The worry was that it also folds Cyrillic and Greek, which would +-- be a real cost on this corpus -- half of it is not Latin. +-- +-- MEASURED: IT FOLDS LATIN AND NOTHING ELSE. Stored against queried, across all +-- three modes of unicode61 and both modes of trigram: +-- +-- German "Gerat" for stored "Gerat" folded when on, missed when off +-- Russian "esche" for stored "eschyo" NEVER folded (yo stays yo) +-- Ukrainian "Kyiv" with i for yi NEVER folded +-- Greek "odigies" for stored "odigies" NEVER folded (tonos stays) +-- Hebrew without niqqud for stored with NEVER folded +-- +-- FTS5's folding table covers precomposed Latin and does not reach Cyrillic, Greek +-- or Hebrew, so the cost this decision was weighed against is not there. It is +-- turned ON, and it has to be said explicitly here: unicode61 folds by default but +-- TRIGRAM DOES NOT, and with it off "Gerat" finds 0 of the 96 blocks holding +-- "Gerat". The index is 4,096 bytes SMALLER with folding on. +-- +-- WHAT IS NOT FIXED BY ANY OF THIS, and it is worth knowing before someone tests +-- with Hebrew. The stored Hebrew of the sequential manual is in VISUAL order -- +-- internal/doc reads the runs a right-to-left page paints, and the PDF paints them +-- reversed. The word for "manual" is stored as its own reverse, so it is findable +-- by a query typed backwards (5 blocks) and not by one a Hebrew speaker would type +-- (0 blocks). No tokeniser touches that; it is upstream of the index, in +-- extraction, and it belongs to internal/doc rather than here. + +-- +goose Up + +-- The index over every stored block's text. One indexed column, because a block's +-- other columns are how a hit is described rather than what is searched: matching +-- on the language code or the kind would let a query for "table" find every table. +CREATE VIRTUAL TABLE doc_blocks_fts USING fts5( + text, + content='doc_blocks', + content_rowid='rowid', + tokenize='trigram remove_diacritics 1' +); + +-- The three triggers that keep it correct. See the header: these are the whole +-- maintenance story, including for the ON DELETE CASCADE from documents, which no +-- Go code observes. +-- +-- Each needs goose's StatementBegin/StatementEnd, because a trigger body contains +-- semicolons and goose otherwise cuts the statement at the first one. + +-- +goose StatementBegin +CREATE TRIGGER doc_blocks_fts_insert AFTER INSERT ON doc_blocks BEGIN + INSERT INTO doc_blocks_fts(rowid, text) VALUES (new.rowid, new.text); +END; +-- +goose StatementEnd + +-- The 'delete' command has to be given the OLD text, not just the rowid: FTS5 has +-- no copy of it to work out which terms to remove, which is exactly what external +-- content means. +-- +goose StatementBegin +CREATE TRIGGER doc_blocks_fts_delete AFTER DELETE ON doc_blocks BEGIN + INSERT INTO doc_blocks_fts(doc_blocks_fts, rowid, text) + VALUES ('delete', old.rowid, old.text); +END; +-- +goose StatementEnd + +-- An update is a delete of the old terms and an insert of the new ones. It fires on +-- the upsert path in registry.saveBlocks, which updates a block in place when a +-- re-conversion produces the same key with different text -- a paragraph promoted +-- to a heading, or the same paragraph folded differently. +-- +goose StatementBegin +CREATE TRIGGER doc_blocks_fts_update AFTER UPDATE ON doc_blocks BEGIN + INSERT INTO doc_blocks_fts(doc_blocks_fts, rowid, text) + VALUES ('delete', old.rowid, old.text); + INSERT INTO doc_blocks_fts(rowid, text) VALUES (new.rowid, new.text); +END; +-- +goose StatementEnd + +-- Blocks that are already stored. 00005 shipped, so a database reaching this +-- migration can already hold a converted manual, and the triggers above only see +-- what happens next. Without this, an existing household would have to re-approve +-- every document to become searchable. 'rebuild' costs one pass over doc_blocks and +-- is a no-op on a fresh database. +INSERT INTO doc_blocks_fts(doc_blocks_fts) VALUES ('rebuild'); + +-- +goose Down +DROP TRIGGER doc_blocks_fts_update; +DROP TRIGGER doc_blocks_fts_delete; +DROP TRIGGER doc_blocks_fts_insert; +DROP TABLE doc_blocks_fts; diff --git a/internal/db/migrations/00007_neutral_pages.sql b/internal/db/migrations/00007_neutral_pages.sql new file mode 100644 index 0000000..519f464 --- /dev/null +++ b/internal/db/migrations/00007_neutral_pages.sql @@ -0,0 +1,65 @@ +-- M1: the pages no language owns, offered at the gate as an extra scope. +-- +-- A reader of the sequential fixture cannot see its exploded parts diagram. The +-- plate is on PDF page 5, its four sub-drawings ARE found by the figure pass, 31 +-- places in the content pages say "see A-1", and page 5 falls inside no language +-- region -- so no conversion has ever reached it. docs/design/conversion.md records +-- both the measurement and the user's intended answer: let the reader choose those +-- pages rather than guess at a facing-page rule. +-- +-- Two columns, and they answer two different questions. +-- +-- ADDITIVE, NO REBUILD, which matters more here than usual. doc_blocks is the +-- external-content source of the FTS5 table 00006 builds, and 00005's header +-- records that widening anything in it costs dropping three triggers and +-- reindexing. Neither column below is on doc_blocks, and neither existing table is +-- rewritten, so 00006's triggers are untouched and no reindex happens. ALTER TABLE +-- ADD COLUMN is legal on a STRICT table; NOT NULL needs a non-null DEFAULT, which +-- is why the second column has one and the first, being nullable, must not. + +-- +goose Up + +-- How many pictures a page holds, and NULL when nobody counted. +-- +-- NULL AND 0 ARE DIFFERENT ANSWERS, which is why this is nullable in a schema whose +-- house style is '' over NULL. Counting a page's drawings is one pdftocairo spawn -- +-- 42.3 s over the sequential manual's 560 pages, measured in doc.pageRegionsWithTables +-- for the same reason -- so the probe counts only the pages no named region claims: +-- 7 of 560 and 2 of 68 on the fixtures. Every other row is NULL for ever. +-- +-- A 0 default would tell the gate that page 5 holds no pictures, which is the exact +-- inverse of the truth and the mistake registry.FolioOffset already records under +-- "absent, never 0". Every row written before this migration is NULL, correctly: +-- nobody counted them. +ALTER TABLE doc_pages ADD COLUMN figures INTEGER; + +-- Whether the user, at the gate, asked for the pages no language owns as well. +-- +-- WHY THIS IS A COLUMN AND NOT A REQUEST FIELD ON approve, OR A JOB PAYLOAD FIELD. +-- +-- ingest.Approve deliberately takes no scope argument: the gate rendered the +-- household's languages out of configuration and told the user what converting them +-- would involve, so a scope arriving in a request body could differ from the scope +-- the user was shown. That promise has to survive this feature, and the neutral set +-- is not configuration -- it is a per-document choice, because one manual's unowned +-- pages are diagram plates and another's are a page of service addresses. +-- +-- Storing the decision on the document reconciles the two. The endpoint writes it +-- here in the same transaction as the state change; doc.convert reads it back from +-- this row exactly as it reads the household from configuration. So the handler +-- still takes its whole scope from stored state and never from a caller, the +-- ConvertPayload stays document-only, and the dedupe key stays the document -- which +-- a flag in the payload would have broken, since approving false then true would +-- have deduped onto the first job and silently converted the wrong scope. +-- +-- It also makes the decision durable: re-running the conversion converges on the +-- same scope instead of quietly reverting to the smaller one. +-- +-- 0 for every existing row, which is today's behaviour exactly. +ALTER TABLE documents ADD COLUMN include_neutral_pages INTEGER NOT NULL DEFAULT 0 + CHECK (include_neutral_pages IN (0, 1)); + +-- +goose Down + +ALTER TABLE documents DROP COLUMN include_neutral_pages; +ALTER TABLE doc_pages DROP COLUMN figures; diff --git a/internal/db/migrations/00008_doc_figure_labels.sql b/internal/db/migrations/00008_doc_figure_labels.sql new file mode 100644 index 0000000..a322b79 --- /dev/null +++ b/internal/db/migrations/00008_doc_figure_labels.sql @@ -0,0 +1,108 @@ +-- M1: a figure's callout labels, as text with a position, so the reader can draw +-- them beside the picture instead of relying on the crop to contain them. +-- +-- A crop is a rectangle and a diagram's labels are not arranged in one. doc.growToLabels +-- could only reach a label by widening the crop over everything between, so page 521 of +-- the sequential fixture held 23 of its 34 labels whole and 41 of that document's +-- figures held 88 labels their crops never reached. docs/design/conversion.md recorded +-- the complete answer and this is its storage half: keep each label as a string with a +-- position and let the reader place it. +-- +-- ADDITIVE, ONE NEW TABLE, NO REBUILD -- and that is the constraint that shaped it +-- rather than a happy accident. +-- +-- The obvious alternative was a sixth doc_blocks kind, and it is exactly what +-- 00005's header and the contents-entry note in conversion.md forbid: doc_blocks is +-- the external-content source of the FTS5 table 00006 builds, its `kind` column +-- carries a CHECK listing the five by name, and widening a closed set there costs +-- dropping three triggers and reindexing the search table. 00003 is the precedent and +-- records that procedure. NOTHING BELOW TOUCHES doc_blocks. No existing table is +-- rewritten, no trigger is dropped and no reindex happens. +-- +-- A label is not a block for a better reason than the cost, though. A block has a place +-- in reading order and a label does not: it belongs beside a picture, which is not a +-- position in the prose. doc.Block.Callout marks the runs that left the flow so +-- verify.checkCoverage can still account for every character, and those blocks are +-- filtered at the save boundary exactly as page furniture is -- so this table, not +-- doc_blocks, is the only place a label is stored. + +-- +goose Up + +CREATE TABLE doc_figure_labels ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + + -- The figure this labels, by its natural key. doc_figures is keyed + -- (document_id, page, idx), so these three columns name one picture. + page INTEGER NOT NULL CHECK (page >= 1), + figure_idx INTEGER NOT NULL CHECK (figure_idx >= 0), + + -- The label's position among its figure's labels, from 0, ordered by side then + -- down then across. Not a printed callout number: the paper's "12" is in text + -- below, and doc.figureLabels has one page in view and never reads a parts list. + idx INTEGER NOT NULL CHECK (idx >= 0), + + -- The label as printed, trimmed. Stored in LOGICAL order like every other string + -- here, which internal/doc/bidi.go is responsible for and search.md measured: the + -- sequential manual has a Hebrew and an Arabic section and their labels are + -- claimed like any other. + -- + -- '' is rejected rather than stored. An empty label is a claim that produced no + -- text, which is a defect in the claim rule and not a label a reader can be shown. + text TEXT NOT NULL CHECK (length(text) > 0), + + -- Which edge of the figure the label sits beyond: 0 left, 1 right, 2 top, 3 + -- bottom, matching doc's edgeLeft..edgeBottom. PHYSICAL and not logical -- a label + -- printed left of a drawing is left of it on a right-to-left page too, because the + -- picture is a picture and mirroring it would move the label off the part it names. + -- + -- The CHECK is a closed set of four and will never widen: a rectangle has four + -- edges. That is the opposite of doc_blocks.kind, whose set is a reading decision, + -- and it is why a CHECK is safe here. + side INTEGER NOT NULL CHECK (side IN (0, 1, 2, 3)), + + -- The label's own box as FRACTIONS OF THE CROP, from the crop's top-left. + -- + -- ROUTINELY NEGATIVE OR GREATER THAN ONE, and there is deliberately no CHECK + -- bounding them to 0..1. A label sits OUTSIDE the picture -- that is the entire + -- reason it was being cropped away -- so a bound of 0..1 would reject every row + -- this table exists to hold. What IS checked is the only thing that is always + -- true of a box, that it is not inside out. + -- + -- Fractions rather than the 1.5-scaled units doc_figures and doc_blocks carry, and + -- this is the one place the house style is broken on purpose. Those rectangles are + -- compared against each other, so they have to share a space. A label is compared + -- against nothing: its only consumer draws it against a rendered image whose CSS + -- size it chose itself, and given fractions it multiplies by the width it drew and + -- needs neither the page box, the dpi, nor the crop's rectangle. REAL for + -- doc_figures' reason -- not in the key, so there is nothing to gain by quantising. + x0 REAL NOT NULL, + y0 REAL NOT NULL, + x1 REAL NOT NULL, + y1 REAL NOT NULL, + + created_at INTEGER NOT NULL, + + -- Natural, on doc_blocks' and doc_figures' reasoning: a label is identified by the + -- figure it belongs to and its place among that figure's labels, so a re-conversion + -- upserts onto the same row instead of appending a second copy. doc.figureLabels + -- sorts by side then down then across precisely so that this index is stable. + PRIMARY KEY (document_id, page, figure_idx, idx), + + -- Cascaded from the figure as well as from the document, so a figure a re-conversion + -- no longer finds cannot leave its labels behind. registry.saveFigures deletes a + -- document's figures and rewrites them, and without this the orphaned rows would + -- outlive the picture and be served against whatever figure later took that index. + FOREIGN KEY (document_id, page, figure_idx) + REFERENCES doc_figures(document_id, page, idx) ON DELETE CASCADE, + + CHECK (x1 >= x0), + CHECK (y1 >= y0) +) STRICT; + +-- The read path is "every label of this document's figures, in figure order", which +-- the primary key already serves. This index is for the other one: the conversion +-- response joins labels onto the figures of one page. +CREATE INDEX doc_figure_labels_page_idx ON doc_figure_labels(document_id, page); + +-- +goose Down +DROP TABLE doc_figure_labels; diff --git a/internal/db/migrations/00009_figure_labels_lose_their_boxes.sql b/internal/db/migrations/00009_figure_labels_lose_their_boxes.sql new file mode 100644 index 0000000..018c7a2 --- /dev/null +++ b/internal/db/migrations/00009_figure_labels_lose_their_boxes.sql @@ -0,0 +1,149 @@ +-- M1: a figure's callout labels keep their text and lose their geometry. +-- +-- 00008's header is now historically wrong and this is where that is recorded, since +-- a shipped migration is immutable. It said the labels were stored "as text with a +-- position, so the reader can draw them beside the picture instead of relying on the +-- crop to contain them", and that "a label sits OUTSIDE the picture -- that is the +-- entire reason it was being cropped away". Neither holds any more. The crop is a +-- BAND: the drawing together with every run the claim rule reaches for it, so the +-- rendered picture already prints its own labels, exactly where the paper prints +-- them. Nothing re-lays them out, so nothing needs to know where they are. +-- +-- Measured before committing to the band, over both fixture manuals, in runes of +-- prose the crop prints that the block flow also emits: 1.8% on the columns manual +-- and 9.8% on the sequential one, against 84.0% and 65.4% for the full-page band +-- that was the original plan. The parsed text survives for one reason only -- it is +-- the image's accessible description -- so what is left of a label is its string and +-- its place in the reading order of its figure's labels. +-- +-- WHAT GOES: side, x0, y0, x1, y1 and the two table CHECKs that bound the box. +-- WHAT STAYS: document_id, page, figure_idx, idx, text, created_at, the primary key, +-- both foreign keys and doc_figure_labels_page_idx. Every surviving column, type, +-- NOT NULL, CHECK, STRICT and cascade below is reproduced verbatim from 00008. +-- +-- Why a rebuild and not five DROP COLUMNs. `side` alone would in fact drop: its CHECK +-- is part of its own column definition and goes with it. The box columns will not, +-- because x1 >= x0 and y1 >= y0 are TABLE-level CHECKs, and SQLite refuses to drop a +-- column a constraint still names. Measured against this project's own driver +-- (modernc.org/sqlite v1.54.0, SQLite 3.53.3): dropping `side` succeeds and dropping +-- `x0` fails with "error in table doc_figure_labels after drop column: no such column: +-- x0". Even had it succeeded, the two CHECKs would remain and there is no ALTER that +-- removes a CHECK -- 00003 records that, and 00003 is the procedure followed here. +-- +-- Why this rebuild is safe. doc_figure_labels is a foreign-key LEAF: it references +-- documents(id) and doc_figures(document_id, page, idx), and NOTHING references it. So +-- the drop cannot orphan a child row and the rename cannot leave a dangling reference. +-- Both parents are untouched. goose runs this inside a transaction with +-- _pragma foreign_keys(1) set (internal/db/db.go), so a failure part-way leaves the old +-- table intact, and the copy is checked against both parents as it is inserted. +-- +-- EXISTING ROWS SURVIVE. This is a projection, not a truncation: every row is carried +-- across on its named surviving columns, and the demo database's 95 labels are the case +-- that was actually run. +-- +-- NOTHING BELOW TOUCHES doc_blocks -- 00008's constraint still holds, and the reader of +-- this file needs to know it was checked rather than forgotten. doc_blocks is the +-- external-content source of the FTS5 table 00006 builds; rewriting it would cost +-- dropping three triggers and reindexing the search table. No trigger is dropped here, +-- no reindex happens, and the search index is not touched at all. +-- +-- The natural key does not move, and it still identifies a row. idx is a label's place +-- among its figure's labels, so (document_id, page, figure_idx, idx) is unique by +-- construction: doc.figureLabels assigns it from a position in one sorted sequence. What +-- changed is the sort -- side then down then across has become down then across, because +-- there are no longer any sides to group by -- and a sort key changing does not make a +-- sequence index collide. Index 0 is now simply the topmost label. + +-- +goose Up + +CREATE TABLE doc_figure_labels_new ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + + -- The figure this labels, by its natural key. doc_figures is keyed + -- (document_id, page, idx), so these three columns name one picture. + page INTEGER NOT NULL CHECK (page >= 1), + figure_idx INTEGER NOT NULL CHECK (figure_idx >= 0), + + -- The label's position among its figure's labels, from 0, ordered down then + -- across, so 0 is the topmost. Not a printed callout number: the paper's "12" is + -- in text below, and doc.figureLabels has one page in view and never reads a + -- parts list. + idx INTEGER NOT NULL CHECK (idx >= 0), + + -- The label as printed, trimmed. Stored in LOGICAL order like every other string + -- here, which internal/doc/bidi.go is responsible for and search.md measured: the + -- sequential manual has a Hebrew and an Arabic section and their labels are + -- claimed like any other. + -- + -- '' is rejected rather than stored. An empty label is a claim that produced no + -- text, which is a defect in the claim rule and not a label a reader can be shown. + -- It is also, now, an empty accessible description, which is worse than none. + text TEXT NOT NULL CHECK (length(text) > 0), + + created_at INTEGER NOT NULL, + + -- Natural, on doc_blocks' and doc_figures' reasoning: a label is identified by the + -- figure it belongs to and its place among that figure's labels, so a re-conversion + -- upserts onto the same row instead of appending a second copy. + PRIMARY KEY (document_id, page, figure_idx, idx), + + -- Cascaded from the figure as well as from the document, so a figure a re-conversion + -- no longer finds cannot leave its labels behind. registry.saveFigures deletes a + -- document's figures and rewrites them, and without this the orphaned rows would + -- outlive the picture and be served against whatever figure later took that index. + FOREIGN KEY (document_id, page, figure_idx) + REFERENCES doc_figures(document_id, page, idx) ON DELETE CASCADE +) STRICT; + +-- Named columns on both sides, so a future column added to one table and not the +-- other fails loudly here instead of shifting values silently. +INSERT INTO doc_figure_labels_new (document_id, page, figure_idx, idx, text, created_at) +SELECT document_id, page, figure_idx, idx, text, created_at FROM doc_figure_labels; + +DROP TABLE doc_figure_labels; +ALTER TABLE doc_figure_labels_new RENAME TO doc_figure_labels; + +-- The index belongs to the dropped table, so it is recreated, not renamed. The read +-- path is "every label of this document's figures, in figure order", which the primary +-- key already serves; this is for the other one, where the conversion response joins +-- labels onto the figures of one page. +CREATE INDEX doc_figure_labels_page_idx ON doc_figure_labels(document_id, page); + +-- +goose Down + +-- The same rebuild in reverse, restoring 00008's table exactly. Rows are carried across +-- rather than dropped, which is the whole point of writing a real Down here -- a +-- downgrade must not silently delete a document's labels. +-- +-- THE GEOMETRY CANNOT BE RESTORED, because it was not kept anywhere: this is a lossy +-- migration and its inverse can only be honest about that. Every recovered row gets +-- side 0 and a zero-area box at the crop's top-left, which satisfies both CHECKs and +-- 00008's closed set of four sides. Those are placeholders, not measurements. A client +-- of the downgraded schema that draws labels by position would stack all of them in one +-- corner; the fix is to re-run the conversion, which recomputes geometry from the page. +CREATE TABLE doc_figure_labels_old ( + document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + page INTEGER NOT NULL CHECK (page >= 1), + figure_idx INTEGER NOT NULL CHECK (figure_idx >= 0), + idx INTEGER NOT NULL CHECK (idx >= 0), + text TEXT NOT NULL CHECK (length(text) > 0), + side INTEGER NOT NULL CHECK (side IN (0, 1, 2, 3)), + x0 REAL NOT NULL, + y0 REAL NOT NULL, + x1 REAL NOT NULL, + y1 REAL NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (document_id, page, figure_idx, idx), + FOREIGN KEY (document_id, page, figure_idx) + REFERENCES doc_figures(document_id, page, idx) ON DELETE CASCADE, + CHECK (x1 >= x0), + CHECK (y1 >= y0) +) STRICT; + +INSERT INTO doc_figure_labels_old (document_id, page, figure_idx, idx, text, side, x0, y0, x1, y1, created_at) +SELECT document_id, page, figure_idx, idx, text, 0, 0.0, 0.0, 0.0, 0.0, created_at FROM doc_figure_labels; + +DROP TABLE doc_figure_labels; +ALTER TABLE doc_figure_labels_old RENAME TO doc_figure_labels; + +CREATE INDEX doc_figure_labels_page_idx ON doc_figure_labels(document_id, page); diff --git a/internal/db/queries/devices.sql b/internal/db/queries/devices.sql new file mode 100644 index 0000000..ae47fcc --- /dev/null +++ b/internal/db/queries/devices.sql @@ -0,0 +1,29 @@ +-- name: CreateDevice :one +INSERT INTO devices (id, name, brand, model, category, location_id, notes, purchased_at, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +RETURNING *; + +-- name: GetDevice :one +SELECT * FROM devices WHERE id = ?; + +-- name: ListDevices :many +SELECT * FROM devices ORDER BY name; + +-- Filtering by location is a separate query rather than a nullable parameter on +-- ListDevices. CONTRIBUTING.md: an "IS NULL OR =" filter defeats sqlc's type +-- inference and reads worse than two explicit queries. +-- name: ListDevicesByLocation :many +SELECT * FROM devices WHERE location_id = ? ORDER BY name; + +-- name: UpdateDevice :one +UPDATE devices +SET name = ?, brand = ?, model = ?, category = ?, location_id = ?, notes = ?, + purchased_at = ?, updated_at = ? +WHERE id = ? +RETURNING *; + +-- name: DeleteDevice :exec +DELETE FROM devices WHERE id = ?; + +-- name: CountDevices :one +SELECT CAST(count(*) AS INTEGER) AS total FROM devices; diff --git a/internal/db/queries/docblocks.sql b/internal/db/queries/docblocks.sql new file mode 100644 index 0000000..f3c9a51 --- /dev/null +++ b/internal/db/queries/docblocks.sql @@ -0,0 +1,172 @@ +-- Queries over doc_blocks and doc_figures: what a conversion produced. See +-- 00005_doc_blocks.sql for the schema's reasoning and docs/design/conversion.md +-- for the contract. +-- +-- THIS FILE MUST STAY PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 +-- (pinned in tools/go.mod) mixes up character and byte offsets when it cuts +-- statements out of a file, so one non-ASCII character anywhere above corrupts +-- every statement after it -- silently, in the dangerous case: `make sqlc` exits +-- 0, the Go compiles, the linter passes, and the statement fails at PREPARE time +-- inside a background job against a user's database. The full measurement is in +-- the header of docregions.sql; TestQueryFilesAreASCII is the cause-side guard +-- and TestDocBlockQueriesExecute the symptom-side one. +-- +-- Columns are listed explicitly rather than with SELECT *, so that adding a +-- column later cannot silently change every caller's row shape. + +-- Upsert on the natural key (document_id, page, region_x0, idx), because a +-- conversion job may run twice and must converge on the same rows rather than +-- duplicating them. +-- +-- Every non-key column is updated, kind and lang included. Nothing about a +-- block's classification is in the key, so a paragraph that a better heading rule +-- promotes to a heading is the same block updated in place -- see the note above +-- the primary key in 00005_doc_blocks.sql. +-- +-- This is belt and braces beside the delete SaveConversion does first, and it +-- cannot be the whole story: a re-conversion that produces FEWER blocks in a +-- region would otherwise leave the tail of the previous run behind at higher +-- indices, where it reads as content. +-- name: UpsertDocBlock :exec +INSERT INTO doc_blocks (document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page, region_x0, idx) DO UPDATE SET + kind = excluded.kind, + level = excluded.level, + text = excluded.text, + lang = excluded.lang, + x0 = excluded.x0, + x1 = excluded.x1, + y0 = excluded.y0, + y1 = excluded.y1, + lines = excluded.lines, + chars = excluded.chars, + note = excluded.note, + created_at = excluded.created_at; + +-- Reading order across the whole document: down the pages, then left to right +-- across each, then in order within a region. A whole-page region sorts first on +-- its page because its region_x0 is 0. +-- name: ListDocBlocks :many +SELECT document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at +FROM doc_blocks +WHERE document_id = ? +ORDER BY page, region_x0, idx; + +-- The funnel's own query: one household's language, and nothing else. A German +-- reader of the columns manual gets the German column of each page rather than +-- the page, which conversion.md measures as a fifth of the work. +-- +-- Blocks whose language was never established have lang = '' and are therefore +-- NOT returned by any language's query. That is deliberate rather than an +-- oversight: passing '' asks for exactly those, which is how the unnamed content +-- of a document stays reachable instead of becoming invisible. +-- name: ListDocBlocksByLang :many +SELECT document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at +FROM doc_blocks +WHERE document_id = ? AND lang = ? +ORDER BY page, region_x0, idx; + +-- name: ListDocBlocksForPage :many +SELECT document_id, page, region_x0, idx, kind, level, text, lang, + x0, x1, y0, y1, lines, chars, note, created_at +FROM doc_blocks +WHERE document_id = ? AND page = ? +ORDER BY region_x0, idx; + +-- Replacing a document's blocks wholesale is how a re-conversion stays honest, +-- and it is required rather than merely tidy: a region that converted to 12 +-- blocks and now converts to 9 would otherwise keep rows at idx 9, 10 and 11, +-- which a reader renders as three paragraphs of the previous run's text. +-- name: DeleteDocBlocks :exec +DELETE FROM doc_blocks WHERE document_id = ?; + +-- What a conversion cost and covered, for the pipeline to report without reading +-- every block back. Every aggregate is wrapped in CAST(... AS INTEGER): without +-- it sqlc cannot infer an aggregate's type in SQLite and emits interface{}, +-- pushing a type assertion onto every caller. +-- name: SummarizeDocBlocks :many +SELECT lang, + CAST(count(*) AS INTEGER) AS blocks, + CAST(sum(chars) AS INTEGER) AS chars, + CAST(sum(lines) AS INTEGER) AS lines, + CAST(count(DISTINCT page) AS INTEGER) AS pages, + CAST(min(page) AS INTEGER) AS first_page +FROM doc_blocks +WHERE document_id = ? +GROUP BY lang +ORDER BY first_page, lang; + +-- Upsert on (document_id, page, idx). A figure has no region and no language in +-- its key, because conversion.md settles that a picture belonging to no language +-- belongs to every language. +-- name: UpsertDocFigure :exec +INSERT INTO doc_figures (document_id, page, idx, x0, y0, x1, y1, ink, text_fraction, + dpi, pixel_width, pixel_height, blob_sha256, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page, idx) DO UPDATE SET + x0 = excluded.x0, + y0 = excluded.y0, + x1 = excluded.x1, + y1 = excluded.y1, + ink = excluded.ink, + text_fraction = excluded.text_fraction, + dpi = excluded.dpi, + pixel_width = excluded.pixel_width, + pixel_height = excluded.pixel_height, + blob_sha256 = excluded.blob_sha256, + created_at = excluded.created_at; + +-- name: ListDocFigures :many +SELECT document_id, page, idx, x0, y0, x1, y1, ink, text_fraction, + dpi, pixel_width, pixel_height, blob_sha256, created_at +FROM doc_figures +WHERE document_id = ? +ORDER BY page, idx; + +-- name: ListDocFiguresForPage :many +SELECT document_id, page, idx, x0, y0, x1, y1, ink, text_fraction, + dpi, pixel_width, pixel_height, blob_sha256, created_at +FROM doc_figures +WHERE document_id = ? AND page = ? +ORDER BY idx; + +-- name: DeleteDocFigures :exec +DELETE FROM doc_figures WHERE document_id = ?; + +-- A figure's callout labels: the text a leader points at. No geometry -- 00009 +-- removed it, because the crop is a band that already prints its own labels and the +-- stored text is now the picture's accessible description rather than something a +-- reader re-lays out. Kept out of doc_blocks on purpose -- see 00008's header for why +-- a sixth block kind was the wrong shape and what it would have cost. +-- Upsert on (document_id, page, figure_idx, idx), so a re-conversion converges. +-- name: UpsertDocFigureLabel :exec +INSERT INTO doc_figure_labels (document_id, page, figure_idx, idx, text, created_at) +VALUES (?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page, figure_idx, idx) DO UPDATE SET + text = excluded.text, + created_at = excluded.created_at; + +-- ORDER BY idx is the label's own reading order, down then across, and it is the +-- order the description is read in. Both list queries carry it. +-- name: ListDocFigureLabels :many +SELECT document_id, page, figure_idx, idx, text, created_at +FROM doc_figure_labels +WHERE document_id = ? +ORDER BY page, figure_idx, idx; + +-- name: ListDocFigureLabelsForPage :many +SELECT document_id, page, figure_idx, idx, text, created_at +FROM doc_figure_labels +WHERE document_id = ? AND page = ? +ORDER BY figure_idx, idx; + +-- Deleting a document's figures already cascades to its labels, so this exists for +-- the one case the cascade does not cover: rewriting the labels of figures that are +-- themselves unchanged. Cheaper and clearer than reasoning about which rows the +-- figure delete happened to take with it. +-- name: DeleteDocFigureLabels :exec +DELETE FROM doc_figure_labels WHERE document_id = ?; diff --git a/internal/db/queries/doclangs.sql b/internal/db/queries/doclangs.sql new file mode 100644 index 0000000..7914ceb --- /dev/null +++ b/internal/db/queries/doclangs.sql @@ -0,0 +1,51 @@ +-- name: UpsertDocLang :exec +INSERT INTO doc_langs (document_id, source, pdf_start, pdf_end, code, lang, title, + printed_page, confidence, conflict, note, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, source, code, pdf_start) DO UPDATE SET + pdf_end = excluded.pdf_end, + lang = excluded.lang, + title = excluded.title, + printed_page = excluded.printed_page, + confidence = excluded.confidence, + conflict = excluded.conflict, + note = excluded.note; + +-- name: ListDocLangs :many +SELECT * FROM doc_langs WHERE document_id = ? ORDER BY source, pdf_start; + +-- name: ListDocLangsBySource :many +SELECT * FROM doc_langs WHERE document_id = ? AND source = ? ORDER BY pdf_start; + +-- Replacing one signal's view wholesale is how a re-probe stays honest: a run +-- that no longer exists must disappear rather than linger from the previous +-- attempt. Scoped to one source so the other signals' rows survive. +-- name: DeleteDocLangsBySource :exec +DELETE FROM doc_langs WHERE document_id = ? AND source = ?; + +-- name: DeleteDocLangs :exec +DELETE FROM doc_langs WHERE document_id = ?; + +-- The language map as shown to the user: one row per language in the reconciled +-- view, with its page total and whether any of its runs are disputed. +-- +-- A run with pdf_start = 0 named a language it could not place, so it covers no +-- pages at all. Counting its span reported a language the printed index merely +-- mentioned as a one-page section. +-- name: SummarizeDocLangs :many +SELECT code, + lang, + CAST(sum(CASE WHEN pdf_start = 0 THEN 0 ELSE pdf_end - pdf_start + 1 END) + AS INTEGER) AS pages, + CAST(count(*) AS INTEGER) AS runs, + CAST(max(conflict) AS INTEGER) AS disputed, + CAST(min(pdf_start) AS INTEGER) AS first_page +FROM doc_langs +WHERE document_id = ? AND source = ? +GROUP BY code, lang +ORDER BY first_page; + +-- name: CountDocLangConflicts :one +SELECT CAST(count(*) AS INTEGER) AS total +FROM doc_langs +WHERE document_id = ? AND source = ? AND conflict = 1; diff --git a/internal/db/queries/docpages.sql b/internal/db/queries/docpages.sql new file mode 100644 index 0000000..9286c37 --- /dev/null +++ b/internal/db/queries/docpages.sql @@ -0,0 +1,57 @@ +-- Upsert on the natural key, because a probe job may run twice and must converge +-- on the same rows rather than duplicating them. +-- name: UpsertDocPage :exec +INSERT INTO doc_pages (document_id, page_no, chars, script, page_tag, printed_folio, lang, lang_source, figures) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, page_no) DO UPDATE SET + chars = excluded.chars, + script = excluded.script, + page_tag = excluded.page_tag, + printed_folio = excluded.printed_folio, + lang = excluded.lang, + lang_source = excluded.lang_source, + figures = excluded.figures; + +-- name: ListDocPages :many +SELECT * FROM doc_pages WHERE document_id = ? ORDER BY page_no; + +-- name: GetDocPage :one +SELECT * FROM doc_pages WHERE document_id = ? AND page_no = ?; + +-- name: DeleteDocPages :exec +DELETE FROM doc_pages WHERE document_id = ?; + +-- How many pages the document holds in each resolved language. The CAST is +-- required: without it sqlc infers interface{} for the aggregate. +-- name: CountDocPagesByLang :many +SELECT lang, CAST(count(*) AS INTEGER) AS pages +FROM doc_pages +WHERE document_id = ? AND lang <> '' +GROUP BY lang +ORDER BY pages DESC, lang; + +-- name: CountDocPagesWithText :one +SELECT CAST(count(*) AS INTEGER) AS pages +FROM doc_pages +WHERE document_id = ? AND chars > 0; + +-- How far each page's PDF number runs ahead of the number printed on the paper, +-- as a histogram over the pages that print one at all. +-- +-- This is derived on read rather than stored, because doc_pages already holds the +-- whole answer and a stored copy could only go stale against it: the folio is +-- re-read on every probe, so a change to how it is read must move this number in +-- the same breath. It is one small grouped scan per document over rows the probe +-- already wrote, asked once when a conversion is served, not per block or per page. +-- +-- The caller decides which row to believe -- see registry.FolioOffset -- so the +-- whole histogram comes back rather than just its first row. The CASTs are +-- required: without them sqlc infers interface{} for both columns. "offset" is a +-- SQL keyword, hence the name. +-- name: DocPageFolioOffsets :many +SELECT CAST(page_no - printed_folio AS INTEGER) AS folio_offset, + CAST(count(*) AS INTEGER) AS pages +FROM doc_pages +WHERE document_id = ? AND printed_folio IS NOT NULL +GROUP BY folio_offset +ORDER BY pages DESC, folio_offset; diff --git a/internal/db/queries/docregions.sql b/internal/db/queries/docregions.sql new file mode 100644 index 0000000..4e9a3f7 --- /dev/null +++ b/internal/db/queries/docregions.sql @@ -0,0 +1,103 @@ +-- Queries over doc_regions: one language's territory on a page. See +-- 00004_doc_regions.sql for the schema's reasoning and docs/design/regions.md for +-- the contract. +-- +-- TWO RULES FOR THIS FILE, BOTH LEARNED THE HARD WAY WHILE WRITING IT. +-- +-- 1. KEEP THIS FILE PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 +-- (pinned in tools/go.mod) mixes up character and byte offsets when it cuts +-- statements out of a file, so a single non-ASCII character anywhere earlier +-- corrupts every statement after it. What is measured is the rule, not the +-- internals: the damage equals the extra bytes those characters occupy, one +-- character of SQL lost per extra byte. +-- +-- Two shapes were observed, and the quiet one is the dangerous one. With +-- em-dashes in a comment above, "ORDER BY first_page, code" generated as +-- "ORDER BY first_page, co" for one and "ORDER BY first_pa" for four -- clean +-- Go, broken SQL. With em-dashes placed differently, sqlc instead garbled a +-- statement badly enough to fail its own parser, printing tokens like +-- "SELdocument_id" and exiting noisily. Which of the two you get depends on +-- where the character sits, so neither a clean run nor a loud failure tells +-- you the file is safe. Only ASCII does. +-- +-- The direction of sqlc's own mismatch is deliberately not asserted here. It +-- was not read out of sqlc's source, and the two published guesses point +-- opposite ways -- byte offsets applied to characters would overshoot a +-- statement's end rather than cut it short, which is not what happens. The +-- rule above is what was measured and is what protects this file. +-- +-- This is the worst failure shape available: `make sqlc` exits 0, the generated +-- Go compiles, the linter is happy, and the statement fails at PREPARE time +-- inside a background job against a user's database. All ten pre-existing query +-- files happen to be pure ASCII, which is the only reason this had not bitten +-- anyone yet. That was checked rather than assumed: 0 non-ASCII bytes across +-- every one of them. TestDocRegionQueriesExecute in internal/db is the guard; +-- it runs every statement below against a real migrated database, so a mangled +-- one cannot reach a user. +-- +-- 2. Columns are listed explicitly rather than with SELECT *, so that adding a +-- column to doc_regions later cannot silently change every caller's row shape. + +-- Upsert on the natural key (document_id, source, page, x0), because a probe job +-- may run twice and must converge on the same rows rather than duplicating them. +-- +-- This is belt and braces beside the delete that SaveProbe does first, and it +-- cannot be the whole story: source is part of the key and a region's source can +-- change between probes, so an upsert alone would leave the superseded row behind +-- at the same x0. The note at the foot of 00004_doc_regions.sql explains why. +-- name: UpsertDocRegion :exec +INSERT INTO doc_regions (document_id, source, page, x0, x1, code, lang, chars, runs, + conflict, note, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(document_id, source, page, x0) DO UPDATE SET + x1 = excluded.x1, + code = excluded.code, + lang = excluded.lang, + chars = excluded.chars, + runs = excluded.runs, + conflict = excluded.conflict, + note = excluded.note, + created_at = excluded.created_at; + +-- Reading order: down the page, then left to right across it. A whole-page region +-- sorts first on its page because it begins at x0 = 0. +-- name: ListDocRegions :many +SELECT document_id, source, page, x0, x1, code, lang, chars, runs, conflict, note, created_at +FROM doc_regions +WHERE document_id = ? +ORDER BY page, x0; + +-- name: ListDocRegionsForPage :many +SELECT document_id, source, page, x0, x1, code, lang, chars, runs, conflict, note, created_at +FROM doc_regions +WHERE document_id = ? AND page = ? +ORDER BY x0; + +-- Replacing a document's regions wholesale is how a re-probe stays honest, and it +-- is required rather than merely tidy: a region whose attribution changed is a new +-- row under this key, so without the delete the superseded one lingers and the +-- page reports itself twice. +-- name: DeleteDocRegions :exec +DELETE FROM doc_regions WHERE document_id = ?; + +-- The region map as shown to the user: one row per language label, with the +-- characters and runs it holds, how many pages it appears on, and whether any of +-- its regions are disputed. +-- +-- Characters rather than pages is the point, because a page holding three +-- languages is not a unit of size; pages are still what a reader is shown, so both +-- are reported. Every aggregate is wrapped in CAST(... AS INTEGER): without it +-- sqlc cannot infer the type and emits interface{}, pushing a type assertion onto +-- every caller. +-- name: SummarizeDocRegions :many +SELECT code, + lang, + CAST(sum(chars) AS INTEGER) AS chars, + CAST(sum(runs) AS INTEGER) AS runs, + CAST(count(DISTINCT page) AS INTEGER) AS pages, + CAST(min(page) AS INTEGER) AS first_page, + CAST(max(conflict) AS INTEGER) AS disputed +FROM doc_regions +WHERE document_id = ? +GROUP BY code, lang +ORDER BY first_page, code; diff --git a/internal/db/queries/documents.sql b/internal/db/queries/documents.sql new file mode 100644 index 0000000..845d834 --- /dev/null +++ b/internal/db/queries/documents.sql @@ -0,0 +1,68 @@ +-- Uploading the same bytes against the same device twice is the same document, +-- enforced by documents_device_blob_idx. DO NOTHING plus a follow-up lookup makes +-- the upload handler idempotent without the caller having to check first. +-- name: CreateDocument :execrows +INSERT INTO documents (id, device_id, blob_sha256, filename, media_type, kind, state, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(device_id, blob_sha256) DO NOTHING; + +-- name: GetDocument :one +SELECT * FROM documents WHERE id = ?; + +-- name: GetDocumentByDeviceAndBlob :one +SELECT * FROM documents WHERE device_id = ? AND blob_sha256 = ?; + +-- name: ListDocumentsForDevice :many +SELECT * FROM documents WHERE device_id = ? ORDER BY created_at DESC; + +-- name: ListDocumentsByState :many +SELECT * FROM documents WHERE state = ? ORDER BY created_at DESC; + +-- name: SetDocumentState :exec +UPDATE documents SET state = ?, last_error = ?, updated_at = ? WHERE id = ?; + +-- Records everything stages 0 and 1 discovered, in one statement. Writing the +-- probe result and the new state together keeps a crash from leaving a document +-- that claims to be probed but has no page count. +-- name: RecordDocumentProbe :exec +UPDATE documents +SET page_count = ?, + encrypted = ?, + tagged = ?, + has_text_layer = ?, + median_chars_per_page = ?, + content_start_page = ?, + content_end_page = ?, + state = ?, + last_error = '', + probed_at = ?, + updated_at = ? +WHERE id = ?; + +-- Records the scope the user approved at the gate, together with the state that +-- says the work is authorised. One statement rather than two, for +-- RecordDocumentProbe's reason: a crash between them would leave a document that +-- is converting under a scope nobody chose. +-- +-- include_neutral_pages is the whole of the extra scope, and it is a flag and not a +-- page list on purpose -- see 00007's header. The server recomputes the set of pages +-- from the stored region map, so a stale client cannot name a page the gate never +-- offered it. +-- name: ApproveDocumentScope :exec +UPDATE documents +SET include_neutral_pages = ?, + state = ?, + last_error = '', + updated_at = ? +WHERE id = ?; + +-- name: DeleteDocument :exec +DELETE FROM documents WHERE id = ?; + +-- name: CountDocuments :one +SELECT CAST(count(*) AS INTEGER) AS total FROM documents; + +-- Used to decide whether a blob is still referenced before deleting it, since +-- two devices can legitimately share one uploaded file. +-- name: CountDocumentsForBlob :one +SELECT CAST(count(*) AS INTEGER) AS total FROM documents WHERE blob_sha256 = ?; diff --git a/internal/db/queries/locations.sql b/internal/db/queries/locations.sql new file mode 100644 index 0000000..109fcc3 --- /dev/null +++ b/internal/db/queries/locations.sql @@ -0,0 +1,22 @@ +-- name: CreateLocation :one +INSERT INTO locations (id, name, parent_id, notes, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?) +RETURNING *; + +-- name: GetLocation :one +SELECT * FROM locations WHERE id = ?; + +-- name: ListLocations :many +SELECT * FROM locations ORDER BY name; + +-- name: UpdateLocation :one +UPDATE locations +SET name = ?, parent_id = ?, notes = ?, updated_at = ? +WHERE id = ? +RETURNING *; + +-- name: DeleteLocation :exec +DELETE FROM locations WHERE id = ?; + +-- name: CountLocations :one +SELECT CAST(count(*) AS INTEGER) AS total FROM locations; diff --git a/internal/db/queries/search.sql b/internal/db/queries/search.sql new file mode 100644 index 0000000..5305bc5 --- /dev/null +++ b/internal/db/queries/search.sql @@ -0,0 +1,142 @@ +-- Queries over doc_blocks_fts: which manual says X, and where. See +-- 00006_block_search.sql for the index's reasoning and the measurement behind the +-- tokeniser, and docs/design/search.md for the contract. +-- +-- THIS FILE MUST STAY PURE ASCII. No em-dashes, no curly quotes. sqlc v1.31.1 +-- mixes up character and byte offsets when it cuts statements out of a file, so +-- one non-ASCII character anywhere above corrupts every statement after it -- +-- silently in the dangerous case: `make sqlc` exits 0, the Go compiles, the +-- linter passes, and the statement fails at PREPARE time inside a request. The +-- full measurement is in the header of docregions.sql; TestQueryFilesAreASCII is +-- the cause-side guard and TestSearchQueriesExecute the symptom-side one. +-- +-- TWO THINGS SQLC CANNOT PARSE, BOTH LEARNED HERE AND BOTH LOAD-BEARING. +-- +-- 1. `WHERE doc_blocks_fts MATCH ?` -- the documented FTS5 form, where the left +-- side is the table's own hidden column -- fails generation with `column +-- "doc_blocks_fts" does not exist`, because sqlc models the virtual table as +-- its declared columns only. `doc_blocks_fts.text MATCH ?` generates and is +-- the same query: text is the only indexed column, so a column-scoped match +-- over it covers the whole index. Verified against a real database rather than +-- assumed, in TestSearchQueriesExecute. +-- +-- 2. `AS rank` fails generation with `mismatched input 'rank'`, so the ordering +-- column is named `score`. That is a happy accident: `rank` is also FTS5's own +-- magic column, and a result column of that name reads as if it were that. +-- +-- Columns are listed explicitly rather than with SELECT *, so that adding a +-- column later cannot silently change every caller's row shape. +-- +-- WHY EVERY QUERY JOINS documents AND devices. A hit has to say WHICH manual, not +-- merely that something matched: README's first problem is that the paper pile is +-- unsearchable, and "page 47 of something" does not solve it. The filename and the +-- device's name are what a household recognises, and they cost one join each +-- against a primary key. +-- +-- WHY THE HEADING BONUS IS 1.0. bm25 is negative and lower is better, so the +-- bonus is subtracted. Measured on both real manuals: within one query bm25 spans +-- about -9 to -2, and adjacent hits differ by 0.05 to 0.5, so 1.0 moves a heading +-- past hits of comparable quality without overturning a decisively better one. On +-- "Filter" in the column manual it lifts the maintenance heading "Ausblasfilter +-- austauschen" over the parts-list fragments ("1. Filter", "13. Filter") that +-- bm25's short-document bias otherwise puts first; on "Saugkraft" the +-- troubleshooting cell "Saugkraft ist zu gering" at -8.5 stays first, which is +-- right. Both numbers are returned, so the judgement can be argued with rather +-- than merely trusted. + +-- name: SearchBlocks :many +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + snippet(doc_blocks_fts, 0, '', '', '...', 64) AS snippet, + CAST(bm25(doc_blocks_fts) AS REAL) AS bm25, + CAST(bm25(doc_blocks_fts) + - (CASE b.kind WHEN 'heading' THEN 1.0 ELSE 0.0 END) AS REAL) AS score +FROM doc_blocks_fts +JOIN doc_blocks b ON b.rowid = doc_blocks_fts.rowid +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE doc_blocks_fts.text MATCH sqlc.arg(match) +ORDER BY score, b.document_id, b.page, b.region_x0, b.idx +LIMIT sqlc.arg(limit); + +-- The same question narrowed to one manual, which is what a reader already inside +-- a document asks. A separate statement rather than an optional parameter, because +-- sqlc has no optional parameters and `b.document_id = ? OR ? = ''` would put the +-- widest query in the household on the sentinel path. +-- name: SearchBlocksInDocument :many +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + snippet(doc_blocks_fts, 0, '', '', '...', 64) AS snippet, + CAST(bm25(doc_blocks_fts) AS REAL) AS bm25, + CAST(bm25(doc_blocks_fts) + - (CASE b.kind WHEN 'heading' THEN 1.0 ELSE 0.0 END) AS REAL) AS score +FROM doc_blocks_fts +JOIN doc_blocks b ON b.rowid = doc_blocks_fts.rowid +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE doc_blocks_fts.text MATCH sqlc.arg(match) + AND b.document_id = sqlc.arg(document_id) +ORDER BY score, b.page, b.region_x0, b.idx +LIMIT sqlc.arg(limit); + +-- THE HOLE THE TOKENISER LEAVES, AND WHAT FILLS IT. +-- +-- A trigram index holds no token shorter than three characters, so a query of one +-- or two characters matches nothing at all -- not "fewer results", none. That is +-- tolerable in German and Russian, where a two-letter query is not a word anyone +-- searches for, and it is not tolerable in Chinese or Japanese, where two +-- characters is an ordinary word: measured on the sequential manual, the two +-- characters for "power" occur in 27 stored blocks and those for "product" in 24, +-- and the index finds 0 of each. +-- +-- So a query the index cannot represent is answered by scanning instead. Measured +-- over the 3,122 blocks of both real manuals: 1.9 ms for a two-character Japanese +-- query, against 0.2 ms for the same question through the index. A household's +-- whole library is a small multiple of that corpus, so the scan stays inside a +-- request rather than becoming a job. +-- +-- instr rather than LIKE, because `%` and `_` in a user's query are LIKE wildcards +-- and a search box must not have a pattern language. lower() on both sides is +-- SQLite's own, which folds ASCII and nothing else -- exact for the CJK queries +-- this path exists for, and case-sensitive for a two-letter Cyrillic one, which is +-- the honest limit of a scan that must not build an index to fix. +-- +-- There is no bm25 here because there is no index term to weigh, so score is 0 on +-- every row and the order is the heading rule followed by reading order. A caller +-- tells the two paths apart by the mode the API reports, not by inferring it from +-- the numbers. +-- name: SearchBlocksSubstring :many +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + substr(b.text, max(1, instr(lower(b.text), lower(sqlc.arg(needle))) - 24), 64) AS snippet, + CAST(0.0 AS REAL) AS bm25, + CAST(0.0 AS REAL) AS score +FROM doc_blocks b +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE instr(lower(b.text), lower(sqlc.arg(needle))) > 0 +ORDER BY (CASE b.kind WHEN 'heading' THEN 0 ELSE 1 END), + b.document_id, b.page, b.region_x0, b.idx +LIMIT sqlc.arg(limit); + +-- name: SearchBlocksSubstringInDocument :many +SELECT b.document_id, d.filename, d.state, v.id AS device_id, v.name AS device_name, + b.page, b.region_x0, b.idx, b.kind, b.level, b.lang, b.chars, + substr(b.text, max(1, instr(lower(b.text), lower(sqlc.arg(needle))) - 24), 64) AS snippet, + CAST(0.0 AS REAL) AS bm25, + CAST(0.0 AS REAL) AS score +FROM doc_blocks b +JOIN documents d ON d.id = b.document_id +JOIN devices v ON v.id = d.device_id +WHERE instr(lower(b.text), lower(sqlc.arg(needle))) > 0 + AND b.document_id = sqlc.arg(document_id) +ORDER BY (CASE b.kind WHEN 'heading' THEN 0 ELSE 1 END), + b.page, b.region_x0, b.idx +LIMIT sqlc.arg(limit); + +-- How many blocks are indexed at all, so a caller can tell "nothing matched" from +-- "nothing has been converted yet". Wrapped in CAST(... AS INTEGER) for the reason +-- every other aggregate in these files is: without it sqlc cannot infer an +-- aggregate's type in SQLite and emits interface{}. +-- name: CountSearchableBlocks :one +SELECT CAST(count(*) AS INTEGER) FROM doc_blocks; diff --git a/internal/db/search_generated_test.go b/internal/db/search_generated_test.go new file mode 100644 index 0000000..ec5241a --- /dev/null +++ b/internal/db/search_generated_test.go @@ -0,0 +1,377 @@ +package db + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/id" +) + +// searchFixture is a migrated database holding one device, one document and a few +// blocks in the scripts that decided the tokeniser. +func searchFixture(t *testing.T) (database *DB, documentID string) { + t.Helper() + ctx := context.Background() + + database, err := Open(ctx, Options{Path: filepath.Join(t.TempDir(), "search.db")}) + if err != nil { + t.Fatalf("open database: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + + w := gen.New(database.Write()) + docID, deviceID := id.New(id.Document), id.New(id.Device) + sha := strings.Repeat("a", 64) + if err := w.UpsertBlob(ctx, gen.UpsertBlobParams{ + Sha256: sha, SizeBytes: 1, MediaType: "application/pdf", CreatedAt: Now(), + }); err != nil { + t.Fatalf("blob: %v", err) + } + if _, err := database.Write().ExecContext(ctx, + `INSERT INTO devices (id, name, created_at, updated_at) VALUES (?, 'Robot vacuum', ?, ?)`, + deviceID, Now(), Now()); err != nil { + t.Fatalf("device: %v", err) + } + if _, err := w.CreateDocument(ctx, gen.CreateDocumentParams{ + ID: docID, DeviceID: deviceID, BlobSha256: sha, Filename: "manual.pdf", + Kind: "manual", State: "ready", CreatedAt: Now(), UpdatedAt: Now(), + }); err != nil { + t.Fatalf("document: %v", err) + } + + blocks := []gen.UpsertDocBlockParams{ + {Page: 48, RegionX0: 43, Idx: 0, Kind: "heading", Level: 2, Lang: "de", + Text: "Ausblasfilter austauschen", X1: 300, Y1: 118, Lines: 1, Chars: 25}, + {Page: 48, RegionX0: 43, Idx: 1, Kind: "paragraph", Lang: "de", + Text: "Zubehör und Düsen alle drei Monate reinigen.", X1: 300, Y1: 170, Lines: 2, Chars: 43}, + {Page: 539, RegionX0: 0, Idx: 0, Kind: "paragraph", Lang: "ja", + Text: "本製品を使用する前に取扱説明書をお読みください。", + X1: 800, Y1: 200, Lines: 2, Chars: 27}, + } + for i := range blocks { + blocks[i].DocumentID = docID + blocks[i].CreatedAt = Now() + if err := w.UpsertDocBlock(ctx, blocks[i]); err != nil { + t.Fatalf("UpsertDocBlock %d: %v", i, err) + } + } + return database, docID +} + +// TestSearchQueriesExecute is a SMOKE TEST FOR THE GENERATOR, the twin of +// TestDocBlockQueriesExecute and TestDocRegionQueriesExecute, and it earns its keep +// twice over here. +// +// The first reason is theirs: sqlc v1.31.1 silently truncates the tail of a +// generated statement when a query file holds a non-ASCII character, so `make sqlc` +// exits 0, the Go compiles, the linter passes and the statement fails at PREPARE +// time. Executing each statement once turns that into a build failure. +// +// The second is specific to search. queries/search.sql cannot use the documented +// FTS5 form `WHERE doc_blocks_fts MATCH ?`, because sqlc models a virtual table as +// its declared columns and rejects the table's own hidden column as unknown. The +// form that generates is `doc_blocks_fts.text MATCH ?`, a column-scoped match, and +// nothing but running it against a real FTS5 table proves the two are the same +// query. So this test is also the evidence for that workaround. +func TestSearchQueriesExecute(t *testing.T) { + ctx := context.Background() + database, docID := searchFixture(t) + r := gen.New(database.Read()) + + hits, err := r.SearchBlocks(ctx, gen.SearchBlocksParams{Match: `"Ausblasfilter"`, Limit: 10}) + if err != nil { + t.Fatalf("SearchBlocks: %v", err) + } + if len(hits) != 1 { + t.Fatalf("SearchBlocks returned %d rows, want 1", len(hits)) + } + got := hits[0] + // Every joined column, because a truncated statement is exactly what loses the + // tail of a SELECT list and a hit without a device name answers nothing. + if got.DocumentID != docID || got.Filename != "manual.pdf" || got.DeviceName != "Robot vacuum" { + t.Errorf("hit = %+v; want it to name the document, its file and its device", got) + } + if got.Page != 48 || got.RegionX0 != 43 || got.Idx != 0 { + t.Errorf("hit is at %d/%d/%d, want page 48, region 43, index 0", + got.Page, got.RegionX0, got.Idx) + } + if got.Kind != "heading" || got.Level != 2 || got.Lang != "de" || got.State != "ready" { + t.Errorf("hit = %+v; want the heading's own columns", got) + } + if !strings.Contains(got.Snippet, "Ausblasfilter") { + t.Errorf("snippet %q does not hold the term", got.Snippet) + } + // bm25 is negative and the heading bonus is subtracted, so score is lower still. + if got.Bm25 >= 0 { + t.Errorf("bm25 = %v, want a negative score", got.Bm25) + } + if diff := got.Bm25 - got.Score; diff < 0.99 || diff > 1.01 { + t.Errorf("heading bonus = %v, want 1.0", diff) + } + + narrowed, err := r.SearchBlocksInDocument(ctx, gen.SearchBlocksInDocumentParams{ + Match: `"Filter"`, DocumentID: docID, Limit: 10, + }) + if err != nil { + t.Fatalf("SearchBlocksInDocument: %v", err) + } + if len(narrowed) != 1 { + t.Errorf("SearchBlocksInDocument returned %d rows, want 1", len(narrowed)) + } + if elsewhere, err := r.SearchBlocksInDocument(ctx, gen.SearchBlocksInDocumentParams{ + Match: `"Filter"`, DocumentID: "doc_nope", Limit: 10, + }); err != nil || len(elsewhere) != 0 { + t.Errorf("narrowing to another document returned %d rows (err %v)", len(elsewhere), err) + } + + // The scan, whose ORDER BY puts a heading first and then reads in page order. + scanned, err := r.SearchBlocksSubstring(ctx, gen.SearchBlocksSubstringParams{ + Needle: "Filter", Limit: 10, + }) + if err != nil { + t.Fatalf("SearchBlocksSubstring: %v", err) + } + if len(scanned) != 1 || scanned[0].Kind != "heading" { + t.Errorf("SearchBlocksSubstring returned %+v, want the one heading", scanned) + } + if scanned[0].Bm25 != 0 || scanned[0].Score != 0 { + t.Errorf("the scan reported bm25 %v score %v; there is no term to weigh", + scanned[0].Bm25, scanned[0].Score) + } + // Lowercased on both sides, because the scan's own matching is SQLite's lower() + // and the snippet is a slice of the block's text exactly as stored. + if !strings.Contains(strings.ToLower(scanned[0].Snippet), "filter") { + t.Errorf("scan snippet %q does not hold the needle", scanned[0].Snippet) + } + if inDoc, err := r.SearchBlocksSubstringInDocument(ctx, + gen.SearchBlocksSubstringInDocumentParams{ + Needle: "Filter", DocumentID: docID, Limit: 10, + }); err != nil || len(inDoc) != 1 { + t.Errorf("SearchBlocksSubstringInDocument returned %d rows (err %v)", len(inDoc), err) + } + + if n, err := r.CountSearchableBlocks(ctx); err != nil || n != 3 { + t.Errorf("CountSearchableBlocks = %d (err %v), want 3", n, err) + } +} + +// TestTheIndexHoldsWhatTheTokeniserWasChosenFor: a word inside a run with no spaces +// in it. This is the whole reason the tokeniser is trigram, and it is the assertion +// that fails on FTS5's default unicode61, which indexes the entire Japanese +// sentence as one token and finds nothing inside it. +func TestTheIndexHoldsWhatTheTokeniserWasChosenFor(t *testing.T) { + ctx := context.Background() + database, _ := searchFixture(t) + r := gen.New(database.Read()) + + // "Instruction manual", in the middle of a Japanese sentence. + hits, err := r.SearchBlocks(ctx, gen.SearchBlocksParams{ + Match: `"取扱説明書"`, Limit: 10, + }) + if err != nil { + t.Fatalf("SearchBlocks: %v", err) + } + if len(hits) != 1 || hits[0].Lang != "ja" { + t.Errorf("a Japanese word inside a spaceless run found %d hits, want the ja "+ + "block: %+v", len(hits), hits) + } + + // And the Latin fold, which trigram does not do unless it is asked to: without + // `remove_diacritics 1` in the migration this is 0 hits. + folded, err := r.SearchBlocks(ctx, gen.SearchBlocksParams{Match: `"Zubehor"`, Limit: 10}) + if err != nil { + t.Fatalf("SearchBlocks folded: %v", err) + } + if len(folded) != 1 { + t.Errorf("searching without the umlaut found %d hits, want 1", len(folded)) + } +} + +// TestBlockSearchIndexSurvivesEveryWriteToDocBlocks holds the index against FTS5's +// own integrity check after each of the three paths that change doc_blocks, and the +// third is the one no Go code observes. +// +// 'integrity-check' compares the index against the content table and fails if they +// disagree, which is exactly the failure an unmaintained external content index +// produces: a hit whose text no longer exists. The control at the end proves the +// check is not vacuous. +func TestBlockSearchIndexSurvivesEveryWriteToDocBlocks(t *testing.T) { + ctx := context.Background() + database, docID := searchFixture(t) + w := gen.New(database.Write()) + r := gen.New(database.Read()) + + integrity := func(stage string) error { + _, err := database.Write().ExecContext(ctx, + `INSERT INTO doc_blocks_fts(doc_blocks_fts, rank) VALUES ('integrity-check', 1)`) + if err != nil { + t.Errorf("the index is corrupt after %s: %v", stage, err) + } + return err + } + hits := func(term string) int { + rows, err := r.SearchBlocks(ctx, gen.SearchBlocksParams{Match: `"` + term + `"`, Limit: 50}) + if err != nil { + t.Fatalf("search %q: %v", term, err) + } + return len(rows) + } + + _ = integrity("the initial inserts") + if hits("Ausblasfilter") != 1 { + t.Fatalf("setup: the heading is not in the index") + } + + // 1. The upsert path: same key, new text. The old term must go and the new one + // must arrive, which is what the UPDATE trigger's delete-then-insert is for. + if err := w.UpsertDocBlock(ctx, gen.UpsertDocBlockParams{ + DocumentID: docID, Page: 48, RegionX0: 43, Idx: 0, Kind: "heading", Level: 2, + Lang: "de", Text: "Motorschutzfilter waschen", X1: 300, Y1: 118, + Lines: 1, Chars: 25, CreatedAt: Now(), + }); err != nil { + t.Fatalf("upsert over an existing key: %v", err) + } + _ = integrity("an upsert in place") + if n := hits("Ausblasfilter"); n != 0 { + t.Errorf("the replaced text is still findable %d times", n) + } + if n := hits("Motorschutzfilter"); n != 1 { + t.Errorf("the new text is findable %d times, want 1", n) + } + + // 2. The wholesale replace registry.saveBlocks does. + if err := w.DeleteDocBlocks(ctx, docID); err != nil { + t.Fatalf("delete blocks: %v", err) + } + _ = integrity("the wholesale delete") + if n := hits("Motorschutzfilter"); n != 0 { + t.Errorf("a deleted block is findable %d times", n) + } + + // 3. The ON DELETE CASCADE from documents, which runs no Go at all. SQLite's own + // documentation makes trigger firing on a foreign key action conditional on + // recursive_triggers, which internal/db does not set -- so this is measured here + // rather than assumed anywhere. + if err := w.UpsertDocBlock(ctx, gen.UpsertDocBlockParams{ + DocumentID: docID, Page: 1, RegionX0: 0, Idx: 0, Kind: "paragraph", Lang: "de", + Text: "Der Wasserfilter sitzt hinten.", X1: 300, Y1: 118, Lines: 1, Chars: 30, + CreatedAt: Now(), + }); err != nil { + t.Fatalf("re-insert: %v", err) + } + if hits("Wasserfilter") != 1 { + t.Fatalf("setup for the cascade: the block is not in the index") + } + if _, err := database.Write().ExecContext(ctx, + `DELETE FROM documents WHERE id = ?`, docID); err != nil { + t.Fatalf("delete document: %v", err) + } + _ = integrity("the cascade from documents") + if n := hits("Wasserfilter"); n != 0 { + t.Errorf("a document deleted by cascade is still findable %d times", n) + } + + revertCheckTheDeleteTrigger(t, database) +} + +// revertCheckTheDeleteTrigger drops the delete trigger and shows what goes wrong, +// because everything above would otherwise pass for the wrong reason. +// +// AND THE FAILURE IS NOT WHAT IT LOOKS LIKE, which is why this is worth its own +// function. Removing the trigger does NOT leave a deleted manual findable: every +// search joins the index to doc_blocks, so an index entry whose row is gone joins to +// nothing and silently disappears from the results. Measured that way round first, +// and it made the obvious control assertion pass while the index was corrupt. +// +// What actually goes wrong is worse. SQLite hands a new row max(rowid)+1, so +// deleting the highest block frees a rowid that the next insert takes. The stale +// index entry then points at a REAL row belonging to a DIFFERENT document, and a +// search for a word from the deleted manual returns a confident hit naming another +// manual, another page and text that does not contain the word. A wrong citation, +// not a missing one. +func revertCheckTheDeleteTrigger(t *testing.T, database *DB) { + t.Helper() + ctx := context.Background() + w := gen.New(database.Write()) + + doomed := reinsertDocument(t, database, "doomed.pdf") + if err := w.UpsertDocBlock(ctx, gen.UpsertDocBlockParams{ + DocumentID: doomed, Page: 1, RegionX0: 0, Idx: 0, Kind: "paragraph", Lang: "de", + Text: "Der Hygienefilter ist gewaschen.", X1: 300, Y1: 118, Lines: 1, Chars: 32, + CreatedAt: Now(), + }); err != nil { + t.Fatalf("control insert: %v", err) + } + + if _, err := database.Write().ExecContext(ctx, `DROP TRIGGER doc_blocks_fts_delete`); err != nil { + t.Fatalf("drop trigger: %v", err) + } + if _, err := database.Write().ExecContext(ctx, + `DELETE FROM documents WHERE id = ?`, doomed); err != nil { + t.Fatalf("control delete: %v", err) + } + + // FTS5's own check sees the damage even though a search does not. + if _, err := database.Write().ExecContext(ctx, + `INSERT INTO doc_blocks_fts(doc_blocks_fts, rank) VALUES ('integrity-check', 1)`); err == nil { + t.Error("with the delete trigger dropped, the cascade left a consistent index. " + + "Either the trigger is not what keeps it correct, or integrity-check does " + + "not detect this -- and then every assertion above is worthless.") + } + + // Now the consequence. A block of an unrelated document takes the freed rowid. + other := reinsertDocument(t, database, "unrelated.pdf") + if err := w.UpsertDocBlock(ctx, gen.UpsertDocBlockParams{ + DocumentID: other, Page: 9, RegionX0: 0, Idx: 0, Kind: "paragraph", Lang: "de", + Text: "Ganz andere Anleitung, anderes Gerät.", X1: 300, Y1: 118, Lines: 1, Chars: 37, + CreatedAt: Now(), + }); err != nil { + t.Fatalf("control re-insert: %v", err) + } + + rows, err := gen.New(database.Read()).SearchBlocks(ctx, gen.SearchBlocksParams{ + Match: `"Hygienefilter"`, Limit: 10, + }) + if err != nil { + t.Fatalf("control search: %v", err) + } + if len(rows) == 0 { + t.Error("without the delete trigger, the freed rowid was not reused and the " + + "stale entry stayed invisible. The trigger still has to exist -- FTS5's " + + "integrity check above says the index is corrupt -- but this assertion no " + + "longer demonstrates the harm, so find the shape that does before " + + "trusting it.") + return + } + if rows[0].DocumentID != other { + t.Errorf("control: expected the stale entry to resolve to the unrelated "+ + "document, got %+v", rows[0]) + } + t.Logf("without the delete trigger, searching for a word from the deleted manual "+ + "returns %q on page %d of %s, which does not contain it", + rows[0].Snippet, rows[0].Page, rows[0].Filename) +} + +// reinsertDocument adds a second document on the existing device, for the control +// run that needs something to delete after the first document is gone. +func reinsertDocument(t *testing.T, database *DB, filename string) string { + t.Helper() + ctx := context.Background() + var deviceID string + if err := database.Read().QueryRowContext(ctx, + `SELECT id FROM devices LIMIT 1`).Scan(&deviceID); err != nil { + t.Fatalf("read device: %v", err) + } + docID := id.New(id.Document) + if _, err := gen.New(database.Write()).CreateDocument(ctx, gen.CreateDocumentParams{ + ID: docID, DeviceID: deviceID, BlobSha256: strings.Repeat("a", 64), + Filename: filename, Kind: "manual", State: "ready", + CreatedAt: Now(), UpdatedAt: Now(), + }); err != nil { + t.Fatalf("second document: %v", err) + } + return docID +} diff --git a/internal/doc/band_internal_test.go b/internal/doc/band_internal_test.go new file mode 100644 index 0000000..14561be --- /dev/null +++ b/internal/doc/band_internal_test.go @@ -0,0 +1,450 @@ +package doc + +import ( + "fmt" + "math" + "sort" + "testing" + "unicode/utf8" +) + +// TestABandCropDuplicatesThisMuchProse is the measurement [labelBand] was chosen on, +// re-run over both whole documents so the choice can be checked instead of +// remembered. +// +// A band crop prints as pixels whatever the paper set beside the drawing. The labels +// leave the block flow, so they are not duplicated; the rest is prose that is ALSO +// emitted as a text block, and a reader meets it twice. That is the cost, and the +// only question asked before this shipped was how large it is. +// +// Five rules, four of them refused. The shipped row calls [labelBand] itself rather +// than a copy of it, so this measures the pipeline and not a model of it. +// +// All five are measured over what [findFigures] finds, which is the geometry and not +// what a reader is served: [ServedFigures] removes the crops the band nests, and the +// overlapping-pair column below is what it removes them FROM. Keeping the two apart is +// deliberate — this measures a crop rule, and a serving decision folded into it would +// make every row a hybrid. +// +// drawn the drawing alone, which is what the crop was before this +// tight the drawing plus the labels the gate believes -- refused: it leaves +// `Кнопка сброса` and three more floating, the defect this fixes +// claims SHIPPED: the drawing plus every run the claim rule reaches +// column the printed column the drawing sits in, over that band of y +// full the page's full width over that band of y -- the first choice, refused +// because 59,618 of the columns manual's 67,662 duplicated runes are a +// NEIGHBOURING COLUMN, which on that document is a different language +func TestABandCropDuplicatesThisMuchProse(t *testing.T) { + for _, tc := range []struct { + name string + want map[bandRule]bandWant + }{ + // The safety document. All nine of its claims are false, so the gate keeps + // every one of them out of the alt text -- and the crop takes them in, which + // is what the 708-rune difference between `drawn` and `claims` is: page 22's + // eight lines of German prose about emptying the DryBOX, printed inside the + // picture the page prints them beside. + {columnsManual, map[bandRule]bandWant{ + bandDrawn: {crops: 59, dup: 773, foreign: 56, labels: 0, refused: 0, pairs: 0, cut: 0}, + bandTight: {crops: 59, dup: 773, foreign: 56, labels: 0, refused: 0, pairs: 0, cut: 0}, + bandClaims: {crops: 59, dup: 1481, foreign: 615, labels: 0, refused: 9, pairs: 0, cut: 0}, + bandColumn: {crops: 58, dup: 45736, foreign: 44870, labels: 0, refused: 9, pairs: 1, cut: 0}, + bandFull: {crops: 58, dup: 67662, foreign: 59618, labels: 0, refused: 9, pairs: 1, cut: 12}, + }}, + // 244 labels on the pages that carry a figure. The shipped rule prints every + // one of them and 45 of the 50 claims the gate refuses, which is the fix: + // those 45 include the labels docs/design/conversion.md recorded as + // unreachable after both ways of DRAWING them had been measured and refused. + {sequentialManual, map[bandRule]bandWant{ + bandDrawn: {crops: 195, dup: 51, foreign: 0, labels: 9, refused: 0, pairs: 0, cut: 16}, + bandTight: {crops: 195, dup: 116, foreign: 0, labels: 244, refused: 1, pairs: 27, cut: 22}, + bandClaims: {crops: 195, dup: 1714, foreign: 647, labels: 244, refused: 45, pairs: 35, cut: 48}, + bandColumn: {crops: 70, dup: 4355, foreign: 2238, labels: 244, refused: 39, pairs: 352, cut: 2}, + bandFull: {crops: 43, dup: 11377, foreign: 2713, labels: 244, refused: 44, pairs: 429, cut: 0}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + pages, ink := loadFigureInk(t, tc.name) + for _, rule := range []bandRule{bandDrawn, bandTight, bandClaims, bandColumn, bandFull} { + s := measureBands(pages, ink, rule) + t.Logf("\n%s", s.report(tc.name, rule)) + s.check(t, rule, tc.want[rule]) + } + }) + } +} + +// bandWant is one rule's cost over one document. +type bandWant struct { + // crops served, after crops that overlap are merged -- which only the two page + // bands do, because only they routinely produce a crop that is another crop. + crops int + // dup is runes of prose the crop prints that a block also prints; foreign is how + // many of those belong to a different printed column from any figure on the page. + dup, foreign int + // labels is carried labels the crop contains, refused is claims the gate turned + // down that it contains anyway. + labels, refused int + // pairs of crops that overlap, and runs a crop edge cuts. + pairs, cut int +} + +type bandRule int + +const ( + // bandDrawn is what the crop was before this: the drawing exactly. + bandDrawn bandRule = iota + // bandTight is the drawing plus the labels [figureLabels] keeps. + bandTight + // bandClaims is SHIPPED. It calls [labelBand]. + bandClaims + // bandColumn is the printed column the drawing sits in, over the band of y the + // drawing and its labels occupy. + bandColumn + // bandFull is the whole page width over that same band of y. + bandFull +) + +func (r bandRule) String() string { + switch r { + case bandDrawn: + return "drawn, as it was" + case bandTight: + return "drawing + gated labels" + case bandClaims: + return "drawing + every claim (SHIPPED)" + case bandColumn: + return "the printed column" + default: + return "the full page width" + } +} + +// merges reports whether this rule can produce one crop that is another crop, which +// is true only of the two page bands: two drawings side by side occupy the same band +// of y, so serving both would show a reader the same pixels twice. +func (r bandRule) merges() bool { return r == bandColumn || r == bandFull } + +// bandStats is what one rule costs, kept per page so a total can be attributed. +type bandStats struct { + figures, crops int + dupRunes, pageRunes, foreignRunes map[int]int + rescued, cutLabels, carriedTotal map[int]int + refusedTotal, overlapPairs map[int]int + cutRuns map[int]int + areaShare map[int]float64 + pagesWithCrops map[int]bool +} + +func measureBands(pages []PageRuns, ink [][]Ink, rule bandRule) *bandStats { + s := &bandStats{ + dupRunes: map[int]int{}, pageRunes: map[int]int{}, foreignRunes: map[int]int{}, + rescued: map[int]int{}, cutLabels: map[int]int{}, carriedTotal: map[int]int{}, + refusedTotal: map[int]int{}, overlapPairs: map[int]int{}, cutRuns: map[int]int{}, + areaShare: map[int]float64{}, pagesWithCrops: map[int]bool{}, + } + for i := range pages { + p := &pages[i] + var dropped DroppedRuns + text := usableRuns(p.Runs, p.Width, p.Height, &dropped) + for j := range text { + s.pageRunes[p.No] += utf8.RuneCountInString(text[j].Text) + } + drawn := onPageInk(ink[i], p.Width, p.Height) + marks := marksOf(drawn) + figs := findFigures(ink[i], p, defaultGuards) + if len(figs) == 0 { + continue + } + s.figures += len(figs) + cols := pageColumns(p) + + // The labels a figure ACTUALLY carries, read off the figures rather than + // recomputed: [absorbNested] moves an absorbed figure's labels onto the one + // that swallowed it, so recomputing from the survivors' drawn boxes would lose + // exactly the labels the absorption preserved. + var carriedAll, refusedAll []*TextRun + for k := range figs { + for b := range figs[k].LabelBoxes { + for i := range text { + if runBox(&text[i]) == figs[k].LabelBoxes[b] && !claims(carriedAll, &text[i]) { + carriedAll = append(carriedAll, &text[i]) + } + } + } + } + + var bands []CellRect + for k := range figs { + area := figs[k].InkRect + _, refused := claimSides(area, text, marks, defaultGuards) + for _, r := range refused { + if !claims(refusedAll, r) && !claims(carriedAll, r) { + refusedAll = append(refusedAll, r) + } + } + // The figure's OWN labels, not every label its band happens to hold: the + // tight rule is "the drawing plus the labels the gate kept FOR IT", and + // unioning a neighbour's would measure a different rule under its name. + want := area + for _, b := range figs[k].LabelBoxes { + want = unionRect(want, b) + } + bands = append(bands, bandFor(rule, want, area, cols, text, marks, p)) + } + for a := range bands { + for b := a + 1; b < len(bands); b++ { + if boxOverlap(bands[a], bands[b]) > 0 { + s.overlapPairs[p.No]++ + } + } + } + if rule.merges() { + bands = mergeBands(bands) + } + s.crops += len(bands) + s.pagesWithCrops[p.No] = true + for _, b := range bands { + s.areaShare[p.No] += b.Width() * b.Height() / (p.Width * p.Height) + } + + for j := range text { + box := runBox(&text[j]) + for _, b := range bands { + if o := boxOverlap(box, b); o > 0 && o < 1 { + s.cutRuns[p.No]++ + break + } + } + } + for j := range text { + r := &text[j] + if !inAnyBand(bands, runBox(r)) { + continue + } + if claims(carriedAll, r) { + continue // it left the flow, so the picture is its only copy + } + s.dupRunes[p.No] += utf8.RuneCountInString(r.Text) + if claims(refusedAll, r) { + s.rescued[p.No]++ + } + if foreignColumn(r, figs, cols) { + s.foreignRunes[p.No] += utf8.RuneCountInString(r.Text) + } + } + for _, r := range carriedAll { + s.carriedTotal[p.No]++ + if !inAnyBand(bands, runBox(r)) { + s.cutLabels[p.No]++ + } + } + s.refusedTotal[p.No] += len(refusedAll) + } + return s +} + +// pageColumns is the page's printed columns, the same two-step question +// [readingGroups] asks: the layout gates first, the reading gates where those +// decline. +func pageColumns(p *PageRuns) []Column { + cols := DetectColumns(p.Runs, p.Width, p.Height).Columns + if len(cols) < 2 { + cols = readingStrips(p.Runs, p.Width, p.Height) + } + return cols +} + +// claimSides splits a figure's claims into the ones [figureLabels] keeps and the ones +// its clean-corridor gate refuses. +func claimSides(area CellRect, text []TextRun, marks []CellRect, g figureGuards) (carried, refused []*TextRun) { + for side := range 4 { + claimed := claimLabels(area, text, marks, side, g) + if len(claimed) == 0 { + continue + } + _, ok := labelExtent(area, text, claimed, side) + for _, r := range claimed { + if ok { + if !claims(carried, r) { + carried = append(carried, r) + } + continue + } + if !claims(refused, r) { + refused = append(refused, r) + } + } + } + // A run carried from one side and refused from another is carried. + kept := refused[:0] + for _, r := range refused { + if !claims(carried, r) { + kept = append(kept, r) + } + } + return carried, kept +} + +func bandFor(rule bandRule, want, area CellRect, cols []Column, text []TextRun, + marks []CellRect, p *PageRuns) CellRect { + switch rule { + case bandDrawn: + return area + case bandTight: + return snapBandToLines(want, text) + case bandClaims: + return labelBand(area, text, marks, defaultGuards) + } + b := CellRect{X0: 0, Y0: want.Y0, X1: p.Width, Y1: want.Y1} + if rule == bandColumn { + if c := columnHolding(area, cols); c != nil { + b.X0, b.X1 = math.Min(c.Min, want.X0), math.Max(c.Max, want.X1) + } + } + return snapBandToLines(b, text) +} + +// columnHolding is the printed column the drawing sits inside, or nil where the page +// has none or the drawing straddles two. +func columnHolding(area CellRect, cols []Column) *Column { + for i := range cols { + if area.X0 >= cols[i].Min-1 && area.X1 <= cols[i].Max+1 { + return &cols[i] + } + } + return nil +} + +// mergeBands unions the crops that overlap, to a fixpoint. +func mergeBands(in []CellRect) []CellRect { + out := append([]CellRect(nil), in...) + for again := true; again; { + again = false + for i := 0; i < len(out) && !again; i++ { + for j := i + 1; j < len(out); j++ { + if boxOverlap(out[i], out[j]) <= 0 { + continue + } + out[i] = unionRect(out[i], out[j]) + out = append(out[:j], out[j+1:]...) + again = true + break + } + } + } + return out +} + +// inAnyBand reports whether most of a box is printed by one of the crops. Half, +// because a run mostly inside is legible in the picture. +func inAnyBand(bands []CellRect, r CellRect) bool { + for _, b := range bands { + if boxOverlap(r, b) >= 0.5 { + return true + } + } + return false +} + +// foreignColumn reports whether a run sits in a different printed column from every +// figure on the page. On the columns manual that is a different LANGUAGE, which is +// what refuses the two page bands. +func foreignColumn(r *TextRun, figs []Figure, cols []Column) bool { + if len(cols) < 2 { + return false + } + k := columnOf(r, cols) + if k < 0 { + return false + } + for i := range figs { + if c := columnHolding(figs[i].InkRect, cols); c != nil && c == &cols[k] { + return false + } + } + return true +} + +// bandTotal is one rule's cost summed over the pages that serve a crop. +type bandTotal struct { + dup, page, foreign int + rescued, cutLabels, carried int + refused, pairs, cutRuns int +} + +func (s *bandStats) totals() bandTotal { + var t bandTotal + for p := range s.pagesWithCrops { + t.dup += s.dupRunes[p] + t.page += s.pageRunes[p] + t.foreign += s.foreignRunes[p] + t.rescued += s.rescued[p] + t.cutLabels += s.cutLabels[p] + t.carried += s.carriedTotal[p] + t.refused += s.refusedTotal[p] + t.pairs += s.overlapPairs[p] + t.cutRuns += s.cutRuns[p] + } + return t +} + +func (s *bandStats) check(t *testing.T, rule bandRule, want bandWant) { + t.Helper() + t2 := s.totals() + for _, c := range []struct { + what string + got, want int + }{ + {"crops served", s.crops, want.crops}, + {"runes duplicated", t2.dup, want.dup}, + {"of them another column's", t2.foreign, want.foreign}, + {"carried labels the crop prints", t2.carried - t2.cutLabels, want.labels}, + {"refused claims the crop prints", t2.rescued, want.refused}, + {"overlapping crop pairs", t2.pairs, want.pairs}, + {"runs cut by a crop edge", t2.cutRuns, want.cut}, + } { + if c.got != c.want { + t.Errorf("%s: %s is %d, expected %d", rule, c.what, c.got, c.want) + } + } + // The invariant that replaced carrying a position: what the alt text says is in + // the picture has to be in the picture. + if rule == bandClaims && t2.cutLabels != 0 { + t.Errorf("%d of the %d carried labels lie outside the crop that is supposed "+ + "to print them", t2.cutLabels, t2.carried) + } +} + +func (s *bandStats) report(name string, rule bandRule) string { + v := s.totals() + out := fmt.Sprintf("%s / %s: %d figures -> %d crops served\n", name, rule, s.figures, s.crops) + out += fmt.Sprintf(" over the %d pages that serve a crop: %d runes duplicated of %d printed (%.1f%%), %d another column's\n", + len(s.pagesWithCrops), v.dup, v.page, 100*float64(v.dup)/math.Max(1, float64(v.page)), v.foreign) + out += fmt.Sprintf(" labels: %d of %d carried printed by the crop; %d of %d refused claims printed\n", + v.carried-v.cutLabels, v.carried, v.rescued, v.refused) + out += fmt.Sprintf(" %d overlapping crop pairs; %d runs cut by a crop edge\n", v.pairs, v.cutRuns) + + type row struct { + page, dup, total int + share float64 + } + var rows []row + for p := range s.pagesWithCrops { + if s.pageRunes[p] == 0 { + continue + } + rows = append(rows, row{p, s.dupRunes[p], s.pageRunes[p], + float64(s.dupRunes[p]) / float64(s.pageRunes[p])}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].share > rows[j].share }) + out += " worst pages by share of the page duplicated:\n" + for i, r := range rows { + if i >= 5 { + break + } + out += fmt.Sprintf(" page %3d: %5d of %5d runes (%.0f%%), crops cover %.2f of the page\n", + r.page, r.dup, r.total, 100*r.share, s.areaShare[r.page]) + } + return out +} diff --git a/internal/doc/bidi.go b/internal/doc/bidi.go new file mode 100644 index 0000000..90a0245 --- /dev/null +++ b/internal/doc/bidi.go @@ -0,0 +1,368 @@ +package doc + +import ( + "strings" + + "golang.org/x/text/unicode/bidi" +) + +// Right-to-left text arrives from `pdftohtml -xml` in VISUAL order, and this file +// puts it back into the order the page is written in. +// +// It is a defect in the pipeline rather than a limitation of it, and it bit twice: +// a Hebrew section read backwards on screen, and — because the same text is what +// the search index holds — a Hebrew word was findable only if it was typed +// backwards. One fix, at the one place a line's order is decided, repairs both. +// +// # What the tool actually returns +// +// Two separate reversals, and missing either one leaves the line wrong. Page 185 of +// the sequential manual, its Hebrew safety section, is the worked example. +// +// The runes inside a run are reversed: the run reads `שומיש תולבגה` where the page +// prints `הגבלות שימוש`, "usage restrictions". +// +// And the RUNS THEMSELVES are in visual order along the line. That page's second +// paragraph is three runs at x=89, x=643 and x=653 — a chunk of Hebrew, the digit 8, +// and another chunk of Hebrew — and the line begins at the RIGHT, so the run at 653 +// is the first thing read and the run at 89 the last. Joining them left to right, +// which is what every other line in these documents wants, interleaves the sentence. +// +// # Why not simply reverse the string +// +// Because a right-to-left line carries left-to-right islands, and they are not +// reversed on the page: "8" is printed "8" inside Hebrew prose, not "8" backwards, +// and a Latin product name reads forwards. Reversing the whole line would turn `8` +// into `8` harmlessly and `MopExtend` into `dnetxEpoM` — which is why the reversal +// is followed by putting each left-to-right island back the way it was. That is the +// standard visual-to-logical reading, and it is checked rather than assumed: see +// [visualToLogical]'s own note for what it reproduces. +// +// # The reference this was measured against +// +// `pdftotext` reads the same bytes with different code and gets the logical order +// right, wrapping it in the bidi controls U+202B and U+202C. So every line here has +// a free second opinion, which is the same stance internal/verify takes, and the +// check that measures this defect — 32 pages and 8,120 words reported reversed on +// the sequential manual — is the one that says whether the fix worked. It is also +// what found the two defects in the first version of this file: see +// [lineIsRightToLeft] and [leftToRightIsland]. +// +// # What this does NOT fix, measured +// +// **Arabic is unshaped**, in both tools. It arrives in isolated letter forms rather +// than the presentation forms the page prints, and `pdftotext` does no better — +// `السالمة` where the page prints `السلامة`, in both readings. That is a property of +// how the font maps its glyphs and is not an ordering question, so putting the order +// right is all this can do for Arabic. It is still worth doing: the words are now in +// the order they are read in, so search finds them. +// +// **Brackets come out right, and that is measured rather than assumed.** A bracket is +// bidi class ON, so it is not an island and reverses with the text around it — which +// is correct here, because poppler emits the glyph the page DRAWS. On the sequential +// manual's page 190 a parenthesised aside arrives as `)םייפיצפס םירוזאב קר ןימז(`, +// closing glyph first, and reversing puts the opening one back at the start. Five +// such runs on pages 189 and 190; none needs mirroring. +// +// **The language signals were never affected and are not changed here.** They count +// characters, and a reversed string has the same characters; the printed page tag +// already strips bidi controls for the reason [stripFormatting] gives. What was +// wrong was the readable text, and therefore search and, later, translation. + +// IsRightToLeftLanguage reports whether a language is written right to left. +// +// The scripts these documents actually contain, and no more: a list is honest about +// what has been seen, where a table of every RTL script would be a claim about +// documents this project has never read. Hebrew and Arabic are the sequential +// manual's; the others are here because they are the rest of the right-to-left world +// a household might plausibly configure, and because getting one wrong costs a +// section rather than a line. +func IsRightToLeftLanguage(lang string) bool { + switch BaseLanguage(lang) { + case "he", "ar", "fa", "ur", "ps", "sd", "ug", "yi", "dv", "ku", "arc": + return true + } + return false +} + +// lineIsRightToLeft reports whether a line's base direction is right to left. +// +// The REGION's language decides it, and counting the line's own characters is the +// fallback. That order is the correction to what shipped first, and six lines of the +// sequential manual are why. +// +// The first version counted strong characters and took the majority — which cannot +// be the Unicode algorithm's P2 rule, since P2 wants the first character in LOGICAL +// order and logical order is precisely what has been lost. Its header claimed the +// majority "agrees with P2 on every line of both documents". That was wrong, and it +// was wrong in the way that matters: a Hebrew line carrying a URL has more Latin than +// Hebrew, so it was read left to right and never repaired. Page 188 prints +// +// https://global.dreametech.com/pages/user-manuals-and-faqs :האבה תבותכב ןייעל שי +// +// — about 55 Latin letters against 30 Hebrew. The same shape appears on 204 (its +// Arabic twin), `Dreamehome תייצקלפא` on 191, `Dreamehome App قيبطت ليزنت` on 207, +// and a Wi-Fi label on 189 and 205. Those six lines were every word the verifier +// still reported as reversed, and the one Hebrew query that still had to be typed +// backwards to find anything. +// +// The region's language is the right authority because it is a document-wide answer +// to a question one line cannot settle, and because the whole probe exists to +// establish it. A line only ever gets this treatment inside a region a language was +// named for. +// +// A line with NO right-to-left character is left alone whatever its region says, and +// that guard is load-bearing rather than defensive: a pure-Latin line is entirely one +// island, so reversing its runes is a no-op, but reversing the ORDER OF ITS RUNS is +// not — a two-run Latin line in a Hebrew region would come out backwards. That is the +// case the mutation testing on this file found by accident, when a one-run control +// line read identically in both directions and proved nothing. +func lineIsRightToLeft(runs []TextRun, rtlRegion bool) bool { + var rtl, ltr int + for i := range runs { + for _, r := range runs[i].Text { + switch p, _ := bidi.LookupRune(r); p.Class() { + case bidi.R, bidi.AL: + rtl++ + case bidi.L: + ltr++ + } + } + } + if rtl == 0 { + return false + } + if rtlRegion { + return true + } + return rtl >= ltr +} + +// visualToLogical turns one visually-ordered right-to-left string into the order it +// is written in: reverse it, then put every left-to-right island back. +// +// An island is a stretch of characters that runs left to right inside +// right-to-left text — Latin letters, European and Arabic-Indic digits, and the +// separators that belong to a number — plus a space between two of them, so that +// `Dreame L40 Ultra` survives as one island instead of three. +// +// Checked against `pdftotext` on the sequential manual's Hebrew page 185 and Arabic +// page 201: every line matches the reference's reading, including the `8` in +// `אין לתת לילדים מתחת לגיל 8`, where a naive whole-string reversal is +// indistinguishable on one digit and wrong on two. +func visualToLogical(s string) string { + rs := []rune(s) + for i, j := 0, len(rs)-1; i < j; i, j = i+1, j-1 { + rs[i], rs[j] = rs[j], rs[i] + } + for i := 0; i < len(rs); { + if !leftToRightIsland(rs, i) { + i++ + continue + } + j := i + for j < len(rs) && leftToRightIsland(rs, j) { + j++ + } + for a, b := i, j-1; a < b; a, b = a+1, b-1 { + rs[a], rs[b] = rs[b], rs[a] + } + i = j + } + return string(rs) +} + +// leftToRightIsland reports whether the rune at i runs left to right inside +// right-to-left text. +// +// A space counts only when the runes on BOTH sides of it do, and the second half of +// that was a real defect rather than a refinement. Requiring only the next rune — +// which is what keeps `Dreame L40 Ultra` one island — also swallows the space that +// SEPARATES the island from the right-to-left word before it, so the space came out +// on the wrong side of the island and, once [collapseSpaces] had run, the word and +// the island were one token: `מכשירMopExtend` for a page printing +// `מכשיר MopExtend`. A lost word boundary is a lost search hit, which is most of +// what this file exists to repair. +// +// It bites on real pages and only where poppler emits both scripts in ONE run, which +// is why page 185 never showed it and the pdftotext comparison could not see it: the +// digit there is a run of its own. Measured over the sequential manual's +// right-to-left pages, the runs that mix scripts are page 188's three +// (`Class 1 רזייל`, `IEC 60825-1:2014`), page 189's `Wi-Fi ןווחמ`, and one each on +// the Arabic pages 201 and 202. Found by the agent writing this file's tests, from a +// case the brief did not list. +func leftToRightIsland(rs []rune, i int) bool { + switch p, _ := bidi.LookupRune(rs[i]); p.Class() { + case bidi.L, bidi.EN, bidi.AN, bidi.ES, bidi.ET, bidi.CS: + return true + } + if rs[i] == ' ' && i > 0 && i+1 < len(rs) { + return strongLeftToRight(rs[i-1]) && strongLeftToRight(rs[i+1]) + } + return false +} + +// strongLeftToRight reports whether a rune is a letter or digit that reads left to +// right — what a space has to sit between to belong to an island. +func strongLeftToRight(r rune) bool { + switch p, _ := bidi.LookupRune(r); p.Class() { + case bidi.L, bidi.EN, bidi.AN: + return true + } + return false +} + +// joinRunsRightToLeft is [joinRuns] for a line that reads right to left: the runs are +// taken from the rightmost, EXCEPT that a stretch of runs which is entirely +// left-to-right keeps its own order. +// +// That exception is [leftToRightIsland] one level up, and it is not hypothetical — it +// was a regression this file shipped and the verifier caught. A left-to-right island +// can span many runs, because poppler splits a run at a font change: page 204 of the +// sequential manual sets the punctuation of +// +// https://global.dreametech.com/pages/user-manuals-and-faqs +// +// in one font and the words in another, so that URL arrives as SEVENTEEN runs, broken +// at every `:`, `/`, `.` and `-`. Reversing the order of a line's runs is right for +// the Arabic prose around it and wrong for those seventeen, whose visual order already +// IS their logical order: the line came out reading +// `faqs-and- manuals-user/pages/com.dreametech.global://https`. +// +// Page 188's Hebrew twin never showed it, because there the same URL is a single run — +// the same reason the character-level version of this bug hid from the pdftotext +// comparison. A line is not a reliable witness to how poppler will cut it up. +// +// # Where this rule stops, and why it stops there +// +// A run of only digits at the OUTER end of an island goes with the right-to-left text, +// because the both-sides rule sees a left-to-right letter on one side of it only. That +// is deliberate, and the reason is not the one first written here. +// +// The first reason given was that no page prints the shape. It was wrong, and the +// correction came from scanning every line of both manuals that holds a right-to-left +// character — 1,136 of them — for exactly it. Three lines have it, all in the sequential +// manual, and **all three come out right**: +// +// page 196 `GHz`, ` `, `5` -> `בחיבור Wi-Fi של 5 GHz` +// page 205 `AI`, ` `, `IR`, ` .`, `11` -> `11. AI IR كاميرا` +// page 205 `AI`, ` `, `HD`, ` .`, `12` -> `12. AI HD كامير` +// +// So the rule is right where the digits LEAD their phrase — a quantity, a list marker — +// and wrong only where they TRAIL a Latin token, which was a synthetic line: Arabic, +// then `A`, then `11:2021` as its own run. That case is structural rather than absent: a +// trailing number belongs to the Latin token beside it and a printer sets it in the same +// run, which is precisely what page 204 does with `A11:2021`. So at run level the wrong +// case is unreachable, and reaching it would need a mixed-run split at character level. +// +// Kept as a limit with that reason, because the shape a document prints is a fact to be +// measured and not a claim to be made — this comment made the claim and the measurement +// contradicted it. +// +// The runs slice is not reordered — the caller's geometry is computed from it and +// every other reader of a line wants it left to right. Only the text is built this +// way round. +func joinRunsRightToLeft(runs []TextRun) string { + order := make([]int, 0, len(runs)) + // Which runs are emitted in the order they are printed. Those must NOT go through + // [visualToLogical]: they are already in logical order, and for a run holding only + // digits and punctuation the character-level repair is not the identity — it moves + // a space that has a strong neighbour on one side only. Page 204's ` 60825- 1:2014/` + // came out `1:2014/ 60825- `, halves transposed at the `- 1`, which is how the + // laser standard `IEC 60825-1:2014/EN 60825-1:2014/A11:2021` lost a space and + // swapped two of its parts. + printed := make([]bool, len(runs)) + for i := len(runs) - 1; i >= 0; { + if hasRightToLeft(runs[i].Text) { + order = append(order, i) + i-- + continue + } + // A maximal stretch of runs holding no right-to-left letter. Its island is the + // part BETWEEN the outermost runs that hold a left-to-right letter; a run of + // only digits, punctuation or spaces at either end belongs to the + // right-to-left text beside it and reverses with it. That is the both-sides + // rule [leftToRightIsland] applies to characters, applied to runs, and the + // second half of it is what a printed list marker needs. + j := i + for j >= 0 && !hasRightToLeft(runs[j].Text) { + j-- + } + lo, hi := j+1, i + first, last := -1, -1 + for k := lo; k <= hi; k++ { + if hasLeftToRight(runs[k].Text) { + if first < 0 { + first = k + } + last = k + } + } + switch { + case first < 0: + // Nothing left-to-right anywhere in the stretch: it is all neutral, so all + // of it reverses. + for k := hi; k >= lo; k-- { + order = append(order, k) + } + default: + for k := hi; k > last; k-- { + order = append(order, k) + } + for k := first; k <= last; k++ { + order = append(order, k) + printed[k] = true + } + for k := first - 1; k >= lo; k-- { + order = append(order, k) + } + } + i = j + } + + var b strings.Builder + for n, i := range order { + if n > 0 { + prev := &runs[order[n-1]] + cur := &runs[i] + // Whichever way round the two sit on the page, the gap is between their + // facing edges: reading order and left-to-right order are not the same + // thing here, so the subtraction cannot assume one of them. + gap := cur.X - prev.right() + if cur.X < prev.X { + gap = prev.X - cur.right() + } + if gap > 0 && !endsWithSpace(prev.Text) && !startsWithSpace(cur.Text) { + b.WriteByte(' ') + } + } + if printed[i] { + b.WriteString(runs[i].Text) + } else { + b.WriteString(visualToLogical(runs[i].Text)) + } + } + return collapseSpaces(b.String()) +} + +// hasLeftToRight reports whether a string carries any left-to-right letter — what an +// island has to be anchored by at both ends. +func hasLeftToRight(s string) bool { + for _, r := range s { + if p, _ := bidi.LookupRune(r); p.Class() == bidi.L { + return true + } + } + return false +} + +// hasRightToLeft reports whether a string carries any right-to-left letter. +func hasRightToLeft(s string) bool { + for _, r := range s { + switch p, _ := bidi.LookupRune(r); p.Class() { + case bidi.R, bidi.AL: + return true + } + } + return false +} diff --git a/internal/doc/bidi_internal_test.go b/internal/doc/bidi_internal_test.go new file mode 100644 index 0000000..4a003c8 --- /dev/null +++ b/internal/doc/bidi_internal_test.go @@ -0,0 +1,446 @@ +package doc + +import ( + "strings" + "testing" +) + +// Hermetic tests for the visual-to-logical repair in bidi.go. No poppler and no +// PDF: every string here is written out in both orders, so what the tool returns +// and what the page prints can be read side by side. +// +// The strings are the ones bidi.go's header measured on the sequential manual's +// Hebrew page 185 and Arabic page 201. They are written the way Go source holds +// them — a rune sequence — so `שומיש תולבגה` below is the VISUAL reading, the runes +// in the order poppler emits them, and `הגבלות שימוש` is what the page prints. +// An editor that reorders bidi text for display makes the two look confusingly +// alike; the tests compare runes, which do not care. +// +// Arabic is unshaped in both tools — isolated letter forms rather than the +// presentation forms the page prints — and that is a property of the font's glyph +// map, not of the ordering. The Arabic cases below are therefore written in the +// same unshaped form the pipeline actually sees. + +// TestVisualToLogicalReproducesTheReferenceReading is the reference case from +// bidi.go's header: the run reads `שומיש תולבגה` where the page prints +// `הגבלות שימוש`, "usage restrictions". +// +// The run-level cases are the ones the pipeline actually feeds this function: +// [joinRunsRightToLeft] calls it once per run, and on page 185 the digit is a run +// of its own. +func TestVisualToLogicalReproducesTheReferenceReading(t *testing.T) { + for _, tc := range []struct { + name string + visual, prints string + }{ + {"the header's worked example, page 185", "שומיש תולבגה", "הגבלות שימוש"}, + {"a Hebrew heading", "תוחיטב תוארוה", "הוראות בטיחות"}, + {"Arabic, unshaped in both tools, page 201", "ةمالسلا تاداشرإ", "إرشادات السلامة"}, + + {"a run that is only a digit", "8", "8"}, + {"a run that is only a two-digit number", "10", "10"}, + {"a run that is only a Latin product name", "MopExtend", "MopExtend"}, + {"a run that is only a Latin phrase", "Dreame L40 Ultra", "Dreame L40 Ultra"}, + + {"an empty run", "", ""}, + {"a run of spaces", " ", " "}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := visualToLogical(tc.visual); got != tc.prints { + t.Errorf("visualToLogical(%q)\n = %q\nwant %q", tc.visual, got, tc.prints) + } + }) + } +} + +// TestALeftToRightIslandIsNotReversed is the reason the repair is not a whole-string +// reversal. `8` is printed `8` inside Hebrew prose and `MopExtend` reads forwards, so +// each island has to be put back the way it was. +// +// Stated as "the island survives and its reversal does not appear" rather than as a +// whole expected string, because the island's own SPACING is a separate and currently +// wrong thing — see TestAnIslandKeepsTheSpaceThatSeparatesIt, which was that defect. +// +// One digit is not enough to see this: a naive whole-string reversal is +// indistinguishable on `8` and wrong on `10`, which is why both are here. +func TestALeftToRightIslandIsNotReversed(t *testing.T) { + for _, tc := range []struct { + name string + visual string + want string // the island, forwards + reversed string // what a whole-string reversal would have produced + }{ + {"one digit, indistinguishable from a naive reversal", "8 ליגל תחתמ", "8", ""}, + {"two digits, where a naive reversal shows", "10 ליגל תחתמ", "10", "01"}, + {"two digits, interior", "ליגל 21 תחתמ", "21", "12"}, + {"a Latin product name", "תשרבמ MopExtend רישכמ", "MopExtend", "dnetxEpoM"}, + {"a Latin phrase stays one island", "שדח Dreame L40 Ultra רישכמ", "Dreame L40 Ultra", "artlU 04L emaerD"}, + } { + t.Run(tc.name, func(t *testing.T) { + got := visualToLogical(tc.visual) + if !strings.Contains(got, tc.want) { + t.Errorf("visualToLogical(%q)\n = %q\ndoes not contain the island %q", tc.visual, got, tc.want) + } + if tc.reversed != "" && strings.Contains(got, tc.reversed) { + t.Errorf("visualToLogical(%q)\n = %q\nholds %q — the island was reversed with the line", + tc.visual, got, tc.reversed) + } + // The Hebrew around the island is reversed, which is the other half: the + // island rule must not have exempted the whole line. + if strings.Contains(got, "תחתמ") { + t.Errorf("visualToLogical(%q) = %q still reads the Hebrew visually", tc.visual, got) + } + }) + } +} + +// TestATrailingSpaceIsNotDraggedIntoAnIsland is [leftToRightIsland]'s own note: a +// space counts as part of an island only when the next rune does too, so +// `Dreame L40 Ultra` survives as one island while a space at the end of one does not +// join it. +func TestATrailingSpaceIsNotDraggedIntoAnIsland(t *testing.T) { + for _, tc := range []struct { + name string + rs string + i int + want bool + }{ + {"a Latin letter", "MopExtend", 0, true}, + {"a European digit", "8", 0, true}, + {"an Arabic-Indic digit", "٨", 0, true}, + {"a Hebrew letter", "ם", 0, false}, + {"an Arabic letter", "ا", 0, false}, + + {"a space between two Latin words", "L40 Ultra", 3, true}, + {"a space between a digit and a letter", "8 x", 1, true}, + {"a space before Hebrew ends the island", "10 ם", 2, false}, + {"a space at the end of the string ends the island", "10 ", 2, false}, + } { + t.Run(tc.name, func(t *testing.T) { + rs := []rune(tc.rs) + if got := leftToRightIsland(rs, tc.i); got != tc.want { + t.Errorf("leftToRightIsland(%q, %d) = %v, want %v", tc.rs, tc.i, got, tc.want) + } + }) + } +} + +// TestAnIslandKeepsTheSpaceThatSeparatesIt is the defect this test was written to +// pin, now fixed rather than pinned. +// +// [leftToRightIsland] used to take a space whose NEXT rune was left to right — which +// is what keeps `Dreame L40 Ultra` in one piece — wherever it sat, including where +// the rune before it was Hebrew. The island put back was then " MopExtend", the space +// landed on the wrong side of it, and after [collapseSpaces] the word before it was +// glued on: `מכשירMopExtend` for a page printing `מכשיר MopExtend`. That is a lost +// word boundary and therefore a lost search hit. +// +// A space now has to sit BETWEEN two left-to-right runes to belong to an island. The +// case is real rather than constructed: poppler emits both scripts in one run on +// five of the sequential manual's right-to-left pages — `Wi-Fi ןווחמ` on 189, +// `Class 1 רזייל` on 188 — and it is invisible on page 185, where every digit is a +// run of its own, which is why the pdftotext comparison never caught it. +func TestAnIslandKeepsTheSpaceThatSeparatesIt(t *testing.T) { + for _, tc := range []struct { + name string + visual string + want string + }{ + {"a Latin name between two Hebrew words", "תשרבמ MopExtend רישכמ", "מכשיר MopExtend מברשת"}, + {"a digit at the end of a Hebrew phrase", "8 ליגל תחתמ", "מתחת לגיל 8"}, + {"a hyphenated Latin name, from page 189", "ןווחמ Wi-Fi", "Wi-Fi מחוון"}, + {"a phrase whose own spaces must survive", "שדח Dreame L40 Ultra רישכמ", "מכשיר Dreame L40 Ultra חדש"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := visualToLogical(tc.visual); got != tc.want { + t.Errorf("visualToLogical(%q)\n = %q\nwant %q", tc.visual, got, tc.want) + } + }) + } +} +func TestVisualToLogicalIsItsOwnInverse(t *testing.T) { + for _, s := range []string{ + "שומיש תולבגה", + "הגבלות שימוש", + "תוחיטב תוארוה", + "ةمالسلا تاداشرإ", + "8", + "10", + "MopExtend", + "Dreame L40 Ultra", + "8 ליגל תחתמ", + "ליגל 21 תחתמ", + "תשרבמ MopExtend רישכמ", + "שדח Dreame L40 Ultra רישכמ", + "", + " ", + } { + t.Run(s, func(t *testing.T) { + once := visualToLogical(s) + if twice := visualToLogical(once); twice != s { + t.Errorf("visualToLogical twice over %q\n = %q\nby way of %q", s, twice, once) + } + }) + } +} + +// TestLineIsRightToLeftByMajorityOfTheStrongCharacters covers the direction test. +// By majority rather than by the first character, for the reason the function's own +// note gives: the Unicode P2 rule wants the first character in LOGICAL order, and +// logical order is what has been lost. +// +// The region's language now decides and the majority is the fallback, so each case +// states both answers. The row that made the change necessary is the URL line: its +// Latin outweighs its Hebrew, so the majority reads it left to right and the six +// lines like it across pages 188 to 207 were never repaired. See [lineIsRightToLeft]. +func TestLineIsRightToLeftByMajorityOfTheStrongCharacters(t *testing.T) { + for _, tc := range []struct { + name string + texts []string + want bool // with no region language to go on + wantInRTL bool // inside a region a right-to-left language was named for + }{ + {"Hebrew", []string{"שומיש תולבגה"}, true, true}, + {"Arabic, unshaped", []string{"ةمالسلا تاداشرإ"}, true, true}, + {"mostly Hebrew with a Latin island", []string{"תשרבמ MopExtend רישכמ"}, true, true}, + {"Hebrew and a digit across two runs", []string{"8", "ליגל תחתמ"}, true, true}, + + // Page 188: about 55 Latin letters of URL against 30 Hebrew. The majority gets + // this wrong, the region gets it right, and this row is the whole reason the + // region is asked first. + {"a Hebrew line carrying a URL", []string{ + "https://global.dreametech.com/pages/user-manuals-and-faqs :האבה תבותכב ןייעל שי", + }, false, true}, + {"Dreamehome in a Hebrew line, page 191", []string{"Dreamehome תייצקלפא"}, false, true}, + + {"Latin", []string{"Sicherheitshinweise"}, false, false}, + {"Cyrillic", []string{"Меры предосторожности"}, false, false}, + {"Greek", []string{"Οδηγίες ασφαλείας"}, false, false}, + {"Japanese", []string{"安全上のご注意"}, false, false}, + + // A line with no right-to-left character is left alone whatever its region + // says: reversing its runes is a no-op, but reversing the ORDER of its runs is + // not, so a two-run Latin line in a Hebrew region would come out backwards. + {"Latin inside a right-to-left region", []string{"Wi-Fi", "5 GHz"}, false, false}, + {"no strong characters at all", []string{"", " ", "10 – 22"}, false, false}, + {"nothing at all", nil, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + runs := make([]TextRun, len(tc.texts)) + for i, s := range tc.texts { + runs[i] = TextRun{Text: s} + } + if got := lineIsRightToLeft(runs, false); got != tc.want { + t.Errorf("lineIsRightToLeft(%q, no region language) = %v, want %v", + tc.texts, got, tc.want) + } + if got := lineIsRightToLeft(runs, true); got != tc.wantInRTL { + t.Errorf("lineIsRightToLeft(%q, right-to-left region) = %v, want %v", + tc.texts, got, tc.wantInRTL) + } + }) + } +} + +// TestARightToLeftLanguageIsNamedByItsBaseTag pins the list [IsRightToLeftLanguage] +// keeps, including that it reads a regional tag through BaseLanguage the way every +// other language decision in this package does. +func TestARightToLeftLanguageIsNamedByItsBaseTag(t *testing.T) { + for _, tc := range []struct { + lang string + want bool + }{ + {"he", true}, {"ar", true}, {"fa", true}, {"ur", true}, + {"he-IL", true}, {"ar-EG", true}, + {"de", false}, {"ru", false}, {"ja", false}, {"", false}, + {"pt-BR", false}, + } { + if got := IsRightToLeftLanguage(tc.lang); got != tc.want { + t.Errorf("IsRightToLeftLanguage(%q) = %v, want %v", tc.lang, got, tc.want) + } + } +} + +// TestJoinRunsRightToLeftTakesTheRunsFromTheRightmost uses the geometry bidi.go's +// header measured: page 185's second paragraph is three runs at x=89 (width 555), +// x=643 (width 9, the digit 8) and x=653 (width 207), and the line begins at the +// RIGHT, so the run at 653 is read first and the run at 89 last. +// +// The boxes overlap by a point where the widest run ends (89+555 = 644) and the digit +// begins (643), so no space is inserted there — the gap rule sees -1. That is the +// measured geometry and not a rounding of it, so the expected reading below carries +// the join it produces. +func TestJoinRunsRightToLeftTakesTheRunsFromTheRightmost(t *testing.T) { + runs := []TextRun{ + {X: 89, Y: 300, Width: 555, Height: 22, Text: "םידליל תתל ןיא"}, + {X: 643, Y: 300, Width: 9, Height: 22, Text: "8"}, + {X: 653, Y: 300, Width: 207, Height: 22, Text: "ליגל תחתמ"}, + } + const want = "מתחת לגיל 8אין לתת לילדים" + + got := joinRunsRightToLeft(runs) + if got != want { + t.Errorf("joinRunsRightToLeft\n = %q\nwant %q", got, want) + } + + // Reading order, stated separately so a failure says which half broke: the + // rightmost run's words come first and the leftmost run's last. + first, last := strings.Index(got, "מתחת"), strings.Index(got, "אין") + if first < 0 || last < 0 { + t.Fatalf("joinRunsRightToLeft = %q, missing one of the two Hebrew chunks", got) + } + if first > last { + t.Errorf("joinRunsRightToLeft = %q reads the run at x=89 before the one at x=653", got) + } + + // The runs slice itself must not be reordered: the caller's geometry is + // computed from it and every other reader of a line wants it left to right. + for i, wantX := range []float64{89, 643, 653} { + if runs[i].X != wantX { + t.Errorf("runs[%d].X = %g after the join, want %g — the slice was reordered", + i, runs[i].X, wantX) + } + } + if runs[1].Text != "8" { + t.Errorf("runs[1].Text = %q after the join, want the untouched visual %q", runs[1].Text, "8") + } +} + +// TestRegionBlocksReadsARightToLeftLineLogically is the wiring: the repair lives in +// one place, textLine.finish, so a block built from Hebrew runs must come out in the +// order the page is written in — that text is what the reader shows and what the +// search index holds. A left-to-right line in the same page must be untouched. +func TestRegionBlocksReadsARightToLeftLineLogically(t *testing.T) { + const ( + pageWidth = 918 + pageHeight = 620 + body = 17 + ) + // Two paragraphs on one page: Hebrew at the top, German lower down, far enough + // apart that the gap rule keeps them separate blocks. + // + // The German line is TWO runs on one baseline, which is what makes it a real + // control: a line of one run reads the same in either direction, so it would + // pass even if the direction test said every line was right to left. + hebrew := []string{"שומיש תולבגה", "תוחיטב תוארוה"} + + p := &PageRuns{No: 185, Width: pageWidth, Height: pageHeight} + y := 20.0 + for _, s := range hebrew { + p.Runs = append(p.Runs, TextRun{X: 200, Y: y, Width: 600, Height: body + 5, Text: s, + Font: Font{Size: body, Family: "Test-Face"}}) + y += 22 + } + y += 66 + for _, r := range []struct { + x, w float64 + text string + }{ + {55, 240, "Lesen Sie die Anleitung"}, + {300, 200, "vor der Verwendung"}, + } { + p.Runs = append(p.Runs, TextRun{X: r.x, Y: y, Width: r.w, Height: body + 5, Text: r.text, + Font: Font{Size: body, Family: "Test-Face"}}) + } + + got := RegionBlocks(p, &Region{Page: 185, X0: 0, X1: pageWidth, Lang: "he"}, nil, nil) + if len(got) != 2 { + t.Fatalf("got %d blocks, want the Hebrew paragraph and the German one: %+v", len(got), got) + } + + // Both lines are repaired, and their order down the page is unchanged: the + // repair is inside a line, not across them. + const wantHebrew = "הגבלות שימוש הוראות בטיחות" + if got[0].Text != wantHebrew { + t.Errorf("the Hebrew block reads\n %q\nwant %q", got[0].Text, wantHebrew) + } + for _, visual := range hebrew { + if strings.Contains(got[0].Text, visual) { + t.Errorf("the Hebrew block %q still holds the visual run %q", got[0].Text, visual) + } + } + + const wantGerman = "Lesen Sie die Anleitung vor der Verwendung" + if got[1].Text != wantGerman { + t.Errorf("the left-to-right block reads\n %q\nwant %q — it must not have been touched", + got[1].Text, wantGerman) + } +} + +// TestAMultiRunLeftToRightIslandKeepsItsOrder is a regression this file shipped and +// the verifier caught, so the geometry is the real one. +// +// pdftohtml splits a run at a font change, and page 204 of the sequential manual sets +// the punctuation of its support URL in one font and the words in another — so +// `https://global.dreametech.com/pages/user-manuals-and-faqs` arrives as SEVENTEEN +// runs, broken at every `:`, `/`, `.` and `-`. Reversing the order of a line's runs +// is right for the Arabic prose beside it and wrong for those seventeen, and the line +// came out `faqs-and- manuals-user/pages/com.dreametech.global://https`. +// +// Page 188's Hebrew twin prints the same URL as ONE run and never showed it, which is +// the same way the character-level version of this bug hid from the pdftotext +// comparison. A line is not a reliable witness to how poppler will cut it up. +func TestAMultiRunLeftToRightIslandKeepsItsOrder(t *testing.T) { + const url = "https://global.dreametech.com/pages/user-manuals-and-faqs" + parts := []string{"https", "://", "global", ".", "dreametech", ".", "com", "/", + "pages", "/", "user", "-", "manuals", "-", "and", "-", "faqs"} + + var runs []TextRun + x := 300.0 + for _, p := range parts { + w := float64(len([]rune(p))) * 6 + runs = append(runs, TextRun{X: x, Y: 100, Width: w, Height: 14, Text: p}) + x += w + } + // The Arabic prose sits to the right of the URL, as it does on the page, so it is + // read first. + const arabic = "لىإ لاقتنلاا ىجرُي" + runs = append(runs, TextRun{X: x + 8, Y: 100, Width: 200, Height: 14, Text: arabic}) + + got := joinRunsRightToLeft(runs) + if !strings.Contains(got, url) { + t.Errorf("joinRunsRightToLeft(...)\n = %q\ndoes not hold the URL %q in one piece", got, url) + } + if !strings.HasPrefix(got, visualToLogical(arabic)) { + t.Errorf("joinRunsRightToLeft(...)\n = %q\ndoes not begin with the Arabic, which is "+ + "the rightmost run and therefore read first", got) + } +} + +// TestANeutralRunAtTheEdgeOfAnIslandReversesWithTheText is the mirror of +// [TestAMultiRunLeftToRightIslandKeepsItsOrder], and the two together are why the +// island rule needs both its ends anchored. +// +// Keeping a whole stretch of non-right-to-left runs in printed order — which is what +// fixed the seventeen-run URL — froze a printed list marker too, because a run holding +// only `1` or ` .` has no right-to-left LETTER in it. Page 211 of the sequential +// manual lost the structure of an Arabic maintenance list that way: six list items +// became one paragraph, and 43 blocks went with it across six pages, because +// [leadingMarker] stops recognising `. 1`. +// +// The geometry is that page's line at top=171. Its runs are `)`, `LDS`, the Arabic, +// ` .` and `1`, and only the Arabic run holds a right-to-left letter. The island is +// the part BETWEEN the outermost runs carrying a left-to-right letter, so `LDS` is in +// it and `1`, ` .` and `)` are not. +func TestANeutralRunAtTheEdgeOfAnIslandReversesWithTheText(t *testing.T) { + runs := []TextRun{ + {X: 728, Y: 171, Width: 4, Height: 14, Text: ")"}, + {X: 732, Y: 171, Width: 20, Height: 14, Text: "LDS"}, + {X: 752, Y: 171, Width: 98, Height: 14, Text: "( رزيللاب ةفاسملا رعشتسم"}, + {X: 850, Y: 171, Width: 9, Height: 14, Text: " ."}, + {X: 859, Y: 171, Width: 6, Height: 14, Text: "1"}, + } + const want = "1. مستشعر المسافة بالليزر (LDS)" + if got := joinRunsRightToLeft(runs); got != want { + t.Errorf("joinRunsRightToLeft(...)\n = %q\nwant %q", got, want) + } + + // And the consequence that was actually lost: the marker is recognisable again, + // so the line is a list item rather than the middle of a paragraph. + l := textLine{runs: runs} + l.finish(true) + if marker, _ := leadingMarker(&l); marker != "1." { + t.Errorf("leadingMarker = %q, want \"1.\": the printed list marker is what the "+ + "43 lost blocks were", marker) + } +} diff --git a/internal/doc/blocks.go b/internal/doc/blocks.go new file mode 100644 index 0000000..afd14cd --- /dev/null +++ b/internal/doc/blocks.go @@ -0,0 +1,1540 @@ +package doc + +import ( + "fmt" + "math" + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +// A block is one piece of readable content: a heading, a paragraph, a list item. +// See docs/design/conversion.md for the contract this implements. +// +// Blocks are built one region at a time, and that is the whole funnel: a +// household that reads German gets the German column of a page, not the page. +// [Region] is the unit because it is the unit the language map already stores. +// +// Two things about reading order had to be settled here rather than taken from +// the contract, and the second contradicts the obvious reading of it. +// +// **Down then across, not across then down.** `pdftotext -layout` reflows two +// side-by-side tables on the sequential manual's page 20 into one interleaved +// block, and the contract names that as the mistake to avoid. +// +// **A region is not narrow enough to sort inside.** The contract says reading +// order comes from the region, and for a parallel-columns page it does — a boxed +// region IS one text column. But regions.md rule 3 deliberately stores a page of +// several same-language columns as ONE whole-page region, so on those pages +// sorting inside the region is exactly the interleaving mistake, committed under +// another name. Measured on the column manual's page 62: a whole-page German +// region holding two text columns of prose, whose baselines do not even line up +// across the gutter (y=102 against y=102, then y=118 against y=120). Sorting its +// runs by y then x yields "Die Verpackung schützt..." followed by "Gerät +// Garantie gemäß nachstehenden Bedingungen:" — the left column's second line +// followed by the right column's. The sequential manual has the same shape on the +// 199 pages that read as three columns. +// +// So a region is subdivided by [DetectColumns] before anything is sorted, and +// reading order is column by column, left to right, down then across within each. +// The runs that span a gutter — a banner heading set across the measure — are read +// first, since that is where a banner is. +// +// # What this deliberately does not do +// +// Recorded so the next person does not think these are unsolved by accident. All +// were seen while checking the output against 108 dpi renders of the column +// manual's pages 62 and 14 and the sequential manual's pages 23 and 24. +// +// **Page furniture is not identified HERE, and one page cannot identify it.** A +// printed language tab, a folio and a running head are text on the page, so +// nothing in this file can tell them from content: the sequential manual's "DE" +// tab is 11pt medium beside a 17pt body and classifies as a level-2 heading, its +// folio as a one-character paragraph. Nor is any single page's evidence enough to +// try — the sequential manual genuinely heads 28 pages with a bare "A" and 22 +// with a bare "D", so a one-letter line is not evidence of a tab. What identifies +// furniture is repetition in the same place across the pages of a section, which +// is a comparison between pages and not a property of one. That pass is +// [FindFurniture], it runs from [Convert] with the whole document in view, and +// what it finds arrives here as the `fur` argument. A caller passing nil gets this +// file's own reading, which is furniture and all. +// +// **Hyphenation is not undone.** See [joinRuns] for the German counter-example +// that makes a trailing hyphen ambiguous. +// +// **A two-line heading set with generous leading becomes two headings.** Measured +// on the column manual's page 14: "Trockensaugen mit der DryBOX" and +// "(Zyklon-Filtertechnologie)" sit 24 units apart on a 16-unit pitch, so the same +// gap rule that separates paragraphs separates them. Merging headings across a +// wider gap than prose would need a second threshold with nothing to measure it +// against, since the only case in either document is this one. +// +// **A table's rows are not reassembled beyond the cell.** A table is now read where +// its ruled lines say it is — see [cellRunsOfRegion] and [tableBlocks], which is the +// fix for the limitation this list used to record — but each cell is its own block +// and a row is only the run of blocks that share a row number in the note. Nothing +// here emits a row as one unit, because what a reader should be shown for a table is +// a question the reader answers and not this stage. +// +// **A table's prose is not split around it.** See [mergeTablesByDepth] for the case +// and for why neither manual exercises it. +// +// **A vertically merged cell is still dropped**, 10 of 47 on one measured page, and a +// header row whose top border is not drawn is still outside the table. Both are +// omissions in the cell walk rather than in this one, recorded in conversion.md. The +// header row is at least not lost: it falls outside every cell and so reads with the +// prose, which is where the column manual's page 57 puts it. + +// BlockKind is what a block is. +// +// A string for the same reason [Source] is one: it reaches a database column and +// a JSON payload, where "heading" survives a schema change and 1 does not. +type BlockKind string + +const ( + // BlockHeading is a line, or a few, that titles what follows. See + // [regionBody] for how it is told from body copy, which is the one decision + // in this file with a document-wide consequence. + BlockHeading BlockKind = "heading" + // BlockParagraph is running prose, its printed line breaks removed. + BlockParagraph BlockKind = "paragraph" + // BlockListItem is one item of a list: a line that opens with a marker, + // plus the lines indented under it. + BlockListItem BlockKind = "list-item" + + // BlockTable is one cell of a table recovered from the ruled lines — a different + // input from the text geometry, pdftocairo's strokes. Its note carries the cell's + // place in the grid. See [tableBlocks]. + BlockTable BlockKind = "table" + // BlockFigure is declared and never produced here: a figure needs the image list + // this pipeline does not read. It is named so that the kind vocabulary a reader + // and a database column see does not have to change when that work lands, and so + // that nothing downstream assumes the kinds above are all there will ever be. + BlockFigure BlockKind = "figure" +) + +// Block is one piece of readable content, in one language, on one page. +// +// The key is natural, not a surrogate: Page, RegionX0 and Index. Same reasoning +// as doc_regions, and the same reason — a job handler can run twice, and a +// surrogate ID would make the second conversion insert a parallel set instead of +// converging on the first. It is also what gives extraction the stable block IDs +// ingest.md asks for: "paragraph 4 of the German region of page 62" is a citation +// that survives a re-convert. +// +// RegionX0 rather than a region ID for the same reason [Region] has no ID: the +// left edge is what the page itself determines, so two runs of the pipeline over +// the same bytes agree on it without consulting anything stored. +type Block struct { + // Page is the 1-based page number in the original PDF — the number printed on + // the paper's own page furniture is a different thing and is not this. + Page int + // RegionX0 is the left edge of the region this came from, in the coordinate + // space [ExtractRuns] reports. 0 for a whole-page region. + RegionX0 float64 + // Index is the block's position within its region, from 0, in reading order. + Index int + + // Kind is what the block is. + Kind BlockKind + // Level is the heading level, 1 for the most prominent, and 0 for anything + // that is not a heading. + // + // It is derived from the region's own body face and reaches only two values: + // 1 for a heading set larger than the body, 2 for one set at the body size and + // merely heavier. That is as far as this evidence honestly goes. A document-wide + // outline — this manual has four heading sizes, so level 3 exists — needs the + // sizes of the whole document ranked together, which is a pass over every + // region and not a property of one. Inventing more levels from one region's two + // facts would put a number on the page that the next page contradicts. + Level int + + // Text is the block's content, with the printed line breaks removed: a break + // at the original measure is a property of the paper's column width, not of the + // content, and a reader renders at a different width. Runs on one line are + // joined the same way. See [joinRuns] for the one thing this loses. + Text string + // Lang is the region's language, empty where none was established. Carried on + // the block so that a block is self-describing once it leaves the page it came + // from — which is the state extraction and search will see it in. + Lang string + + // X0, X1, Y0 and Y1 are the block's own bounding box, not the region's. A + // caller wanting to draw the block on a `pdftoppm -r 108` render, which is what + // checking this work by eye needs, uses these. + X0, X1, Y0, Y1 float64 + // Lines is how many printed lines the block was folded from, and Chars its + // rune count. Runes, not bytes, for the reason [Region.Chars] gives. + Lines int + Chars int + // Note says in checkable terms why this block is the kind it is, or — when + // Furniture is set — why it is furniture. The same stance as [Region.Note] and + // [ColumnLayout.Note]: the evidence is countable and a reader can hold it + // against the page. + Note string + + // Furniture reports that this block is on the page because of where the page + // is, not because of what it says: a printed language tab, a folio, a running + // head. See [Furniture] for the rule and every threshold it rests on. + // + // Marked rather than dropped, and the reason is that the rule can be wrong. A + // marked block is evidence a person can look at and a query can count; a + // dropped one is a hole in a page that nothing downstream can tell from a + // conversion defect. What it costs is that every caller between here and a + // screen has to honour it, and the cost of getting THAT wrong is the defect + // unfixed rather than content destroyed. Nothing here filters: a conversion + // carries its furniture, [Conversion.ContentBlocks] is what a reader and an + // index want, and internal/verify counts the two apart on purpose so that a + // runaway rule shows up as lost coverage instead of hiding inside it. + Furniture bool + + // Callout reports that this block is one printed line of a figure's callout + // label, which [Figure.Labels] carries as text and a reader draws beside the + // picture. Marked rather than dropped, on exactly [Block.Furniture]'s reasoning. + // + // IT IS A SEPARATE FLAG FROM Furniture ON PURPOSE, and the difference is what + // `verify.checkCoverage` does with it. Furniture is discarded, so coverage does not + // count it — counting it would hide a rule that wrongly claimed a paragraph. A + // callout label is RELOCATED rather than discarded, so coverage does count it, and + // a conversion that lost one still shows up as a page that dropped text. See + // [Callouts] for the whole argument and the measurement under it. + Callout bool +} + +// Bounds on what a line, a paragraph break and a heading are. +// +// Every one is measured against both fixtures — the 68-page parallel-columns +// manual (testdata/fixtures/thomas-drybox-amfibia.json) and the 560-page +// sequential one (dreame-l40-ultra.json) — read through `pdftohtml -xml`, whose +// space matches a `pdftoppm -r 108` raster 1:1 so a block can be drawn on the +// page and looked at. Where a measurement did not support a clean threshold that +// is recorded here too, rather than a tuned number being left to look measured. +const ( + // paragraphGapFactor is how much bigger than the column's own line pitch a + // vertical gap must be to end a paragraph. + // + // The pitch is measured per column, never assumed, because the two manuals set + // different bodies at different leading: 14pt on a 16-unit pitch in the column + // manual's prose columns, 17pt on 22.5 in the sequential manual's safety pages, + // and 15 in the column manual's parts lists. A fixed pitch reads one document + // and shreds the other. + // + // 1.2 comes from the tightest real break in either document. On the column + // manual's page 62, whose German region is the acceptance target, the left + // column runs body lines at a 16-unit pitch and separates its paragraphs and + // its specification rows by 20 to 21 — "Typenbezeichnung: 788/M" at y=490, + // "Spannungsversorgung: 230 V, 50 Hz" at 511, "Leistungsaufnahme:" at 531, + // "Länge Stromzuleitung:" at 552. 1.2*16 = 19.2 splits all four and keeps every + // 16-, 17- and 18-unit gap inside its paragraph. That matters more than it + // looks: those four lines are the unruled specification table the contract + // records as undetectable, and the contract's claim that such a page "still + // reads correctly, as lines of text" is true only if each row is its own block. + // + // What this does NOT resolve is honest to state: two paragraphs 18 units apart + // on the same page (y=357 and y=375 of that column, two separate sentences + // about the service department) are 1.125 of the pitch and are folded into one + // paragraph. No factor separates them from the 17- and 18-unit gaps that occur + // inside paragraphs on the same page, so the choice is between losing that + // break and inventing breaks inside prose, and losing the break is the smaller + // error. + paragraphGapFactor = 1.2 + + // minPitchLines is how many line gaps a column needs before its own pitch is + // believed. Below that the pitch falls back to the median line height, which is + // the leading of a single-spaced setting to within a few per cent and is the + // only evidence a two-line column offers. + minPitchLines = 3 + + // headingMaxMeasureFraction is how much of its column's measure a line may + // occupy and still be a heading. + // + // Length is measured as a fraction of the measure rather than in characters, + // which the contract states in characters per run. Both readings agree on the + // documents — the column manual's headings are 17.8 characters a run against + // 43.5 for its emphasis, the sequential manual's 15.6 against 65.2 — but a rune + // count is not scale-free and not script-free: a CJK heading of six runes is + // wide and a Thai one of forty is narrow, and this manual has sections in both. + // The fraction says the same thing about the two measured manuals and keeps + // saying it about a third. + // + // Measured, as a fraction of the column's own measure: the column manual's + // page 62 sets "Hinweis zur Entsorgung" at 0.30 and "Technische Daten" at 0.22 + // against body lines at 0.97; the sequential manual's page 24 sets + // "Sicherheitshinweise" at 0.25 against body lines at 0.98. + // + // **This is a soft cut and there is no gap to put it in.** That was measured + // rather than hoped for, and it came out the wrong way. Taking every line that + // passes the other two tests and histogramming its share of the measure gives a + // smooth continuum from 5% to 100% on both documents — the column manual's 632 + // candidates run 33 at 60-64%, 25 at 65-69%, 17 at 70-74%, 116 at 95-99% — with + // no trough anywhere. Counting runes instead, which is what the contract states, + // is no better: 135 candidates at 50-59 runes and 104 at 60-69 against 133 at + // 20-29, again with no valley. The reason is real and not an artifact. A manual + // sets one-line paragraphs ("Saugen Sie im Trockensaugbetrieb keine Flüssigkeiten + // auf.") and two-line headings ("Trockensaugen mit der AQUA-Box + // (Wasserfilter-Technologie)"), so the two populations genuinely overlap. + // + // 0.6 is therefore chosen for precision rather than found in a gap, and the + // asymmetry is deliberate: a false heading is visible wrong furniture in the + // reader, while a missed heading degrades to a paragraph and still reads. What it + // costs, measured: a genuine heading that fills a narrow column comes back as a + // paragraph, because the measure is taken from the column and a column holding + // nothing but its heading has the heading's own width. The sequential manual's + // "Fehlersuche", "Feilsøking" and "Depanare" sit at 100% for exactly that reason + // and are lost. The widest German heading that survives is "Trockensaugen mit der + // AQUA-Box (Wasserfilter-Technologie)" at 60% — on the cut, which is what + // TestBlocksHeadingLengthIsASoftCut pins so that the next person sees the cost + // rather than rediscovering it. + headingMaxMeasureFraction = 0.6 + + // markerGapFactor is how wide the space after a list marker must be, against + // the line's own text height, before the marker is read as a marker rather than + // as the first word of a sentence. + // + // This is what makes a bare number a marker. The column manual's parts lists + // print " 1 " at x=30 and its text at x=60 with no punctuation between them — + // 20 units of gap against 17 of height, 1.2 — and without a gap test those + // numbers are prose: measured, page 13's nine parts fold into one paragraph + // reading "1 Крышка корпуса 2 Ручка для переноски 3 ...". A word space in these + // faces is much narrower: the intra-line gap between runs that already carry a + // space is 1.06 of the height at the 25th percentile on that manual and 0.0 on + // the sequential one, so 0.5 is above a word space in both and below every + // measured tab. + markerGapFactor = 0.5 + + // maxMarkerRunes is how long a leading token may be and still be a list + // marker. Three digits covers "1." to "999.", and the measured manuals never + // number past 21. + maxMarkerRunes = 3 + + // minHangingIndentFactor is how far right of a list item's marker a following + // line must sit to be that item's own text. See [blocksOfColumn] for the three + // measured indents it has to catch and the document shape that would defeat it. + minHangingIndentFactor = 0.5 +) + +// bulletRunes are the characters that open a list item without needing a gap +// test, because none of them starts a word. Every one is in the two fixtures +// except the ASCII asterisk and hyphen, which are here because a manual written +// in a word processor uses them and excluding them would be a guess the other +// way. +const bulletRunes = "•·▪◦‣∙*-–—>»✓" + +// RegionBlocks turns one region of one page into ordered readable blocks. +// +// p is the page's positioned text and r the region to read, which must be a +// region of that page. tables are the page's ruled tables, and nil is a normal +// argument: pdftocairo is optional, so it means "the ruled lines were not read" +// and produces exactly the reading that shipped before they were. The result is +// in reading order with Index assigned from 0, and is empty for a region holding +// no usable text — a region of a diagram's callouts is a normal outcome, not a +// failure. +// +// Only the text inside the region's box is read, through the same two filters the +// region's own character count came from: [usableRuns] drops the sub-legible +// production slugs and the runs parked off the page, and [runsInBox] decides +// membership. Sharing those is not tidiness — a block built from runs the region +// did not count would put text in the reader that the gate never charged for, and +// on a parallel-columns page it would be text in a language nobody asked for. +// The table walk draws from the same [inside] set for the same reason, so a cell +// can never show text the region did not charge for and never the reverse. +// fur says which of the page's runs are page furniture, and nil is a normal +// argument meaning "not looked for" — a single page cannot answer that question, +// so every caller holding one page and not the document passes nil and gets +// exactly the reading that shipped before this existed. See [Furniture]. +// A figure's callout labels are taken out of the flow when the caller passes +// [WithCallouts]; without it every run is read as flow text, which is the reading +// that shipped before labels were carried on a figure. See [Callouts]. +func RegionBlocks(p *PageRuns, r *Region, tables []RuledTable, fur *Furniture, + opts ...BlockOption) []Block { + var o blockOpts + for _, fn := range opts { + fn(&o) + } + + var dropped DroppedRuns + kept := usableRuns(p.Runs, p.Width, p.Height, &dropped) + all := runsInBox(kept, r.X0, r.X1) + if len(all) == 0 { + return nil + } + + // Furniture and callout labels are both taken out here, before a pitch, a body + // face or a line is measured. [splitFurniture] records why that has to happen at + // run level and not on the finished blocks, and [Callouts] records the measurement + // that says the same is true of a label. + // + // Furniture first, so that a run which is both is furniture: a label is claimed by + // geometry from a drawing on the page, while furniture is claimed by repeating + // across pages, and the second is the stronger statement about a run that repeats. + // Neither document has such a run — measured, the two sets are disjoint on both — + // so this is an order chosen for what it would mean rather than for what it does. + inside, furniture := splitFurniture(all, r.Page, fur) + inside, callouts := splitCallouts(inside, r.Page, o.callouts) + if len(inside) == 0 { + out := furnitureBlocks(furniture, r, fur, 0) + return append(out, calloutBlocks(callouts, r, len(out))...) + } + + tol := baselineToleranceFraction * medianHeight(inside) + body := regionBody(inside) + // The REGION's language decides direction, not the characters of one line. See + // [lineIsRightToLeft] for the six lines that made this necessary. + rtlRegion := IsRightToLeftLanguage(r.Lang) + + // The table walk takes the runs that sit in a cell, and the column walk takes + // what is left. See [cellRunsOfRegion] for why membership is a cell and not a + // table's box. + celled, prose := cellRunsOfRegion(inside, tables) + // Columns are found from the region's text INCLUDING its furniture, and only + // then is the furniture left out of the strips. See [readingGroups]. + _, layoutRuns := cellRunsOfRegion(all, tables) + groups := readingGroups(prose, layoutRuns, p, r) + placeTables(groups, celled) + + var out []Block + for gi := range groups { + group := &groups[gi] + var blocks []Block + if lines := groupLines(group.runs, tol, rtlRegion); len(lines) > 0 { + pitch := columnPitch(lines) + blocks = blocksOfColumn(lines, pitch, group.measure, body) + } + // Indexed rather than ranged by value: gocritic rejects copying a struct this + // size per iteration, and CONTRIBUTING.md records why. + blocks = mergeTablesByDepth(blocks, group.tables, tol, rtlRegion) + for i := range blocks { + b := &blocks[i] + b.Page = r.Page + b.RegionX0 = r.X0 + b.Lang = r.Lang + b.Index = len(out) + out = append(out, *b) + } + } + + // The furniture last, and not in the reading order it was printed in. Two + // reasons, and the first is the one that matters: a region's content then keeps + // the contiguous 0..n-1 that makes "paragraph 4 of the German region of page 62" + // mean the fourth paragraph a reader sees, which is the citation ingest.md asks + // for and which a tab sitting at index 1 breaks. The second is that furniture has + // no place in reading order to be put back into — it is what a reader is shown + // beside the page, not within it. + // + // The callout labels after it, on both counts identically: a label's place is + // beside the picture, which is not a position in the prose, and the content indices + // have to stay contiguous. + out = append(out, furnitureBlocks(furniture, r, fur, len(out))...) + return append(out, calloutBlocks(callouts, r, len(out))...) +} + +// BlockOption is an optional input to [RegionBlocks] and [RegionsBlocks]. +// +// A functional option rather than a fifth parameter because the two functions have 58 +// call sites between them in this package's tests alone, every one of which would have +// gained a `nil` that says nothing. The zero set of options is the reading that shipped +// before callout labels existed, which is the property [ConvertOptions] documents for +// the same reason one level up. +type BlockOption func(*blockOpts) + +type blockOpts struct{ callouts *Callouts } + +// WithCallouts takes the runs a figure's labels claimed out of the reading flow, so a +// label the reader draws beside the picture is not also printed in the prose. A nil +// argument is legal and changes nothing. See [Callouts]. +func WithCallouts(c *Callouts) BlockOption { + return func(o *blockOpts) { o.callouts = c } +} + +// RegionsBlocks reads every region of a document that is in scope, in page order. +// +// inScope is keyed on base language, the same key [RegionChars] and ScopeFor use, +// for the same measured reason: a document printing CN, JA and ZH-HK counts as +// three languages under its labels and one under its base tags. A nil map reads +// every region, which is what a caller inspecting a whole document wants. +// +// tables is keyed on page number, and a page missing from it has none — which is +// also what a caller that could not run pdftocairo passes, for every page. +func RegionsBlocks(pages []PageRuns, regions []Region, inScope map[string]bool, + tables map[int][]RuledTable, fur *Furniture, opts ...BlockOption) []Block { + byPage := make(map[int]*PageRuns, len(pages)) + for i := range pages { + byPage[pages[i].No] = &pages[i] + } + + // Regions arrive in page then left-edge order from PageRegions, but nothing + // downstream promises to keep them that way once they have been through a + // database, and the natural key is only stable if the order is. + idx := make([]int, 0, len(regions)) + for i := range regions { + if inScope != nil && !inScope[BaseLanguage(regions[i].Lang)] { + continue + } + idx = append(idx, i) + } + sort.SliceStable(idx, func(a, b int) bool { + ra, rb := ®ions[idx[a]], ®ions[idx[b]] + if ra.Page != rb.Page { + return ra.Page < rb.Page + } + return ra.X0 < rb.X0 + }) + + var out []Block + for _, i := range idx { + p := byPage[regions[i].Page] + if p == nil { + continue + } + out = append(out, RegionBlocks(p, ®ions[i], tables[regions[i].Page], fur, opts...)...) + } + return out +} + +// bodyFace is the size and weight most of a region's text is set in. +type bodyFace struct { + size float64 + weight Weight + chars int +} + +// regionBody derives the body face from the region's own text. +// +// It is derived and not configured, and that is the decision the heading rule +// stands on. The whole-document body of the sequential manual is 11pt, 54.1% of +// its characters — but its safety pages carry no 11pt text at all. Page 24 is +// twenty-two lines of 17pt prose under one 21pt heading, and against a +// hard-coded 11pt every line of it is "larger than body". Against its own page +// the body is 17pt and only the heading stands above it. The column manual needs +// the same treatment from the other side: 84.0% of its characters are one size, +// so on most of its pages size discriminates nothing and only weight is left. +// +// Weight is taken as the character-weighted mode too, not the minimum or the +// mean, because the column manual's body face is FuturaCon-Lig — light — and +// 17.2% of the document is the same size in FuturaCon-Med. Reading the body as +// "light" is what leaves medium available as an emphasis signal; reading it as +// the lightest thing present would too, but reading it as the mean would not. +func regionBody(runs []TextRun) bodyFace { + type face struct { + size float64 + weight Weight + } + chars := make(map[face]int, 8) + for i := range runs { + n := utf8.RuneCountInString(strings.TrimSpace(runs[i].Text)) + if n == 0 { + continue + } + chars[face{runs[i].Font.Size, effectiveWeight(&runs[i].Font)}] += n + } + + var best bodyFace + for f, n := range chars { + // Ties broken towards the smaller size, then the lighter weight, so that a + // region split evenly between a heading face and a body face does not name + // the heading face as the body and lose every heading on the page. + switch { + case n > best.chars, + n == best.chars && f.size < best.size, + n == best.chars && f.size == best.size && f.weight < best.weight: + best = bodyFace{size: f.size, weight: f.weight, chars: n} + } + } + return best +} + +// effectiveWeight folds poppler's own markup into the weight the family name +// declares, taking whichever is heavier. +// +// Both signals are needed and neither can be dropped, which is measured and not a +// hedge: 93.4% of the column manual's characters are in a face whose name states a +// weight and poppler marks only 1.5% of them bold, while 73.2% of the sequential +// manual's are in a face called plainly "MiSans" that states nothing at all and +// poppler's markup is the only weight there is. See [Weight] for the counts. +// +// A marked run is read as semibold and not as bold because that is where poppler +// draws its line, measured over both documents: it wraps every run of every name +// saying Bold, Demibold, SemiBold or Xbold, and not one run of any name saying +// Medium. So means "at least semibold" exactly, and promoting it to bold would +// make a semibold heading outrank a bold one. +func effectiveWeight(f *Font) Weight { + w := f.Weight + if f.MarkedBold && w < WeightSemibold { + w = WeightSemibold + } + return w +} + +// readingGroup is one strip of a region that can be sorted internally: a text +// column, or the band of runs that span across all of them. +type readingGroup struct { + runs []TextRun + // measure is the width the group's lines are set to — the column's own + // measure, which is what a heading's length is judged against. Taken from the + // column the detector reported rather than from the runs in hand, so that a + // column holding nothing but two short headings is not judged to have a short + // measure and both promoted. + measure float64 + // x0 and x1 bound the strip on the page. They exist so a table can be placed + // in the strip that holds it, and they are NOT the measure: for a region read + // as one strip they are the region's whole box, because that strip is the + // region, while the measure stays what the strip's own text reaches. + x0, x1 float64 + // banner marks the strip that holds the runs spanning every column. It is the + // fallback owner of a table, never a candidate: it is read first, and a table + // belongs to the strip that contains it where one does. + banner bool + // tables are the tables placed in this strip, in page order. + tables []cellRuns +} + +// readingGroups subdivides a region into the strips reading order runs down. +// +// See the file comment for why a region is not itself such a strip. The spanning +// runs come first as one group of their own: a heading set across two columns is +// above both of them on the page, and putting it after either would read wrongly +// on the one page of the column manual that does it. +// +// The runs handed in are the region's text minus whatever sits in a table cell, so +// the columns found here are the region's text columns and not a table's cell +// columns. That is the fix docs/design/conversion.md asks for: on the column +// manual's page 57 the four "columns" the detector finds over the whole page are +// two tables' cell dividers, and reading down them is reading a troubleshooting +// table down its question column and then down its answer column. +// +// # prose and forLayout, and why they are two arguments +// +// forLayout is where the columns are found and prose is what gets sorted into +// them, and they differ by the page furniture. Detecting on prose — the region's +// text with the furniture already removed — makes the column split depend on which +// runs another rule happened to claim, and that is not a theory: measured over the +// sequential manual, three pages change their column count when the running head is +// taken out ahead of the detector. Page 552 goes from two columns to one, so its +// Japanese left and right headings merge into `サイドブラシ ロボット掃除機本体のセ +// ンサーと充電端子`, and page 36's German troubleshooting grid goes from three to +// two and merges `Problem` with `Problem`. Page 486's Thai goes the other way and +// improves. +// +// Detecting on forLayout — everything the region prints, furniture included, minus +// only the table cells — makes the answer a property of the page instead. Measured +// over both documents: on its own it moves the sequential manual by 1 block of +// 16,098 and changes no finding of any kind, and with the running-head clause on +// top the reading-order count, the invented-text count, the glued-word count and +// every figure count are exactly what they were before either change. +func readingGroups(prose, forLayout []TextRun, p *PageRuns, r *Region) []readingGroup { + cols := DetectColumns(forLayout, p.Width, p.Height).Columns + if len(cols) < 2 { + // Not two columns, which is not the same as not two strips. See + // [readingStrips] and [readingGates] for the pages that are the difference + // and for why the two questions cannot share one pair of bounds. + cols = readingStrips(forLayout, p.Width, p.Height) + } + if len(cols) < 2 { + // One strip, or too little text to call one. Either way the region is a + // single strip and its measure is what its own text reaches. + lo, hi := extent(prose) + if len(prose) == 0 { + lo, hi = r.X0, r.X0 + } + return []readingGroup{{runs: prose, measure: hi - lo, x0: r.X0, x1: r.X1}} + } + groups := make([]readingGroup, len(cols)+1) + groups[0].measure = func() float64 { lo, hi := extent(prose); return hi - lo }() + groups[0].x0, groups[0].x1, groups[0].banner = r.X0, r.X1, true + for i := range cols { + groups[i+1].measure = cols[i].Width() + groups[i+1].x0, groups[i+1].x1 = cols[i].Min, cols[i].Max + } + + for i := range prose { + run := &prose[i] + k := columnOf(run, cols) + if k < 0 { + // Spanning, or in a gap the detector did not report as a column: read it + // with the banner band rather than dropping it. Losing text is never the + // right answer here — the region's own character count included it. + groups[0].runs = append(groups[0].runs, *run) + continue + } + groups[k+1].runs = append(groups[k+1].runs, *run) + } + + // A right-to-left region's columns are read right to left, and the banner stays + // first because it is above them rather than beside them. Nothing exercised this + // until the strips above reached the Hebrew and Arabic disposal pages: page 216 + // prints its guide in a left column and its warning in a right one, and read left + // first an Arabic reader is handed step 1 before the paragraph that introduces it. + // The direction comes from the REGION's language, the same source and the same + // reason as [lineIsRightToLeft]'s. + if IsRightToLeftLanguage(r.Lang) { + cols := groups[1:] + for i, j := 0, len(cols)-1; i < j; i, j = i+1, j-1 { + cols[i], cols[j] = cols[j], cols[i] + } + } + + out := make([]readingGroup, 0, len(groups)) + for i := range groups { + if len(groups[i].runs) > 0 || groups[i].banner || len(groups) == 1 { + out = append(out, groups[i]) + } + } + return out +} + +// cellRuns is one table's text, as it fell inside one region. +// +// cells is parallel to the table's own Cells, so a cell that took no run of this +// region stays empty and produces nothing. That is what clips a table to a region +// without any arithmetic on boxes: the runs were clipped to the region before they +// were offered to a cell. +type cellRuns struct { + table *RuledTable + cells [][]TextRun + runs int +} + +// cellRunsOfRegion divides a region's runs into the ones a table cell holds and the +// ones it does not. +// +// Membership is a CELL and not the table's box, and the difference is load-bearing +// twice over. +// +// A heading printed across a table straddles the rules that would enclose it, so it +// belongs to no cell and stays with the prose — where it joins the banner group, +// which is read first, and appears exactly once. docs/design/conversion.md names the +// alternatives and both are wrong: putting it in a cell buries it, and suppressing +// banner blocks wherever a table covers the region loses it altogether. The measured +// case is the column manual's page 57, whose "Aufgetretene Störungen/Fehlfunktionen" +// and "Grund / Abhilfe" header row is printed above the table's own top border, +// because that border is not drawn — the same missing rule conversion.md records as +// costing four cells. +// +// And a table may span regions in DIFFERENT languages. Page 57's left table runs +// x=29.7-428.1, which before this change covered a Finnish region and a German one. +// Assuming one table sits inside one region would have pulled that Finnish column +// into a German conversion; here the region's own runs are all a cell can ever be +// offered, so a block cannot reach outside its region's box. +// +// The margin is [cellTextMargin], the same tolerance [countCellText] used to decide +// the same question when it counted the cell's characters. Two answers to "is this +// run in this cell" would eventually disagree. +func cellRunsOfRegion(inside []TextRun, tables []RuledTable) (celled []cellRuns, prose []TextRun) { + if len(tables) == 0 { + return nil, inside + } + + celled = make([]cellRuns, len(tables)) + for i := range tables { + celled[i] = cellRuns{table: &tables[i], cells: make([][]TextRun, len(tables[i].Cells))} + } + + prose = make([]TextRun, 0, len(inside)) + for i := range inside { + run := &inside[i] + ti, ci := cellHolding(run, tables) + if ti < 0 { + prose = append(prose, *run) + continue + } + celled[ti].cells[ci] = append(celled[ti].cells[ci], *run) + celled[ti].runs++ + } + + out := make([]cellRuns, 0, len(celled)) + for i := range celled { + if celled[i].runs > 0 { + out = append(out, celled[i]) + } + } + return out, prose +} + +// cellHolding returns the table and cell that contain a run whole, or -1, -1. +func cellHolding(run *TextRun, tables []RuledTable) (table, cell int) { + for i := range tables { + t := &tables[i] + if run.X < t.Box.X0-cellTextMargin || run.right() > t.Box.X1+cellTextMargin || + run.Y < t.Box.Y0-cellTextMargin || run.bottom() > t.Box.Y1+cellTextMargin { + continue + } + for j := range t.Cells { + c := &t.Cells[j].Rect + if run.X >= c.X0-cellTextMargin && run.right() <= c.X1+cellTextMargin && + run.Y >= c.Y0-cellTextMargin && run.bottom() <= c.Y1+cellTextMargin { + return i, j + } + } + } + return -1, -1 +} + +// placeTables gives each table to the strip it mostly sits in. +// +// Overlap and not containment, and that is a correction rather than a preference. A +// table's box is drawn by [PageTables] from the strokes on the page, while a strip's +// bounds are where its *words* reach, and the two come from different inputs and do +// not nest: the sequential manual's page 537 draws the base station's spec table +// x=478-862 in a strip whose text reaches 845, and the column manual's page 57 draws +// its two troubleshooting tables across strips found from four header runs, the cells +// themselves having been taken out of the projection before it ran. Under containment +// both fall through to the banner band, which reads first — so a reader is shown the +// whole spec table and only then the page's own title. Overlap puts each one back +// where the page prints it. +// +// The banner remains the fallback and remains right for what reaches it: a table that +// overlaps no strip at all is above them, which is where a banner is. +func placeTables(groups []readingGroup, celled []cellRuns) { + if len(celled) == 0 || len(groups) == 0 { + return + } + for i := range celled { + box := &celled[i].table.Box + k, best := 0, 0.0 + for j := range groups { + g := &groups[j] + if g.banner { + continue + } + over := math.Min(box.X1, g.x1+cellTextMargin) - math.Max(box.X0, g.x0-cellTextMargin) + if over > best { + k, best = j, over + } + } + groups[k].tables = append(groups[k].tables, celled[i]) + } +} + +// mergeTablesByDepth interleaves a strip's tables with its text blocks by how far +// down the page each starts. +// +// A table is one unit rather than a set of blocks to be sorted individually, since +// its cells run past each other vertically and sorting them among paragraphs would +// shuffle a table into the prose around it. Which side of a block a table falls on +// is decided by the top of its box against the top of the block. +// +// What this does not do is split a strip's prose around a table. A paragraph printed +// BELOW a table, in a strip that also has prose above it, still reads before the +// table, because the strip's lines are one sequence and the tables are placed in it +// rather than the strip being cut into bands. Neither measured manual does that on a +// table page — the column manual's tables sit at the foot of their column or fill +// the page, and the sequential manual's are the whole page under a heading — so the +// band split would be built against no example. +func mergeTablesByDepth(blocks []Block, tables []cellRuns, tol float64, rtlRegion bool) []Block { + if len(tables) == 0 { + return blocks + } + sort.SliceStable(tables, func(a, b int) bool { + if tables[a].table.Box.Y0 != tables[b].table.Box.Y0 { + return tables[a].table.Box.Y0 < tables[b].table.Box.Y0 + } + return tables[a].table.Box.X0 < tables[b].table.Box.X0 + }) + + out := make([]Block, 0, len(blocks)+4*len(tables)) + next := 0 + for i := range blocks { + for next < len(tables) && tables[next].table.Box.Y0 <= blocks[i].Y0 { + out = append(out, tableBlocks(&tables[next], tol, rtlRegion)...) + next++ + } + out = append(out, blocks[i]) + } + for ; next < len(tables); next++ { + out = append(out, tableBlocks(&tables[next], tol, rtlRegion)...) + } + return out +} + +// tableBlocks turns one table's cells into blocks, one per cell that holds text, +// in row-major order. +// +// Row-major is the whole point of handing a table to this walk. Left to the column +// walk, a two-column troubleshooting table reads down every question and then down +// every answer — which blocks.go recorded as a known limitation and this is its fix. +// Reading across each row pairs each question with its answer, which is how the page +// is printed and how it is read. +// +// One block per cell rather than one per table, because a cell is the unit a later +// stage can cite and translate: "what is the remedy for this fault" is answered by a +// cell. The grid position travels in the note rather than in a field, for the reason +// [Block] gives about its key — the block vocabulary is not widened here, so nothing +// stored or served has to change to hold a table. +func tableBlocks(t *cellRuns, tol float64, rtlRegion bool) []Block { + order := make([]int, 0, len(t.cells)) + for i := range t.cells { + if len(t.cells[i]) > 0 { + order = append(order, i) + } + } + sort.SliceStable(order, func(a, b int) bool { + ca, cb := &t.table.Cells[order[a]], &t.table.Cells[order[b]] + if ca.Row != cb.Row { + return ca.Row < cb.Row + } + return ca.Col < cb.Col + }) + + out := make([]Block, 0, len(order)) + for _, i := range order { + cell := &t.table.Cells[i] + lines := groupLines(t.cells[i], tol, rtlRegion) + texts := make([]string, 0, len(lines)) + b := Block{Kind: BlockTable, + X0: math.Inf(1), X1: math.Inf(-1), Y0: math.Inf(1), Y1: math.Inf(-1)} + for j := range lines { + texts = append(texts, lines[j].text) + b.X0 = math.Min(b.X0, lines[j].x0) + b.X1 = math.Max(b.X1, lines[j].x1) + b.Y0 = math.Min(b.Y0, lines[j].y) + b.Y1 = math.Max(b.Y1, lines[j].bottom) + } + b.Text = collapseSpaces(strings.Join(texts, " ")) + if b.Text == "" { + continue + } + b.Chars = utf8.RuneCountInString(b.Text) + b.Lines = len(lines) + b.Note = fmt.Sprintf("row %d of %d, column %d of %d of a ruled table", + cell.Row+1, t.table.Rows, cell.Col+1, t.table.Cols) + if cell.ColSpan > 1 { + b.Note += fmt.Sprintf(", spanning %d of them", cell.ColSpan) + } + out = append(out, b) + } + return out +} + +// columnOf places a run in the column that contains it, or -1 when it reaches +// past one. Membership is by containment and not by left edge, because a run +// straddling two columns belongs to neither and must be read with the banner band +// — the same distinction [crossesAny] draws, expressed against columns rather +// than gutters because that is what is in hand here. +func columnOf(r *TextRun, cols []Column) int { + for i := range cols { + if r.X >= cols[i].Min-1 && r.right() <= cols[i].Max+1 { + return i + } + } + return -1 +} + +// textLine is the runs of one baseline, in the order they are read. +type textLine struct { + runs []TextRun + y, bottom float64 + x0, x1 float64 + text string + chars int + size float64 + weight Weight + marker string + markerRuneOnly bool +} + +// groupLines folds runs onto shared baselines, then orders the baselines down the +// page and the runs across each one. +// +// The baseline rule is [sameBaseline], which is the rule columns.go already uses +// to fold a list marker into the text it labels — shared rather than restated, +// because two definitions of "these runs are one line" would drift and the second +// one would be the wrong one. The tolerance is small on purpose: runs of one line +// carry the same top to the unit in both documents, so it has rounding to absorb +// and nothing more. +func groupLines(runs []TextRun, tol float64, rtl bool) []textLine { + ordered := make([]TextRun, len(runs)) + copy(ordered, runs) + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].Y != ordered[j].Y { + return ordered[i].Y < ordered[j].Y + } + return ordered[i].X < ordered[j].X + }) + + var lines []textLine + for i := range ordered { + r := &ordered[i] + // Compared against the line's first run, not its last, so that a line of many + // runs cannot drift a tolerance at a time into the line below it. + if n := len(lines); n > 0 && sameBaseline(lines[n-1].runs[0].Y, r.Y, tol) { + lines[n-1].runs = append(lines[n-1].runs, *r) + continue + } + lines = append(lines, textLine{runs: []TextRun{*r}}) + } + + for i := range lines { + lines[i].finish(rtl) + } + return lines +} + +// finish computes everything a line's runs imply, once they are all in. +func (l *textLine) finish(rtlRegion bool) { + sort.SliceStable(l.runs, func(i, j int) bool { return l.runs[i].X < l.runs[j].X }) + + l.y, l.bottom = math.Inf(1), math.Inf(-1) + l.x0, l.x1 = math.Inf(1), math.Inf(-1) + sizes := make(map[float64]int, 4) + weights := make(map[Weight]int, 4) + for i := range l.runs { + r := &l.runs[i] + l.y = math.Min(l.y, r.Y) + l.bottom = math.Max(l.bottom, r.bottom()) + l.x0 = math.Min(l.x0, r.X) + l.x1 = math.Max(l.x1, r.right()) + n := utf8.RuneCountInString(strings.TrimSpace(r.Text)) + sizes[r.Font.Size] += n + weights[effectiveWeight(&r.Font)] += n + } + + // The line's face is its dominant one by characters, which is what makes a + // bold lead-in inside a line of body copy not turn the line into a heading: + // "Achtung: das Gerät nicht ..." is two runs and the light one is longer. + l.size = dominantSize(sizes) + l.weight = dominantWeight(weights) + + // A right-to-left line arrives reversed twice over — see bidi.go — and this is + // the one place a line's order is decided, so it is the one place that repairs it. + if lineIsRightToLeft(l.runs, rtlRegion) { + l.text = joinRunsRightToLeft(l.runs) + } else { + l.text = joinRuns(l.runs) + } + l.chars = utf8.RuneCountInString(l.text) + l.marker, l.markerRuneOnly = leadingMarker(l) +} + +func dominantSize(sizes map[float64]int) float64 { + var best float64 + bestN := -1 + for size, n := range sizes { + if n > bestN || (n == bestN && size < best) { + best, bestN = size, n + } + } + return best +} + +func dominantWeight(weights map[Weight]int) Weight { + best, bestN := WeightUnknown, -1 + for w, n := range weights { + if n > bestN || (n == bestN && w < best) { + best, bestN = w, n + } + } + return best +} + +// joinRuns renders a line's runs as text, inserting a space only where the page +// shows one and neither run already carries it. +// +// Poppler splits a run at every font change, so a line reading "Sollte Ihr +// THOMAS DryBox einmal ausgedient haben" arrives as three runs, and gluing them +// blind produces "THOMASDryBoxeinmal". Measured over both documents, the gap +// between two runs with no whitespace on either side is above zero for the great +// majority — a quarter of them are already more than 1.8 heights apart on the +// column manual and 2.9 on the sequential one, because they are separate cells +// and labels rather than a split word. Where the gap is zero or negative the runs +// touch or overlap, and joining them directly is what the page shows. +// +// Zero is the threshold and it is not tuned: touching glyphs are one word and +// separated ones are two. That is the boundary the page draws. +// +// What this deliberately does not do is undo hyphenation. The column manual +// breaks "brud-/nej wody" and "Verpackungsmate-/rial" across lines, and joining +// leaves the hyphen in. Removing a trailing hyphen would be wrong at least as +// often: German prose in the same document ends a line with a hyphen that must +// stay — "Ein- und Ausschalten", "Elektro-Fachkräfte" — and telling a broken word +// from a compound needs a dictionary of the language, which is a later stage's +// evidence and not this one's. +func joinRuns(runs []TextRun) string { + var b strings.Builder + for i := range runs { + if i > 0 { + prev := &runs[i-1] + gap := runs[i].X - prev.right() + if gap > 0 && !endsWithSpace(prev.Text) && !startsWithSpace(runs[i].Text) { + b.WriteByte(' ') + } + } + b.WriteString(runs[i].Text) + } + return collapseSpaces(b.String()) +} + +// hasLetter reports whether a string carries any letter at all. Format +// characters are stripped first for the reason [stripFormatting] gives: a +// right-to-left line wraps its Latin furniture in bidi controls, and those are +// neither letters nor digits. +func hasLetter(s string) bool { + for _, r := range stripFormatting(s) { + if unicode.IsLetter(r) { + return true + } + } + return false +} + +func endsWithSpace(s string) bool { + r, _ := utf8.DecodeLastRuneInString(s) + return unicode.IsSpace(r) +} + +func startsWithSpace(s string) bool { + r, _ := utf8.DecodeRuneInString(s) + return unicode.IsSpace(r) +} + +// collapseSpaces reduces every run of whitespace to one space and trims the ends. +// A tab stop is many spaces on the page and one on a screen, and a block that +// carries the page's own spacing cannot be rendered at another measure. +func collapseSpaces(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +// leadingMarker reports the list marker a line opens with, if any. +// +// Two shapes, both measured. A bullet character needs nothing else: nothing that +// opens a sentence looks like one. A number or a letter needs either punctuation +// after it — "1." or "2)" — or, when the document prints neither, a gap to the +// text wide enough to be a tab rather than a word space. runeOnly reports which +// it was, because a bare number that was accepted on the strength of a gap is +// weaker evidence and is not allowed to break a paragraph on its own. +func leadingMarker(l *textLine) (marker string, runeOnly bool) { + text := strings.TrimSpace(stripFormatting(l.text)) + if text == "" { + return "", false + } + + first, size := utf8.DecodeRuneInString(text) + if strings.ContainsRune(bulletRunes, first) { + return string(first), false + } + + // A leading token of digits, or one letter, then optional punctuation. + rest := text + token := "" + for rest != "" { + r, n := utf8.DecodeRuneInString(rest) + if !unicode.IsDigit(r) { + break + } + token += string(r) + rest = rest[n:] + if utf8.RuneCountInString(token) > maxMarkerRunes { + return "", false + } + } + if token == "" { + if !unicode.IsLetter(first) || utf8.RuneCountInString(text) < 2 { + return "", false + } + // A single letter followed by punctuation: "a)" and "b)" lists. + token, rest = string(first), text[size:] + next, _ := utf8.DecodeRuneInString(rest) + if next != ')' && next != '.' { + return "", false + } + } + + if next, n := utf8.DecodeRuneInString(rest); next == '.' || next == ')' { + return token + string(next), false + } else if n > 0 && !unicode.IsSpace(next) { + // "230 V" is not item 230; "2026" is not item 202. + return "", false + } + + // No punctuation, so the page must show a tab. Measured on the runs rather + // than on the text, since the space in the text is one character however wide + // the page sets it. + if len(l.runs) < 2 { + return "", false + } + gap := l.runs[1].X - l.runs[0].right() + if gap < markerGapFactor*l.runs[0].Height { + return "", false + } + // The marker has to be the whole of the first run, or the run is a sentence + // that happens to start with a number. + if strings.TrimSpace(stripFormatting(l.runs[0].Text)) != token { + return "", false + } + return token, true +} + +// columnPitch is the normal distance between one line and the next, measured from +// the column's own lines. +// +// The mode of the gaps, rounded to the unit, rather than the median. That is the +// one estimator the measurement forced: a column of prose broken into short +// paragraphs has nearly as many break gaps as body gaps, and the median of the +// column manual's page 62 left column comes out at 18 against a body pitch of 16, +// which is high enough that the 20- and 21-unit breaks it has to find fall inside +// 1.2 of it and are lost. The mode is 16, which is what the page is set on. +// +// Below [minPitchLines] gaps the mode of two numbers means nothing, so the pitch +// falls back to the median line height. +func columnPitch(lines []textLine) float64 { + counts := make(map[int]int, len(lines)) + best, bestN := 0, 0 + gaps := 0 + for i := 1; i < len(lines); i++ { + gap := lines[i].y - lines[i-1].y + if gap <= 0 { + continue + } + gaps++ + k := int(math.Round(gap)) + counts[k]++ + if counts[k] > bestN || (counts[k] == bestN && k < best) { + best, bestN = k, counts[k] + } + } + if gaps >= minPitchLines && best > 0 { + return float64(best) + } + + hs := make([]float64, 0, len(lines)) + for i := range lines { + hs = append(hs, lines[i].bottom-lines[i].y) + } + sort.Float64s(hs) + if len(hs) == 0 { + return 1 + } + return hs[len(hs)/2] +} + +// blocksOfColumn folds one column's lines into blocks. +func blocksOfColumn(lines []textLine, pitch, measure float64, body bodyFace) []Block { + var out []Block + var cur *Block + var curLines []textLine + + flush := func() { + if cur == nil { + return + } + texts := make([]string, 0, len(curLines)) + for i := range curLines { + texts = append(texts, curLines[i].text) + } + cur.Text = collapseSpaces(strings.Join(texts, " ")) + cur.Chars = utf8.RuneCountInString(cur.Text) + cur.Lines = len(curLines) + out = append(out, *cur) + cur, curLines = nil, nil + } + + for i := range lines { + l := &lines[i] + if l.chars == 0 { + continue + } + kind, level, note := classify(l, measure, body) + + // A line with no marker of its own, indented under the item above it and no + // further down the page than one line, is that item's own text rather than a + // paragraph after it. This is the hanging indent, and without it every + // numbered clause of the column manual's guarantee comes back as a one-line + // item followed by an orphaned paragraph: "1. Die Garantiezeit beträgt 24 + // Monate - gerechnet vom Liefertag an den ersten Endabnehmer. Sie reduziert", + // then "sich bei gewerblicher Benutzung ..." as a block of its own. + // + // Measured, as a fraction of the line's own height, the indents that have to + // be caught are 1.29 (the column manual's numbered clauses, marker at x=463 + // and text at 485 on a 17-unit line), 0.65 (its bulleted items, 43 and 54) and + // 1.00 (the sequential manual's safety bullets, 55 and 77 on a 22-unit line). + // 0.5 is below all three. Nothing has to be excluded above it, because neither + // document indents the first line of a paragraph — checked over both, every + // paragraph's first line is flush with its column — so an indent in these + // manuals means a hanging one. A document that indents first lines instead + // would fold each new paragraph into the item above it, and that is the stop + // condition rather than a threshold to retune. + if cur != nil && cur.Kind == BlockListItem && kind == BlockParagraph { + prev := &curLines[len(curLines)-1] + if l.x0 >= curLines[0].x0+minHangingIndentFactor*(l.bottom-l.y) && + l.y-prev.y <= paragraphGapFactor*pitch { + kind, level, note = cur.Kind, cur.Level, cur.Note + } + } + + // A heading has to start a block. This is structural and not a threshold, and + // it is the largest correction the measurement forced: the last line of a + // paragraph is short by definition, so any paragraph set in a face heavier + // than the region's body hands its final line over as a heading. That is not + // a corner case on these documents — the column manual sets 17.2% of its + // characters in FuturaCon-Med at the body size, and reading its lines + // independently produced 280 headings reading "Umgebungen benutzt werden.", + // "во взрывоопасных помещениях.", "gung durchgeführt werden." Requiring the + // space above a heading that the typesetter put there removes all 280 and + // costs nothing: measured over both manuals, every heading that survives is + // separated from what precedes it by more than one line pitch, because that is + // what a heading looks like on paper. + if cur != nil && kind == BlockHeading && cur.Kind != BlockHeading && + l.y-curLines[len(curLines)-1].y <= paragraphGapFactor*pitch { + kind, level, note = cur.Kind, cur.Level, cur.Note + } + + start := cur == nil || kind != cur.Kind || level != cur.Level + if !start { + prev := &curLines[len(curLines)-1] + switch { + case l.y-prev.y > paragraphGapFactor*pitch: + start = true + case IsContentsEntry(note): + // One entry per line, always. Consecutive entries sit at exactly the + // line pitch, so the gap test above cannot separate them — which is + // precisely why a contents page arrived as one run-together paragraph + // of dot leaders: 17 entries glued into one block. + start = true + case kind == BlockListItem && l.marker != "" && !l.markerRuneOnly: + // A second marker is a second item. A bare number accepted on a gap + // alone does not get this power: the column manual's specification rows + // would each become an item of their own list. + start = true + case kind == BlockListItem && l.marker != "" && l.markerRuneOnly && + math.Abs(l.x0-curLines[0].x0) < 1: + // ...unless it sits at exactly the indent the current item's marker does, + // which is what a parts list numbered "1", "2", "3" with no punctuation + // looks like and is the only thing that separates its entries. + start = true + } + } + if start { + flush() + cur = &Block{Kind: kind, Level: level, Note: note, + X0: l.x0, X1: l.x1, Y0: l.y, Y1: l.bottom} + } + curLines = append(curLines, *l) + cur.X0 = math.Min(cur.X0, l.x0) + cur.X1 = math.Max(cur.X1, l.x1) + cur.Y1 = math.Max(cur.Y1, l.bottom) + } + flush() + return out +} + +// contentsNotePrefix opens the note of a block that is one entry of a printed table +// of contents, and [IsContentsEntry] is how a reader asks. +// +// # Why the note and not a kind of its own +// +// A contents entry IS a list item — the paper prints a list — and the note's stated +// job is to say why a block is the kind it is, in checkable terms, which is exactly +// what "a dot leader of 34 and a page number" does. That is the honest reading, and +// it is also the cheap one: [BlockKind] reaches a database column whose CHECK lists +// the five kinds by name, and widening a closed set there costs a table rebuild — +// which for doc_blocks means dropping and recreating 00006's three FTS triggers and +// reindexing the search table, since the index is external-content over this table's +// rowids. Migration 00003 is the precedent for the rebuild and records the procedure; +// nothing here needs it, because nothing here is a sixth kind. +// +// The reader distinguishes an entry by this note, the same way it recovers a list +// marker from `opens with the list marker "•"`. If a later change does want a kind of +// its own, the rebuild is what it costs and 00003 is how it is done. +const contentsNotePrefix = "a dot leader of " + +// IsContentsEntry reports whether a block's note says it is one entry of a printed +// table of contents. Exported because the reader groups a run of them into one list +// and has only the note to go on. +func IsContentsEntry(note string) bool { + return strings.HasPrefix(note, contentsNotePrefix) +} + +// minLeaderDots is how many consecutive dots make a dot leader, and this threshold +// has something almost nothing else in this package has: a real gap to sit in. +// +// Measured over both whole documents, every run of two or more dots: the columns +// manual draws 89 of them, and their lengths are 3, 3, 3, 4 and then **34 to 91**, +// with nothing in between. The four short ones are ellipses in prose and none of +// them is followed by a page number; the 85 long ones are its contents entries, 17 +// per language across the five languages of pages 2 and 3, and all 85 end in a page +// number. The sequential manual has 8 runs of exactly two dots and not one longer, +// so this rule cannot fire on that document at all — its own contents page sets the +// page number in a separate column with no leader between, which is why it needs the +// different signal [Furniture] would want and is not attempted here. +// +// 8 sits in the middle of the gap in log terms and a factor of four below the +// shortest real leader. Anything from 5 to 34 gives the same answer on both +// documents, which is what makes the value uninteresting — the two conditions are +// each sufficient on their own here, since the short runs carry no page number +// either. +const minLeaderDots = 8 + +// contentsEntry reports whether a line is one entry of a printed table of contents: +// a title, a leader of at least [minLeaderDots] dots, and the page it points at. +// +// Both halves are required and the reason is the four short dot runs above. A leader +// alone would take an ellipsis mid-sentence; a trailing number alone would take +// every numbered line in the document, of which the sequential manual has thousands. +// +// The page reference is a number or a range — the columns manual prints +// "Trockensaugen . ...... 14 – 22" — and what is returned is only the leader's length, +// because turning the reference into somewhere a reader can jump needs the printed +// page to be mapped onto a PDF page, which is [Reconcile]'s job and is not done here. +// This makes a contents page READ as a list of entries instead of one run-together +// paragraph; making it navigable is the next step and needs that mapping. +func contentsEntry(text string) (dots int, ok bool) { + trimmed := strings.TrimSpace(text) + if trimmed == "" { + return 0, false + } + // The longest run of dots anywhere in the line. + run, best := 0, 0 + for _, r := range trimmed { + if r == '.' { + run++ + if run > best { + best = run + } + continue + } + run = 0 + } + if best < minLeaderDots { + return 0, false + } + // ...and the line ends in a page reference: a number, or a range of them. Read + // from the end so the title's own digits — "Reinigung der AQUA-Box" has none, but + // "THOMAS 786" would — cannot satisfy it. + rs := []rune(trimmed) + i := len(rs) + for i > 0 && (unicode.IsDigit(rs[i-1]) || unicode.IsSpace(rs[i-1])) { + i-- + } + if i == len(rs) { + return 0, false // does not end in a digit + } + // A range separator, then a second number, is still a page reference. + if i > 0 && (rs[i-1] == '-' || rs[i-1] == '–' || rs[i-1] == '—') { + i-- + for i > 0 && (unicode.IsDigit(rs[i-1]) || unicode.IsSpace(rs[i-1])) { + i-- + } + } + // What is left before the reference must be the leader, not more prose. + for i > 0 && (rs[i-1] == '.' || unicode.IsSpace(rs[i-1])) { + i-- + } + if i == 0 { + return 0, false // dots and digits only, with no title: not an entry + } + return best, true +} + +// classify decides what one line is. +// +// The order is deliberate and the first rule is the one that surprises: a line +// opening with a list marker is a list item even when it is set heavier than the +// body and short enough to be a heading. The column manual's pages 62 to 66 are +// full of exactly that — "• Entsorgung Reinigungsmittel" in FuturaCon-Med at 14pt, +// four to a page, each followed by its own paragraph. The marker is a fact about +// the content and the weight is a fact about the typesetting, so the marker wins. +// Checked against a 108 dpi render of page 62: they are bulleted items with a +// bold lead-in, which is what this calls them. +func classify(l *textLine, measure float64, body bodyFace) (kind BlockKind, level int, note string) { + // Asked before the marker, because a contents entry numbered "1." is still a + // contents entry and neither of these documents has one. The order is the choice; + // the case is hypothetical. + if dots, ok := contentsEntry(l.text); ok { + return BlockListItem, 0, fmt.Sprintf(contentsNotePrefix+"%d and a page number", dots) + } + if l.marker != "" { + return BlockListItem, 0, fmt.Sprintf("opens with the list marker %q", l.marker) + } + if level, note, ok := headingLevel(l, measure, body); ok { + return BlockHeading, level, note + } + return BlockParagraph, 0, "" +} + +// headingLevel decides whether a line is a heading, and how prominent. +// +// Weight and length, never size alone, and the counter-example is measured rather +// than feared: 14.1% of the sequential manual's characters are 17pt in a face +// whose name says nothing, at 65 characters a run, and they are safety prose. +// "Larger than the body means a heading" promotes every line of them. Its real +// headings are 15pt and 21pt semibold at 15.6 and 17.2 characters a run. +// +// So the three tests, in the order they eliminate: +// +// 1. It contains a letter. Not a threshold but a category, and it is the single +// biggest source of false headings measured: the column manual's page 11 is an +// exploded diagram whose 26 numeric callouts are set in FuturaCon-Med at 17pt +// — larger AND heavier than that page's body, and two characters long, so they +// pass every typographic test there is. They were 26 of the 134 headings this +// found in the manual's German regions before this test existed. A figure +// callout, a folio and a chapter number are numbers; a heading is words. +// 2. Heavier than the region's body face. This is the test that does the work, +// and it is the whole reason [effectiveWeight] reads two signals: on the +// column manual the family name carries it and on the sequential one only +// poppler's markup does. +// 3. Short against its column's measure. This is what separates a heading from +// emphasis set as a whole paragraph, which the column manual has 17.2% of. +// +// There is deliberately no floor on the size, and that was measured the other way +// round after being written in. The sequential manual's safety pages are set +// entirely in 17pt, so 17pt is their body, and their real subheadings — +// "Nutzungsbeschränkungen" on page 23 — are 15pt MiSans-Demibold: SMALLER than the +// body they head. Requiring a heading to be at least the body size loses them, and +// it protects against nothing, which is the part that had to be checked rather than +// assumed. The small bold text it looked like it was guarding against is 9pt +// Demibold at 8.0 characters a run on 215 pages, and every one of those is the word +// "Note:" or "Hinweis:" opening a paragraph — a lead-in run, not a line. The +// dominant-face rule in [textLine.finish] already excludes it, because the rest of +// its line is longer and lighter. +// +// Nothing else here reads the text. A heading is otherwise a typographic fact in +// these documents, and requiring, say, no terminal full stop would fail on +// "Wskazóki dotyczące utylizacji | Obsługa serwisowa | Gwarancja", which is a +// running head with two pipes in it. +func headingLevel(l *textLine, measure float64, body bodyFace) (level int, note string, ok bool) { + if !hasLetter(l.text) { + return 0, "", false + } + if l.weight <= body.weight { + return 0, "", false + } + if measure > 0 && l.x1-l.x0 > headingMaxMeasureFraction*measure { + return 0, "", false + } + + level = 2 + if l.size > body.size { + level = 1 + } + return level, fmt.Sprintf("%gpt %s against a %gpt %s body, %.0f%% of the measure", + l.size, l.weight, body.size, body.weight, + 100*(l.x1-l.x0)/math.Max(measure, 1)), true +} + +// BlockSummary describes a document's blocks in one line, for logs and for a test +// that wants the shape rather than every row. +func BlockSummary(blocks []Block) string { + if len(blocks) == 0 { + return "no blocks" + } + byKind := make(map[BlockKind]int, 4) + pages := make(map[int]bool, 64) + chars, headings := 0, 0 + for i := range blocks { + b := &blocks[i] + byKind[b.Kind]++ + pages[b.Page] = true + chars += b.Chars + if b.Kind == BlockHeading { + headings++ + } + } + parts := make([]string, 0, len(byKind)) + for _, k := range []BlockKind{BlockHeading, BlockParagraph, BlockListItem, BlockTable, BlockFigure} { + if byKind[k] > 0 { + parts = append(parts, fmt.Sprintf("%d %s", byKind[k], k)) + } + } + return fmt.Sprintf("%d blocks over %d pages, %d chars: %s", + len(blocks), len(pages), chars, strings.Join(parts, ", ")) +} diff --git a/internal/doc/blocks_fixture_test.go b/internal/doc/blocks_fixture_test.go new file mode 100644 index 0000000..22a39d4 --- /dev/null +++ b/internal/doc/blocks_fixture_test.go @@ -0,0 +1,615 @@ +package doc_test + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + "unicode" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/fixture" +) + +// The acceptance tests docs/design/conversion.md asks for, in its own words: the +// column manual's German must come back as readable content in reading order, +// from the German column alone, with no Polish, Russian, Ukrainian or Kazakh text +// in it; and the sequential manual's German section must come back the same way +// from a page it owns outright. +// +// The negative is the one the contract calls the failure a reader would notice +// immediately, so it is asserted directly rather than inferred from a count. +// +// Both were also checked against 108 dpi renders while this was written — pages 62 +// and 14 of the column manual and pages 23 and 24 of the sequential one — and the +// heading counts those pages produce are pinned below so that a change which +// silently loses them says so. + +// blocksOfFixture reads a document's regions and the runs they were measured from. +// +// Two poppler passes, because doc.Result deliberately does not carry the runs: +// they are 3.8 MB of coordinates for the sequential manual and nothing stored +// needs them. Analyze is the only thing that resolves a page's language, and +// regions are what carry it into a block. +func blocksOfFixture(t *testing.T, name string) (m *fixture.Manifest, pages []doc.PageRuns, regions []doc.Region) { + t.Helper() + m, pages, regions, _ = blocksAndTablesOfFixture(t, name) + return m, pages, regions +} + +// blocksAndTablesOfFixture adds the ruled lines of the pages a scope will read. +// +// Only those pages, because the ruled lines cost a pdftocairo spawn each — 5.9 s +// over the column manual's 68 pages, 42.3 s over the sequential manual's 560 — and +// conversion.md's cost argument is precisely that they are read for the pages in +// scope and no others. A German scope is 26 pages of the one and 16 of the other. +func blocksAndTablesOfFixture(t *testing.T, name string) (m *fixture.Manifest, + pages []doc.PageRuns, regions []doc.Region, tables map[int][]doc.RuledTable) { + t.Helper() + m, pages, regions, path := regionsOfFixture(t, name) + + tables = map[int][]doc.RuledTable{} + want := map[int]bool{} + for i := range regions { + if doc.BaseLanguage(regions[i].Lang) == "de" { + want[regions[i].Page] = true + } + } + for i := range pages { + if !want[pages[i].No] { + continue + } + got, err := doc.PageTables(context.Background(), path, &pages[i]) + if err != nil { + t.Skipf("the ruled lines of page %d could not be read: %v", pages[i].No, err) + } + if len(got) > 0 { + tables[pages[i].No] = got + } + } + return m, pages, regions, tables +} + +func regionsOfFixture(t *testing.T, name string) (m *fixture.Manifest, pages []doc.PageRuns, + regions []doc.Region, path string) { + t.Helper() + if name == "thomas-drybox-amfibia" { + m, path = columnFixture(t) + } else { + m, path = loadFixture(t) + } + + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if res.RegionNote != "" { + t.Skipf("no regions were produced: %s", res.RegionNote) + } + pages, err = doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + return m, pages, res.Regions, path +} + +// scriptsIn reports which of the alphabets these two manuals use appear in a +// string, so that "no Russian in the German blocks" can be asserted on the letters +// rather than on a detector's opinion of them. +// +// Cyrillic is the discriminator that matters and it is exact: the column manual's +// five languages are German, Polish, Russian, Ukrainian and Kazakh, and the last +// three are the only Cyrillic ones. A single Cyrillic letter in a German block is +// the funnel leaking. +func scriptsIn(s string) map[string]int { + out := map[string]int{} + for _, r := range s { + switch { + case unicode.Is(unicode.Cyrillic, r): + out["cyrillic"]++ + case unicode.Is(unicode.Greek, r): + out["greek"]++ + case unicode.Is(unicode.Han, r), unicode.Is(unicode.Hiragana, r), unicode.Is(unicode.Katakana, r): + out["cjk"]++ + case unicode.Is(unicode.Arabic, r): + out["arabic"]++ + case unicode.Is(unicode.Hebrew, r): + out["hebrew"]++ + } + } + return out +} + +// polishOnlyLetters are the letters Polish uses and German does not. German has +// no way to produce any of them, so one in a German block came from the column +// beside it. +const polishOnlyLetters = "ąćęłńśźżĄĆĘŁŃŚŹŻ" + +// TestBlocksOfTheColumnManualsGermanAreGerman is the first half of acceptance and +// the negative the contract insists on: the German regions of a page holding five +// languages must produce German and nothing else. +func TestBlocksOfTheColumnManualsGermanAreGerman(t *testing.T) { + _, pages, regions, tables := blocksAndTablesOfFixture(t, "thomas-drybox-amfibia") + + german := doc.RegionsBlocks(pages, regions, map[string]bool{"de": true}, tables, nil) + if len(german) == 0 { + t.Fatal("the German regions produced no blocks at all") + } + t.Logf("German: %s", doc.BlockSummary(german)) + + // The whole promise. Not a threshold and not a fraction: one Cyrillic letter in + // a German block means a Russian, Ukrainian or Kazakh column was read. + leaked := 0 + for i := range german { + b := &german[i] + if n := scriptsIn(b.Text)["cyrillic"]; n > 0 { + leaked++ + if leaked <= 5 { + t.Errorf("page %d block %d holds %d Cyrillic letters: %q", + b.Page, b.Index, n, truncate(b.Text, 120)) + } + } + } + if leaked > 0 { + t.Errorf("%d of %d German blocks carry Cyrillic text; this manual's other three "+ + "languages are Russian, Ukrainian and Kazakh", leaked, len(german)) + } + + // Polish shares the Latin alphabet, so it needs its own letters rather than its + // script. It is the column immediately beside German on most of these pages. + polish := 0 + for i := range german { + b := &german[i] + if strings.ContainsAny(b.Text, polishOnlyLetters) { + polish++ + if polish <= 5 { + t.Errorf("page %d block %d holds Polish-only letters: %q", + b.Page, b.Index, truncate(b.Text, 120)) + } + } + } + if polish > 0 { + t.Errorf("%d of %d German blocks carry Polish letters", polish, len(german)) + } + + // German has to actually be there, or a test that only forbids other languages + // passes on an empty result. Umlauts and eszett are what German writes and none + // of the other four does. + umlauts := 0 + for i := range german { + umlauts += strings.Count(german[i].Text, "ä") + strings.Count(german[i].Text, "ö") + + strings.Count(german[i].Text, "ü") + strings.Count(german[i].Text, "ß") + } + if umlauts < 200 { + t.Errorf("the German blocks carry only %d umlauts and eszetts over %d blocks; "+ + "this is 26 pages of German", umlauts, len(german)) + } + t.Logf(" %d umlauts and eszetts, no Cyrillic, no Polish-only letters", umlauts) +} + +// TestBlocksNeverReachOutsideTheirRegion states the funnel geometrically, which is +// the only way to state it that does not depend on which languages a document +// happens to hold. +// +// The test above it guards Cyrillic and Polish-only letters, and that is not +// enough. Page 57 of this manual USED to divide into four regions — Finnish at +// x=36-178 and German at 179-424, 457-589 and 601-846 — and Finnish shares the +// Latin alphabet AND ä and ö with German, so a Finnish column bleeding into a +// German block passes every letter test there is. +// +// Those four boundaries turned out to be the cell dividers of the page's two +// tables, measured at x=29.7-428.1 and x=450.2-848.7, and the page is now one +// German region because of it. But the property this test states is the one that +// outlives that: a table's area is handed to the row walk, and a table CAN reach +// past the region being read. The left table did exactly that, across two regions +// in two different languages. So every table of this document is read here against +// every region of its page, and a block that reaches outside its region's box is a +// failure however the page came to be divided. +// +// The hermetic twin of the case that no longer occurs on this document is +// TestRegionBlocksClipATableToTheRegion, which rebuilds page 57's pre-fix shape and +// asserts the same thing about it. +func TestBlocksNeverReachOutsideTheirRegion(t *testing.T) { + _, pages, regions, path := regionsOfFixture(t, "thomas-drybox-amfibia") + + byNo := make(map[int]*doc.PageRuns, len(pages)) + for i := range pages { + byNo[pages[i].No] = &pages[i] + } + + // Every page, not only the ones in a scope: the point is to run every table the + // document draws past every region, and 68 pdftocairo spawns is 6 s. + tables := map[int][]doc.RuledTable{} + tabled := 0 + for i := range pages { + got, err := doc.PageTables(context.Background(), path, &pages[i]) + if err != nil { + t.Skipf("the ruled lines of page %d could not be read: %v", pages[i].No, err) + } + if len(got) > 0 { + tables[pages[i].No] = got + tabled++ + } + } + + checked, boxed, tableBlocks := 0, 0, 0 + for i := range regions { + r := ®ions[i] + p := byNo[r.Page] + if p == nil { + continue + } + if r.X0 != 0 { + boxed++ + } + for _, b := range doc.RegionBlocks(p, r, tables[r.Page], nil) { + checked++ + if b.Kind == doc.BlockTable { + tableBlocks++ + } + // The one-unit slack is runsInBox's own tolerance, which absorbs the + // rounding between a column's reported extent and the runs that produced it. + if b.X0 < r.X0-1 || b.X1 > r.X1+1 { + t.Errorf("page %d block %d spans x=%.1f-%.1f, outside its %q region at "+ + "x=%.1f-%.1f: %q", b.Page, b.Index, b.X0, b.X1, r.Lang, r.X0, r.X1, + truncate(b.Text, 100)) + } + } + } + if checked == 0 || boxed == 0 || tableBlocks == 0 { + t.Fatalf("checked %d blocks over %d boxed regions, %d of them table cells; all "+ + "three must be non-zero or this test asserts nothing", checked, boxed, tableBlocks) + } + t.Logf("%d blocks over %d regions, %d of them boxed, %d table cells from %d tabled "+ + "pages, none reaching outside its box", checked, len(regions), boxed, tableBlocks, tabled) +} + +// TestBlocksOfTheColumnManualsPage62 pins what one page produces, because a +// document-level count can stay right while every page goes wrong. This page was +// rendered at 108 dpi and read: a two-column German page with four headings +// (Hinweis zur Entsorgung, Kundendienst, Technische Daten, Garantie), four +// bulleted disposal items on the left, six numbered guarantee clauses on the +// right, and the unruled specification table between them. +func TestBlocksOfTheColumnManualsPage62(t *testing.T) { + _, pages, regions := blocksOfFixture(t, "thomas-drybox-amfibia") + + blocks := blocksOfPage(t, pages, regions, 62, "de") + for i := range blocks { + t.Logf(" [%2d] %-10s L%d %q", blocks[i].Index, blocks[i].Kind, blocks[i].Level, + truncate(blocks[i].Text, 90)) + } + + var headings []string + for i := range blocks { + if blocks[i].Kind == doc.BlockHeading { + headings = append(headings, blocks[i].Text) + } + } + want := []string{"Garantie", "Hinweis zur Entsorgung", "Kundendienst", "Technische Daten"} + got := append([]string(nil), headings...) + sort.Strings(got) + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("headings = %q, the render of this page shows %q", got, want) + } + + // The six numbered guarantee clauses, each with its hanging-indented body. Not + // six one-line items followed by six orphaned paragraphs, which is what this + // looked like before the hanging indent was handled. + numbered := 0 + for i := range blocks { + if blocks[i].Kind == doc.BlockListItem && strings.HasPrefix(blocks[i].Text, "1.") { + if blocks[i].Lines < 2 { + t.Errorf("guarantee clause 1 is one line: %q", blocks[i].Text) + } + if !strings.Contains(blocks[i].Text, "gewerblicher Benutzung") { + t.Errorf("guarantee clause 1 lost its continuation: %q", blocks[i].Text) + } + } + if blocks[i].Kind == doc.BlockListItem { + numbered++ + } + } + // Four bulleted plus six numbered. + if numbered != 10 { + t.Errorf("%d list items, the render shows four bulleted disposal items and six "+ + "numbered guarantee clauses", numbered) + } + + // The specification table has no ruling anywhere and nothing detects it as a + // table — conversion.md records that and accepts it, on the grounds that the + // page still reads correctly as lines of text. That is only true if each row is + // its own block, so it is asserted rather than assumed. + for _, row := range []string{ + "Spannungsversorgung: 230 V, 50 Hz", + "Länge Stromzuleitung: ca. 8 m", + } { + found := false + for i := range blocks { + if blocks[i].Text == row { + found = true + } + } + if !found { + t.Errorf("no block reads exactly %q; an unruled specification row has to be "+ + "its own line of text or the page does not read correctly", row) + } + } +} + +// TestBlocksOfTheColumnManualsPage14 is the boxed case: a page of three parallel +// columns where German is the middle one. Rendered at 108 dpi and read: an +// underlined section heading, three warning paragraphs, and bold step captions +// each followed by their instruction. +func TestBlocksOfTheColumnManualsPage14(t *testing.T) { + _, pages, regions := blocksOfFixture(t, "thomas-drybox-amfibia") + + blocks := blocksOfPage(t, pages, regions, 14, "de") + for i := range blocks { + t.Logf(" [%2d] %-10s L%d %q", blocks[i].Index, blocks[i].Kind, blocks[i].Level, + truncate(blocks[i].Text, 90)) + } + + if blocks[0].RegionX0 == 0 { + t.Errorf("the German region of this page starts at x=0; it is the middle column of "+ + "three and must be boxed — %d blocks", len(blocks)) + } + for _, want := range []string{"Bedienung zum Trockensaugen", "Öffnen Sie den Gehäusedeckel."} { + found := false + for i := range blocks { + if blocks[i].Kind == doc.BlockHeading && blocks[i].Text == want { + found = true + } + } + if !found { + t.Errorf("no heading reads %q; the render shows it set as one", want) + } + } + // Nothing from the Polish column to its right. + for i := range blocks { + if strings.ContainsAny(blocks[i].Text, polishOnlyLetters) { + t.Errorf("block %d of the middle column holds Polish letters: %q", + i, truncate(blocks[i].Text, 120)) + } + } +} + +// TestBlocksOfTheSequentialManualsGermanSection is the other half of acceptance: +// the same reading from a document that gives German pages of its own, 23 to 38. +func TestBlocksOfTheSequentialManualsGermanSection(t *testing.T) { + _, pages, regions, tables := blocksAndTablesOfFixture(t, "dreame-l40-ultra") + + german := doc.RegionsBlocks(pages, regions, map[string]bool{"de": true}, tables, nil) + if len(german) == 0 { + t.Fatal("the German section produced no blocks at all") + } + t.Logf("German: %s", doc.BlockSummary(german)) + + // The pages must be the section the manifest records, and only those. + seen := map[int]bool{} + for i := range german { + seen[german[i].Page] = true + } + var outside []int + for page := range seen { + if page < 23 || page > 38 { + outside = append(outside, page) + } + } + sort.Ints(outside) + if len(outside) > 0 { + t.Errorf("German blocks appear on pages %v, outside the section's 23-38", outside) + } + + // No other writing system. This document has Greek, Arabic, Hebrew and CJK + // sections, so a block escaping its region here is loud. + for i := range german { + b := &german[i] + for script, n := range scriptsIn(b.Text) { + t.Errorf("page %d block %d holds %d %s characters: %q", + b.Page, b.Index, n, script, truncate(b.Text, 120)) + break + } + } + + umlauts := 0 + for i := range german { + umlauts += strings.Count(german[i].Text, "ä") + strings.Count(german[i].Text, "ö") + + strings.Count(german[i].Text, "ü") + strings.Count(german[i].Text, "ß") + } + if umlauts < 200 { + t.Errorf("the German blocks carry only %d umlauts and eszetts; this is 16 pages "+ + "of German", umlauts) + } + t.Logf(" %d umlauts and eszetts over pages %v", umlauts, sortedKeys(seen)) +} + +// TestBlocksOfTheSequentialManualsPages23And24 pins the two pages that were +// rendered and read. Both are safety pages set entirely in 17pt with one 21pt +// heading, which is the measurement the heading rule turns on: against the +// document's 11pt body every line of them is "larger than body". +func TestBlocksOfTheSequentialManualsPages23And24(t *testing.T) { + _, pages, regions := blocksOfFixture(t, "dreame-l40-ultra") + + for _, tc := range []struct { + page int + wantHeadings []string + wantItems int + }{ + // Page 23: the section heading, the printed DE tab, one intro paragraph, the + // subheading, then seven bullets. The tab is furniture this pass does not + // identify — see the file comment. + {23, []string{"Sicherheitshinweise", "Nutzungsbeschränkungen"}, 7}, + // Page 24 is twelve bullets under the same heading and nothing else. + {24, []string{"Sicherheitshinweise"}, 12}, + } { + blocks := blocksOfPage(t, pages, regions, tc.page, "de") + t.Logf("--- page %d", tc.page) + for i := range blocks { + t.Logf(" [%2d] %-10s L%d %q", blocks[i].Index, blocks[i].Kind, blocks[i].Level, + truncate(blocks[i].Text, 90)) + } + + for _, want := range tc.wantHeadings { + found := false + for i := range blocks { + if blocks[i].Kind == doc.BlockHeading && blocks[i].Text == want { + found = true + } + } + if !found { + t.Errorf("page %d: no heading reads %q", tc.page, want) + } + } + items := 0 + for i := range blocks { + if blocks[i].Kind == doc.BlockListItem { + items++ + } + } + if items != tc.wantItems { + t.Errorf("page %d: %d list items, the render shows %d bullets", + tc.page, items, tc.wantItems) + } + + // Every bullet must keep its own text. A bullet whose second line was + // orphaned reads as a truncated instruction, which is the failure mode that + // matters on a safety page. + for i := range blocks { + b := &blocks[i] + if b.Kind == doc.BlockListItem && b.Chars < 20 { + t.Errorf("page %d block %d is a %d-character list item: %q", + tc.page, b.Index, b.Chars, b.Text) + } + } + } +} + +// TestBlocksConvergeOnASecondRun is the idempotence conversion.md requires of a +// job handler, checked on a real document rather than on a built page: the key is +// the page, the region's left edge and the index, so a second conversion has to +// produce the same keys and the same text. +func TestBlocksConvergeOnASecondRun(t *testing.T) { + _, pages, regions, tables := blocksAndTablesOfFixture(t, "thomas-drybox-amfibia") + + first := doc.RegionsBlocks(pages, regions, map[string]bool{"de": true}, tables, nil) + second := doc.RegionsBlocks(pages, regions, map[string]bool{"de": true}, tables, nil) + + if len(first) != len(second) { + t.Fatalf("two runs produced %d and %d blocks", len(first), len(second)) + } + for i := range first { + a, b := &first[i], &second[i] + if a.Page != b.Page || a.RegionX0 != b.RegionX0 || a.Index != b.Index || a.Text != b.Text { + t.Fatalf("block %d differs between runs: page %d/%d x0 %.0f/%.0f index %d/%d", + i, a.Page, b.Page, a.RegionX0, b.RegionX0, a.Index, b.Index) + } + } + + // The keys have to be unique, or two blocks collide in storage and the second + // conversion overwrites rather than converges. + seen := make(map[string]bool, len(first)) + for i := range first { + b := &first[i] + key := itoa(b.Page) + "/" + itoa(int(b.RegionX0)) + "/" + itoa(b.Index) + if seen[key] { + t.Errorf("two blocks share the key %s", key) + } + seen[key] = true + } +} + +// TestBlocksHeadingLengthIsASoftCut pins the honest state of the length rule +// rather than a gap that is not there. +// +// The comment on headingMaxMeasureFraction records the measurement: the share of +// the measure that heading candidates occupy is a smooth continuum from 5% to +// 100% on both manuals, in runes as well as in width, so 0.6 is a cut chosen for +// precision and not a valley. Two things follow that a test can hold: +// +// - Real headings reach the cut. The widest German heading that survives is at +// 60%, so anyone lowering the threshold is trading headings away and should see +// that in a failure rather than in a diff. +// - Nothing near the full measure is a heading. That is the property the cut +// buys, and it is what keeps the column manual's medium-face warning +// paragraphs — 17.2% of its characters — out of the reader as furniture. +func TestBlocksHeadingLengthIsASoftCut(t *testing.T) { + _, pages, regions, tables := blocksAndTablesOfFixture(t, "thomas-drybox-amfibia") + + blocks := doc.RegionsBlocks(pages, regions, map[string]bool{"de": true}, tables, nil) + widest, widestText := 0.0, "" + for i := range blocks { + b := &blocks[i] + if b.Kind != doc.BlockHeading { + continue + } + // The note carries the share of the measure the heading's first line occupies. + var pct float64 + if _, err := sscanPercent(b.Note, &pct); err != nil { + t.Errorf("heading %q reports no share of the measure: %q", b.Text, b.Note) + continue + } + if pct > widest { + widest, widestText = pct, b.Text + } + if pct > 60 { + t.Errorf("heading %q occupies %.0f%% of its measure, above the cut", b.Text, pct) + } + } + + t.Logf("the widest German heading occupies %.0f%% of its measure: %q", widest, widestText) + // A heading reaching the cut is the expected state, not a defect. If the widest + // one drops well below it, the cut has started to cost headings that used to be + // found and the trade should be looked at again rather than inherited. + if widest < 55 { + t.Errorf("the widest German heading occupies only %.0f%% of its measure against a "+ + "cut at 60%%; headings that reached the cut have been lost", widest) + } +} + +func blocksOfPage(t *testing.T, pages []doc.PageRuns, regions []doc.Region, page int, lang string) []doc.Block { + t.Helper() + for i := range regions { + r := ®ions[i] + if r.Page != page || doc.BaseLanguage(r.Lang) != lang { + continue + } + for j := range pages { + if pages[j].No == page { + got := doc.RegionBlocks(&pages[j], r, nil, nil) + if len(got) == 0 { + t.Fatalf("page %d region x=%.0f produced no blocks", page, r.X0) + } + return got + } + } + } + t.Fatalf("no %s region on page %d", lang, page) + return nil +} + +// sscanPercent reads the trailing "NN% of the measure" out of a heading's note. +func sscanPercent(note string, out *float64) (int, error) { + i := strings.LastIndex(note, ", ") + if i < 0 { + return 0, fmt.Errorf("note %q carries no measure share", note) + } + return fmt.Sscanf(note[i+2:], "%g%%", out) +} + +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} + +func sortedKeys(m map[int]bool) []int { + out := make([]int, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Ints(out) + return out +} diff --git a/internal/doc/blocks_internal_test.go b/internal/doc/blocks_internal_test.go new file mode 100644 index 0000000..3101317 --- /dev/null +++ b/internal/doc/blocks_internal_test.go @@ -0,0 +1,42 @@ +package doc + +import "testing" + +// TestAContentsEntryIsRecognisedByItsLeaderAndItsPageNumber pins both halves of the +// signal, on the real strings of the columns manual's contents page. +// +// Both are required, and the four short dot runs that document also prints are why: +// measured over both whole manuals, the runs of two or more dots are 3, 3, 3, 4 and +// then 34 to 91, with nothing in between, and the four short ones are ellipses in +// prose carrying no page number. Either test alone would be enough here; both are +// kept because "a leader" and "a page at the end of it" are what a contents entry is, +// and the next document gets no say in which of the two it happens to break. +func TestAContentsEntryIsRecognisedByItsLeaderAndItsPageNumber(t *testing.T) { + for _, tc := range []struct { + name string + text string + want bool + }{ + {"a plain entry", "Мы поздравляем Вас ........................................................................2", true}, + {"a page range, en dash", "Trockensaugen . ........................................................................14 – 22", true}, + {"a title holding its own digits", "Reinigung der AQUA-Box ...........................................................40 – 44", true}, + {"the leader in a run of its own, joined on one baseline", "Ihr THOMAS ...................................11", true}, + + {"an ellipsis in prose", "und so weiter ... aber nicht mehr", false}, + {"an ellipsis before a number", "warten Sie ... 30 Sekunden lang", false}, + {"a leader with nothing after it", "Fehlerbehebung ............................................", false}, + {"a page number with no leader", "Fehlerbehebung 57", false}, + {"dots and digits with no title", " ........................................ 12", false}, + {"empty", " ", false}, + } { + t.Run(tc.name, func(t *testing.T) { + dots, got := contentsEntry(tc.text) + if got != tc.want { + t.Errorf("contentsEntry(%q) = %v (leader %d), want %v", tc.text, got, dots, tc.want) + } + if got && dots < minLeaderDots { + t.Errorf("reported a leader of %d, under the floor of %d", dots, minLeaderDots) + } + }) + } +} diff --git a/internal/doc/blocks_test.go b/internal/doc/blocks_test.go new file mode 100644 index 0000000..5e3fd4b --- /dev/null +++ b/internal/doc/blocks_test.go @@ -0,0 +1,583 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for doc.RegionBlocks. No PDF and no poppler: the runs are built +// here, so each rule is stated where it can be read against the reasoning in +// blocks.go. The real-document acceptance lives in blocks_fixture_test.go, and +// every shape below is drawn from something one of the two manuals actually does. +// +// The font sizes are the ones poppler reports for these documents, not point +// sizes: 11, 15, 17 and 21 in the 1.5-scaled space [doc.Font] describes. + +const ( + testBlockPageWidth = 918 + testBlockPageHeight = 620 +) + +// blockPage builds a page of lines, one run each, at a fixed pitch. +type line struct { + y float64 + x float64 + w float64 + size float64 + weight doc.Weight + bold bool + text string +} + +func blockPage(no int, lines ...line) *doc.PageRuns { + p := &doc.PageRuns{No: no, Width: testBlockPageWidth, Height: testBlockPageHeight} + for _, l := range lines { + p.Runs = append(p.Runs, doc.TextRun{ + X: l.x, Y: l.y, Width: l.w, Height: l.size + 5, Text: l.text, + Font: doc.Font{ + Size: l.size, Family: "Test-Face", Weight: l.weight, MarkedBold: l.bold, + }, + }) + } + return p +} + +// wholePage is the region a page in one language produces — rule 1 or 3 of +// doc.PageRegions, which is most of both manuals. +func wholePage(page int) *doc.Region { + return &doc.Region{Page: page, X0: 0, X1: testBlockPageWidth, Lang: "de"} +} + +// bodyLines returns n body lines at the given pitch, each set to the full measure. +func bodyLines(y0, pitch float64, n int, text string) []line { + out := make([]line, 0, n) + for i := 0; i < n; i++ { + out = append(out, line{y: y0 + float64(i)*pitch, x: 55, w: 700, size: 17, text: text}) + } + return out +} + +func kinds(blocks []doc.Block) []doc.BlockKind { + out := make([]doc.BlockKind, len(blocks)) + for i := range blocks { + out[i] = blocks[i].Kind + } + return out +} + +// TestRegionBlocksHeadingThenParagraphs is the ordinary shape of a manual page: +// the sequential manual's page 23, a 21pt semibold heading over 17pt prose. +func TestRegionBlocksHeadingThenParagraphs(t *testing.T) { + lines := []line{{y: 20, x: 55, w: 200, size: 21, weight: doc.WeightSemibold, bold: true, + text: "Sicherheitshinweise"}} + lines = append(lines, bodyLines(70, 22, 4, "Lesen Sie die Bedienungsanleitung vor der Verwendung")...) + lines = append(lines, bodyLines(200, 22, 4, "Bewahren Sie sie zum spaeteren Nachschlagen auf")...) + + got := doc.RegionBlocks(blockPage(23, lines...), wholePage(23), nil, nil) + + if len(got) != 3 { + t.Fatalf("got %d blocks, want a heading and two paragraphs: %v", len(got), kinds(got)) + } + if got[0].Kind != doc.BlockHeading { + t.Errorf("first block is %s, want a heading — %s", got[0].Kind, got[0].Note) + } + if got[0].Level != 1 { + t.Errorf("heading level = %d, want 1: it is set larger than the body", got[0].Level) + } + if got[0].Text != "Sicherheitshinweise" { + t.Errorf("heading text = %q", got[0].Text) + } + for i := 1; i < 3; i++ { + if got[i].Kind != doc.BlockParagraph { + t.Errorf("block %d is %s, want a paragraph — %s", i, got[i].Kind, got[i].Note) + } + if got[i].Lines != 4 { + t.Errorf("block %d folded %d lines, want the 4 written into it", i, got[i].Lines) + } + } +} + +// TestRegionBlocksSplitParagraphsOnAGap is the paragraph rule on its own: the +// same face throughout, so only the vertical gap can separate them. +func TestRegionBlocksSplitParagraphsOnAGap(t *testing.T) { + var lines []line + lines = append(lines, bodyLines(20, 22, 5, "erster Absatz")...) + // 44 is two pitches down, which is what a blank line looks like. + lines = append(lines, bodyLines(20+5*22+44, 22, 5, "zweiter Absatz")...) + + got := doc.RegionBlocks(blockPage(30, lines...), wholePage(30), nil, nil) + + if len(got) != 2 { + t.Fatalf("got %d blocks, want 2: %v", len(got), kinds(got)) + } + if !strings.Contains(got[0].Text, "erster") || strings.Contains(got[0].Text, "zweiter") { + t.Errorf("the first paragraph reads %q; the gap did not end it", got[0].Text) + } +} + +// TestRegionBlocksKeepAParagraphWhoseLinesAreOnePitchApart is the other side of +// that rule, and the one a too-eager gap threshold breaks: every line of a +// paragraph is a gap, and turning each into a block is worse than merging two. +func TestRegionBlocksKeepAParagraphWhoseLinesAreOnePitchApart(t *testing.T) { + got := doc.RegionBlocks(blockPage(31, bodyLines(20, 22, 8, "eine Zeile")...), wholePage(31), nil, nil) + + if len(got) != 1 { + t.Fatalf("got %d blocks for one paragraph of 8 lines: %v", len(got), kinds(got)) + } + if got[0].Lines != 8 { + t.Errorf("paragraph folded %d lines, want 8", got[0].Lines) + } +} + +// TestRegionBlocksReadAListWithHangingIndents is the column manual's guarantee +// clauses and the sequential manual's safety bullets: a marker line, then lines +// indented under it that belong to the same item. +func TestRegionBlocksReadAListWithHangingIndents(t *testing.T) { + lines := []line{ + {y: 20, x: 55, w: 700, size: 17, text: "• Bei Beschaedigung des Netzkabels muss es ersetzt"}, + {y: 42, x: 77, w: 600, size: 17, text: "werden, die Sie beim Hersteller erhalten koennen."}, + {y: 64, x: 55, w: 700, size: 17, text: "• Benutzen Sie den Roboter nicht in einem Bereich,"}, + {y: 86, x: 77, w: 600, size: 17, text: "der ueber dem Boden freisteht."}, + {y: 108, x: 55, w: 700, size: 17, text: "• Stellen Sie den Roboter nicht auf den Kopf."}, + {y: 130, x: 55, w: 700, size: 17, text: "• Halten Sie Haare und Finger fern."}, + } + got := doc.RegionBlocks(blockPage(24, lines...), wholePage(24), nil, nil) + + if len(got) != 4 { + t.Fatalf("got %d blocks, want 4 list items: %v", len(got), kinds(got)) + } + for i := range got { + if got[i].Kind != doc.BlockListItem { + t.Errorf("block %d is %s, want a list item — %s", i, got[i].Kind, got[i].Note) + } + } + // The indented continuation must be inside its item, not a paragraph after it. + if !strings.Contains(got[0].Text, "beim Hersteller") { + t.Errorf("the first item reads %q; its second line was orphaned", got[0].Text) + } + if got[0].Lines != 2 || got[2].Lines != 1 { + t.Errorf("items folded %d and %d lines, want 2 and 1", got[0].Lines, got[2].Lines) + } +} + +// TestRegionBlocksReadAListNumberedWithoutPunctuation is the column manual's +// parts lists, which print " 1 " and then the part's name with nothing between +// but a tab. Without the gap test those numbers are prose and page 11's nine +// parts fold into one paragraph. +func TestRegionBlocksReadAListNumberedWithoutPunctuation(t *testing.T) { + p := &doc.PageRuns{No: 11, Width: testBlockPageWidth, Height: testBlockPageHeight} + names := []string{"Gehaeusedeckel", "Tragegriff", "Ansaugstutzen", "Schnellkupplung", + "Laufraeder", "Netzstecker", "Frischwassertank", "Hauptschalter"} + for i, name := range names { + y := 62 + float64(i)*15 + f := doc.Font{Size: 14, Family: "Test-Face", Weight: doc.WeightLight} + p.Runs = append(p.Runs, + doc.TextRun{X: 591, Y: y, Width: 10, Height: 17, Text: " " + itoa(i+1) + " ", Font: f}, + doc.TextRun{X: 621, Y: y, Width: 120, Height: 17, Text: name, Font: f}) + } + + got := doc.RegionBlocks(p, &doc.Region{Page: 11, X0: 0, X1: testBlockPageWidth, Lang: "de"}, nil, nil) + + if len(got) != len(names) { + t.Fatalf("got %d blocks for %d numbered parts: %v", len(got), len(names), kinds(got)) + } + for i := range got { + if got[i].Kind != doc.BlockListItem { + t.Errorf("part %d is %s, want a list item — %s", i+1, got[i].Kind, got[i].Note) + } + if !strings.Contains(got[i].Text, names[i]) { + t.Errorf("part %d reads %q, want it to contain %q", i+1, got[i].Text, names[i]) + } + } +} + +// TestRegionBlocksNeverPromoteSafetyCopyToAHeading is the measurement the whole +// heading rule exists for, and the revert check for it. +// +// docs/design/conversion.md records it in numbers: 17pt regular is 14.1% of the +// sequential manual at 65 characters a run, and it is safety prose. A rule that +// promotes what is larger than the body turns every line of it into a heading. +// The real heading here is smaller than that prose and heavier, which is also +// measured — "Nutzungsbeschränkungen" is 15pt semibold on a page set in 17. +// This is page 23 of the sequential manual in miniature, which is the shape that +// discriminates: the safety prose IS the body of its own region, so the real +// heading is SMALLER than the text it heads, and the display line above it is +// larger than the body in the same weight. Size alone gets both backwards, and +// the assertion is deliberately two-sided so that reverting the weight test fails +// twice rather than looking like a rounding difference. +func TestRegionBlocksNeverPromoteSafetyCopyToAHeading(t *testing.T) { + lines := []line{ + // A short display line, larger than the body and in the body's own weight. + // This is the safety copy set at a size above the measure on the page it + // leads — the case conversion.md measured at 14.1% of a 560-page manual. + {y: 20, x: 55, w: 260, size: 21, weight: doc.WeightRegular, text: "Wichtiger Hinweis"}, + // The body of this region: 17pt regular, the most characters by far. + {y: 60, x: 55, w: 700, size: 17, weight: doc.WeightRegular, + text: "Lesen Sie die Bedienungsanleitung vor der Verwendung sorgfaeltig"}, + {y: 82, x: 55, w: 700, size: 17, weight: doc.WeightRegular, + text: "durch und bewahren Sie sie zum spaeteren Nachschlagen auf damit"}, + {y: 104, x: 55, w: 700, size: 17, weight: doc.WeightRegular, + text: "Stromschlaege Braende oder Verletzungen durch unsachgemaessen"}, + {y: 126, x: 55, w: 700, size: 17, weight: doc.WeightRegular, + text: "Gebrauch des Geraetes vermieden werden in Wohnraeumen und auch"}, + {y: 148, x: 55, w: 700, size: 17, weight: doc.WeightRegular, + text: "ausserhalb davon auf allen Bodenbelaegen die dafuer geeignet sind"}, + // The real heading: smaller than the body it heads, and heavier. + {y: 190, x: 55, w: 190, size: 15, weight: doc.WeightSemibold, bold: true, + text: "Nutzungsbeschraenkungen"}, + {y: 220, x: 55, w: 700, size: 17, weight: doc.WeightRegular, + text: "Um einen sicheren Betrieb dieses Produkts zu gewaehrleisten darf"}, + {y: 242, x: 55, w: 700, size: 17, weight: doc.WeightRegular, + text: "es nicht von Kindern unter acht Jahren benutzt werden oder von"}, + } + got := doc.RegionBlocks(blockPage(23, lines...), wholePage(23), nil, nil) + + var headings []string + for i := range got { + if got[i].Kind == doc.BlockHeading { + headings = append(headings, got[i].Text) + } + } + if len(headings) != 1 || headings[0] != "Nutzungsbeschraenkungen" { + t.Errorf("headings = %q, want only the semibold one. The 15pt semibold heading is "+ + "SMALLER than the 17pt body it heads, and the 21pt regular display line is "+ + "larger in the body's own weight, so size alone gets both backwards", headings) + for i := range got { + t.Logf(" %-10s L%d %q — %s", got[i].Kind, got[i].Level, got[i].Text, got[i].Note) + } + } + // And its level: at 15pt against a 17pt body it is not the most prominent thing + // on the page, which is what level 2 records. + if len(headings) == 1 && got[len(got)-2].Level == 1 { + t.Errorf("the heading is level 1 although it is set smaller than the body") + } +} + +// TestRegionBlocksNeverPromoteAParagraphTail is the second half of that rule, and +// the correction the measurement forced: the last line of a paragraph is short by +// definition, so a paragraph set in a face heavier than the body hands its final +// line over as a heading. Measured, that produced 280 headings on the column +// manual reading "Umgebungen benutzt werden." and the like. +func TestRegionBlocksNeverPromoteAParagraphTail(t *testing.T) { + lines := []line{ + {y: 20, x: 30, w: 700, size: 14, text: "Der leichte Grundtext dieser Region traegt die meisten Zeichen"}, + {y: 36, x: 30, w: 700, size: 14, text: "und legt damit fest was hier als Grundschrift gilt und was nicht"}, + {y: 52, x: 30, w: 700, size: 14, text: "so dass jede schwerere Schrift als Auszeichnung gelesen wird"}, + {y: 68, x: 30, w: 700, size: 14, text: "und nicht schon deshalb als Ueberschrift durchgehen darf"}, + // A whole paragraph in the heavier face, whose last line is short. + {y: 100, x: 30, w: 700, size: 14, weight: doc.WeightMedium, + text: "Das Geraet darf nicht in explosionsgefaehrdeten Raeumen oder in"}, + {y: 116, x: 30, w: 700, size: 14, weight: doc.WeightMedium, + text: "unmittelbarer Naehe von brennbaren Stoffen und in feuchten"}, + {y: 132, x: 30, w: 250, size: 14, weight: doc.WeightMedium, + text: "Umgebungen benutzt werden."}, + } + got := doc.RegionBlocks(blockPage(4, lines...), wholePage(4), nil, nil) + + for i := range got { + if got[i].Kind == doc.BlockHeading { + t.Errorf("block %d is a heading reading %q — %s; it is the last line of the "+ + "paragraph above it, and a heading has to start a block", + i, got[i].Text, got[i].Note) + } + } + if len(got) != 2 { + t.Errorf("got %d blocks, want the light paragraph and the medium one: %v", + len(got), kinds(got)) + } +} + +// TestRegionBlocksNeverPromoteAFigureCallout is the third: an exploded diagram's +// numbers are larger AND heavier than the page's body and two characters long, so +// they pass every typographic test. 26 of them are on the column manual's page 11. +func TestRegionBlocksNeverPromoteAFigureCallout(t *testing.T) { + lines := []line{ + {y: 20, x: 30, w: 400, size: 14, weight: doc.WeightLight, text: "Der Grundtext dieser Seite"}, + {y: 36, x: 30, w: 400, size: 14, weight: doc.WeightLight, text: "traegt die meisten Zeichen"}, + {y: 52, x: 30, w: 400, size: 14, weight: doc.WeightLight, text: "und legt die Grundschrift fest"}, + {y: 68, x: 30, w: 400, size: 14, weight: doc.WeightLight, text: "damit sie vergleichbar wird"}, + // A callout: bigger, heavier, and set well away from anything. + {y: 200, x: 184, w: 13, size: 17, weight: doc.WeightMedium, text: "14"}, + {y: 240, x: 452, w: 13, size: 17, weight: doc.WeightMedium, text: "16"}, + } + got := doc.RegionBlocks(blockPage(11, lines...), wholePage(11), nil, nil) + + for i := range got { + if got[i].Kind == doc.BlockHeading { + t.Errorf("block %d is a heading reading %q — %s; a figure callout is a number "+ + "and a heading is words", i, got[i].Text, got[i].Note) + } + } +} + +// TestRegionBlocksReadColumnsOneAtATime is the correction to what the contract +// reads like it says. A whole-page region can hold several text columns — +// regions.md rule 3 deliberately stores a page of same-language columns as one +// region — so sorting inside the region interleaves them, which is the mistake +// pdftotext -layout makes and the contract names. +func TestRegionBlocksReadColumnsOneAtATime(t *testing.T) { + var lines []line + for i := 0; i < 8; i++ { + y := 20 + float64(i)*18 + lines = append(lines, + line{y: y, x: 40, w: 350, size: 14, text: "links Zeile " + itoa(i)}, + // A slightly different baseline, which is what the column manual does: + // its two columns drift apart by two units down the page. + line{y: y + 2, x: 470, w: 350, size: 14, text: "rechts Zeile " + itoa(i)}) + } + got := doc.RegionBlocks(blockPage(62, lines...), wholePage(62), nil, nil) + + if len(got) != 2 { + t.Fatalf("got %d blocks, want one paragraph per column: %v", len(got), kinds(got)) + } + if strings.Contains(got[0].Text, "rechts") { + t.Errorf("the first block mixes both columns: %q", got[0].Text) + } + if strings.Contains(got[1].Text, "links") { + t.Errorf("the second block mixes both columns: %q", got[1].Text) + } + // And in the right order: left column first. + if !strings.HasPrefix(got[0].Text, "links") { + t.Errorf("the first block is %q, want the left column", got[0].Text) + } +} + +// TestRegionBlocksReadOnlyInsideTheBox is the funnel, and the one failure a reader +// notices immediately. A boxed region is one language's column of a page that +// holds five, and a block built from a run outside the box is text in a language +// nobody asked for. +func TestRegionBlocksReadOnlyInsideTheBox(t *testing.T) { + var lines []line + for i := 0; i < 8; i++ { + y := 20 + float64(i)*18 + lines = append(lines, + line{y: y, x: 30, w: 250, size: 14, text: "deutsch Zeile " + itoa(i)}, + line{y: y, x: 320, w: 250, size: 14, text: "polnisch Zeile " + itoa(i)}, + line{y: y, x: 610, w: 250, size: 14, text: "russisch Zeile " + itoa(i)}) + } + got := doc.RegionBlocks(blockPage(2, lines...), + &doc.Region{Page: 2, X0: 30, X1: 280, Lang: "de"}, nil, nil) + + if len(got) == 0 { + t.Fatal("the German column produced no blocks") + } + for i := range got { + for _, other := range []string{"polnisch", "russisch"} { + if strings.Contains(got[i].Text, other) { + t.Errorf("block %d of the German region reads %q; it holds %s text", + i, got[i].Text, other) + } + } + if got[i].Lang != "de" { + t.Errorf("block %d carries language %q, want the region's de", i, got[i].Lang) + } + if got[i].RegionX0 != 30 { + t.Errorf("block %d records region x0 %.0f, want 30", i, got[i].RegionX0) + } + } +} + +// TestRegionBlocksSingleLineRegion is a real shape: a caption beside a diagram, +// or a page holding one line of text. +func TestRegionBlocksSingleLineRegion(t *testing.T) { + got := doc.RegionBlocks( + blockPage(12, line{y: 40, x: 55, w: 300, size: 17, text: "Abb. A-1"}), + wholePage(12), nil, nil) + + if len(got) != 1 { + t.Fatalf("got %d blocks for one line: %v", len(got), kinds(got)) + } + if got[0].Text != "Abb. A-1" { + t.Errorf("text = %q", got[0].Text) + } + if got[0].Lines != 1 || got[0].Chars != 8 { + t.Errorf("block reports %d lines and %d chars, want 1 and 8", got[0].Lines, got[0].Chars) + } +} + +func TestRegionBlocksEmptyRegion(t *testing.T) { + page := &doc.PageRuns{No: 3, Width: testBlockPageWidth, Height: testBlockPageHeight} + if got := doc.RegionBlocks(page, wholePage(3), nil, nil); len(got) != 0 { + t.Errorf("got %d blocks for a page with no runs", len(got)) + } + + // A region whose box holds nothing, on a page that does hold text elsewhere. + page = blockPage(4, bodyLines(20, 22, 6, "text weit rechts")...) + if got := doc.RegionBlocks(page, &doc.Region{Page: 4, X0: 800, X1: 890}, nil, nil); len(got) != 0 { + t.Errorf("got %d blocks for a box containing no runs", len(got)) + } +} + +// TestRegionBlocksIgnoreWhatIsNotText guards the shared filter. The column +// manual's text layer carries 522 sub-legible production slugs and parks 218 runs +// of a superseded address list above the top edge of one page; a block built from +// those puts text in the reader that is not on the paper — and that the gate never +// charged for, since Region.Chars comes through the same filter. +func TestRegionBlocksIgnoreWhatIsNotText(t *testing.T) { + lines := bodyLines(20, 22, 6, "echter Text auf der Seite") + page := blockPage(9, lines...) + clean := doc.RegionBlocks(page, wholePage(9), nil, nil) + + page.Runs = append(page.Runs, + // A production slug: real text in the file, two units tall, invisible on paper. + doc.TextRun{X: 55, Y: 300, Width: 250, Height: 2, + Text: "Job_4417_Manual_v3_export_2019-11-08.indd 1 08.11.19 10:16"}, + // A run parked above the page, which is where a superseded address list lives. + doc.TextRun{X: 55, Y: -38, Width: 250, Height: 22, Text: "Superseded address list line"}) + + got := doc.RegionBlocks(page, wholePage(9), nil, nil) + if len(got) != len(clean) { + t.Errorf("blocks went from %d to %d when a sub-legible slug and an off-page run "+ + "were added; neither is text on the page", len(clean), len(got)) + } + for i := range got { + if strings.Contains(got[i].Text, "indd") || strings.Contains(got[i].Text, "Superseded") { + t.Errorf("block %d reads %q", i, got[i].Text) + } + } +} + +// TestRegionBlocksJoinRunsAsThePageShowsThem covers the two halves of joinRuns: +// poppler splits a run at every font change, so touching runs are one word and +// separated ones are two. +func TestRegionBlocksJoinRunsAsThePageShowsThem(t *testing.T) { + p := &doc.PageRuns{No: 5, Width: testBlockPageWidth, Height: testBlockPageHeight} + f := doc.Font{Size: 17, Family: "Test-Face"} + // "Sollte" then a bold "THOMAS" then " einmal": three runs, a real gap between + // the first two and none between the last two. + p.Runs = append(p.Runs, + doc.TextRun{X: 55, Y: 40, Width: 60, Height: 22, Text: "Sollte", Font: f}, + doc.TextRun{X: 120, Y: 40, Width: 70, Height: 22, Text: "THOMAS", Font: f}, + doc.TextRun{X: 190, Y: 40, Width: 60, Height: 22, Text: "-Geraet", Font: f}) + + got := doc.RegionBlocks(p, wholePage(5), nil, nil) + if len(got) != 1 { + t.Fatalf("got %d blocks: %v", len(got), kinds(got)) + } + if want := "Sollte THOMAS-Geraet"; got[0].Text != want { + t.Errorf("text = %q, want %q", got[0].Text, want) + } +} + +// TestRegionBlocksAreNaturallyKeyed is the property conversion.md asks for: a +// second run over the same bytes must converge on the first rather than insert a +// parallel set, which requires the key to come out of the page and not out of a +// counter. +func TestRegionBlocksAreNaturallyKeyed(t *testing.T) { + page := blockPage(62, bodyLines(20, 22, 6, "eine Zeile Text")...) + region := &doc.Region{Page: 62, X0: 30, X1: 800, Lang: "de"} + + first := doc.RegionBlocks(page, region, nil, nil) + second := doc.RegionBlocks(page, region, nil, nil) + + if len(first) != len(second) { + t.Fatalf("two runs produced %d and %d blocks", len(first), len(second)) + } + for i := range first { + a, b := &first[i], &second[i] + if a.Page != b.Page || a.RegionX0 != b.RegionX0 || a.Index != b.Index || a.Text != b.Text { + t.Errorf("block %d differs between runs: %+v against %+v", i, *a, *b) + } + if a.Page != 62 || a.RegionX0 != 30 || a.Index != i { + t.Errorf("block %d is keyed (page %d, x0 %.0f, index %d), want (62, 30, %d)", + i, a.Page, a.RegionX0, a.Index, i) + } + } +} + +// TestRegionsBlocksReadOnlyWhatIsInScope is the funnel at document level. A +// household reading German must not be given the Polish column of the same page. +func TestRegionsBlocksReadOnlyWhatIsInScope(t *testing.T) { + var lines []line + for i := 0; i < 8; i++ { + y := 20 + float64(i)*18 + lines = append(lines, + line{y: y, x: 30, w: 250, size: 14, text: "deutsch Zeile " + itoa(i)}, + line{y: y, x: 320, w: 250, size: 14, text: "polnisch Zeile " + itoa(i)}) + } + pages := []doc.PageRuns{*blockPage(2, lines...)} + regions := []doc.Region{ + {Page: 2, X0: 30, X1: 280, Lang: "de"}, + {Page: 2, X0: 320, X1: 570, Lang: "pl"}, + } + + got := doc.RegionsBlocks(pages, regions, map[string]bool{"de": true}, nil, nil) + if len(got) == 0 { + t.Fatal("no blocks for the German region") + } + for i := range got { + if got[i].Lang != "de" || strings.Contains(got[i].Text, "polnisch") { + t.Errorf("block %d is %q in %q, but only German was in scope", + i, got[i].Text, got[i].Lang) + } + } + + // And with no scope, both regions are read, in left-edge order. + all := doc.RegionsBlocks(pages, regions, nil, nil, nil) + if len(all) <= len(got) { + t.Errorf("reading every region gave %d blocks and German alone %d", len(all), len(got)) + } + if all[0].RegionX0 != 30 { + t.Errorf("first block comes from the region at x0 %.0f, want the leftmost at 30", + all[0].RegionX0) + } +} + +// TestBlockSummaryDescribesTheShape keeps the log line honest, since it is what a +// fixture test reports instead of every row. +func TestBlockSummaryDescribesTheShape(t *testing.T) { + if got := doc.BlockSummary(nil); got != "no blocks" { + t.Errorf("BlockSummary(nil) = %q", got) + } + got := doc.BlockSummary([]doc.Block{ + {Page: 1, Kind: doc.BlockHeading, Chars: 10}, + {Page: 1, Kind: doc.BlockParagraph, Chars: 90}, + {Page: 2, Kind: doc.BlockListItem, Chars: 5}, + }) + for _, want := range []string{"3 blocks", "2 pages", "105 chars", "1 heading", "1 list-item"} { + if !strings.Contains(got, want) { + t.Errorf("summary %q does not mention %q", got, want) + } + } +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + var b []byte + for n > 0 { + b = append([]byte{byte('0' + n%10)}, b...) + n /= 10 + } + return string(b) +} + +// TestEachContentsEntryIsItsOwnBlock is the defect itself: seventeen entries sit at +// exactly the line pitch, so the paragraph rule has nothing to separate them by and +// glued the whole contents page into one block of run-together dot leaders. +func TestEachContentsEntryIsItsOwnBlock(t *testing.T) { + // Page 2 of the columns manual, the Russian column: left edge 604, 16-unit pitch, + // the first three entries at their measured tops. + page := &doc.PageRuns{No: 2, Width: 892, Height: 850, Runs: []doc.TextRun{ + {X: 604, Y: 62, Width: 259, Height: 17, Text: "Мы поздравляем Вас ..............................2"}, + {X: 604, Y: 78, Width: 259, Height: 17, Text: "Использование по назначению ....................4"}, + {X: 604, Y: 94, Width: 259, Height: 17, Text: "Указания по технике безопасности ..............8"}, + }} + blocks := doc.RegionBlocks(page, &doc.Region{Page: 2, X0: 604, X1: 866, Lang: "ru"}, nil, nil) + + var entries int + for i := range blocks { + if blocks[i].Kind == doc.BlockListItem && doc.IsContentsEntry(blocks[i].Note) { + entries++ + } + } + if entries != 3 { + t.Errorf("%d contents entries from 3 printed lines; %s", entries, doc.BlockSummary(blocks)) + for i := range blocks { + t.Logf(" %s %q", blocks[i].Kind, blocks[i].Text) + } + } +} diff --git a/internal/doc/blocksplace_test.go b/internal/doc/blocksplace_test.go new file mode 100644 index 0000000..b8aa267 --- /dev/null +++ b/internal/doc/blocksplace_test.go @@ -0,0 +1,74 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// TestATableWiderThanItsWordsStaysInItsStrip is the sequential manual's page 537: a +// spec table whose ruled box is drawn x=478-862 in a strip whose words reach only +// 845, because the box comes from the strokes and the strip's bounds come from the +// text. Under containment the table belonged to no strip and fell to the banner band, +// which reads first — so the reader was shown the whole spec table and only then the +// page's own title. +func TestATableWiderThanItsWordsStaysInItsStrip(t *testing.T) { + // Two strips of a page that the column detector will split, each with a heading + // and a table under it, and each table drawn 18 units wider than its own words. + left := gridTable(60, 200, 450, 120, 5) + right := gridTable(478, 620, 862, 120, 5) + + lines := []line{ + {y: 20, x: 63, w: 316, size: 21, weight: doc.WeightSemibold, bold: true, + text: "Technische Daten"}, + {y: 70, x: 72, w: 40, size: 15, weight: doc.WeightSemibold, bold: true, text: "Robot"}, + {y: 70, x: 487, w: 115, size: 15, weight: doc.WeightSemibold, bold: true, + text: "Basisstation"}, + } + for r := 0; r < 5; r++ { + y := 130 + float64(r)*40 + lines = append(lines, + line{y: y, x: 70, w: 100, size: 11, text: "Feld " + itoa(r)}, + line{y: y, x: 210, w: 120, size: 11, text: "Wert " + itoa(r)}, + line{y: y, x: 488, w: 100, size: 11, text: "Feldb " + itoa(r)}, + // Short of the drawn right edge, which is the whole point. + line{y: y, x: 630, w: 195, size: 11, text: "Wertb " + itoa(r)}) + } + + page := blockPage(537, lines...) + got := doc.RegionBlocks(page, wholePage(537), []doc.RuledTable{left, right}, nil) + if len(got) == 0 { + t.Fatal("no blocks") + } + if !strings.Contains(got[0].Text, "Technische Daten") { + t.Errorf("the page reads %q first, want its own title — a table whose drawn box "+ + "overhangs its words fell through to the banner band\n%s", + got[0].Text, strings.Join(texts(got), "\n")) + } + // And each table reads under its own heading rather than both under one. + robot := strings.Index(blockTexts(got), "Robot") + basis := strings.Index(blockTexts(got), "Basisstation") + feld0 := strings.Index(blockTexts(got), "Feld 0") + feldb0 := strings.Index(blockTexts(got), "Feldb 0") + if robot >= feld0 || feld0 >= basis || basis >= feldb0 { + t.Errorf("the two tables are not each under their own heading: %s", blockTexts(got)) + } +} + +// gridTable is a plain two-column ruled grid. +func gridTable(x0, mid, x1, y0 float64, rows int) doc.RuledTable { + tab := doc.RuledTable{ + Box: doc.CellRect{X0: x0, Y0: y0, X1: x1, Y1: y0 + float64(rows)*40}, + Rows: rows, Cols: 2, + } + for r := 0; r < rows; r++ { + top := y0 + float64(r)*40 + tab.Cells = append(tab.Cells, + doc.RuledCell{Row: r, Col: 0, ColSpan: 1, + Rect: doc.CellRect{X0: x0, Y0: top, X1: mid, Y1: top + 40}}, + doc.RuledCell{Row: r, Col: 1, ColSpan: 1, + Rect: doc.CellRect{X0: mid, Y0: top, X1: x1, Y1: top + 40}}) + } + return tab +} diff --git a/internal/doc/blocksstrips_fixture_test.go b/internal/doc/blocksstrips_fixture_test.go new file mode 100644 index 0000000..c5ff36c --- /dev/null +++ b/internal/doc/blocksstrips_fixture_test.go @@ -0,0 +1,136 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// The real pages behind blocksstrips_test.go, pinned by the strings they print. +// +// These are the eight two-column Russian maintenance pages of the sequential manual +// and the two parts pages of the column manual: every page of either document where +// a block used to cross the gutter. The reading below was checked against +// `pdftoppm -r 108`, which is this coordinate space 1:1. + +// TestTheRussianMaintenancePagesReadInColumns is the defect, named page by page. +// Before reading order got its own strips, page 530's two section banners were one +// block reading "Мешок для сбора пыли Основная щетка" and its two columns' first +// step was one sentence ending "…мешок для сбора 1. Надавите на". +func TestTheRussianMaintenancePagesReadInColumns(t *testing.T) { + _, pages, regions, _ := regionsOfFixture(t, "dreame-l40-ultra") + + // Page, then the pairs of strings that must not end up in one block: each pair is + // the left column's text and the right column's on the same printed baseline. + welds := map[int][][2]string{ + 525: {{"6. Добавление чистящего раствора", "7. Заполните бак"}}, + 530: {{"Мешок для сбора пыли", "Основная щетка"}, + {"утилизируйте мешок для сбора", "Надавите на"}}, + 531: {{"Боковая щетка", "Держатели насадок"}, {"Всенаправленное колесо", "Насадка для швабры"}}, + 532: {{"Контейнер для пыли и фильтр", "Датчики робота"}}, + 533: {{"Зарядные контакты", "Бак для отработанной воды"}}, + 537: {{"Робот", "Базовая станция"}}, + } + for page, pairs := range welds { + blocks := blocksOfPage(t, pages, regions, page, "ru") + for i := range blocks { + for _, pair := range pairs { + if strings.Contains(blocks[i].Text, pair[0]) && strings.Contains(blocks[i].Text, pair[1]) { + t.Errorf("page %d block %d welds the two columns: %q", + page, blocks[i].Index, truncate(blocks[i].Text, 120)) + } + } + } + if t.Failed() || page == 537 { + // 537 is left out of the order check and only of that: it is the + // specification page, whose reading order comes from its ruled tables, and + // these blocks are built without them. Its welds are still asserted above, + // because those are a property of the text and not of the strokes. + continue + } + // And the columns do not interleave: with the gutter at x=440-495 on all of + // these pages, nothing left of it may be read after anything right of it. + lastLeft, firstRight := -1, len(blocks) + for i := range blocks { + switch { + case blocks[i].X1 < 460: + lastLeft = i + case blocks[i].X0 > 460 && i < firstRight: + firstRight = i + } + } + if lastLeft > firstRight { + t.Errorf("page %d interleaves: block %d is in the left column and is read "+ + "after block %d in the right", page, lastLeft, firstRight) + } + } +} + +// TestTheRussianBannersAreTheirOwnBlocks is the positive form, with the strings the +// page prints. A section banner reaching only its own column is a heading, and there +// are two of them on page 530 rather than one of both. +func TestTheRussianBannersAreTheirOwnBlocks(t *testing.T) { + _, pages, regions, _ := regionsOfFixture(t, "dreame-l40-ultra") + blocks := blocksOfPage(t, pages, regions, 530, "ru") + + want := []string{ + "Мешок для сбора пыли", + "1. Снимите крышку отсека для пыли и утилизируйте мешок для сбора", + "3. Установите новый мешок для сбора пыли, затем установите крышку", + "Основная щетка", + "1. Надавите на зажимы защиты щетки, чтобы извлечь защиту щетки и", + } + var got []string + for i := range blocks { + got = append(got, blocks[i].Text) + } + at := -1 + for _, w := range want { + next := -1 + for i, g := range got { + if i > at && g == w { + next = i + break + } + } + if next < 0 { + t.Errorf("page 530 does not print %q as a block of its own, after the one before "+ + "it\n%s", w, strings.Join(got, "\n")) + return + } + at = next + } +} + +// TestTheColumnManualsPartsListIsItems is the same defect on the other document. Page +// 11's numbered parts list arrived as two run-together blocks of 7 and 19 printed +// lines, each with the diagram's callout numbers spliced into the words: +// "17 Staubbehälter für Grobschmutz und Feinstaub 7 18 Saugschlauch*…". +func TestTheColumnManualsPartsListIsItems(t *testing.T) { + _, pages, regions, _ := regionsOfFixture(t, "thomas-drybox-amfibia") + blocks := blocksOfPage(t, pages, regions, 11, "de") + + items := 0 + for i := range blocks { + b := &blocks[i] + if b.Kind == doc.BlockListItem && b.X0 > 500 { + items++ + } + if strings.Contains(b.Text, "Staubbehälter für Grobschmutz") && + strings.Contains(b.Text, "Saugschlauch") { + t.Errorf("block %d welds the parts list to the diagram's callouts: %q", + b.Index, truncate(b.Text, 140)) + } + } + // The page prints 39 numbered items in a column of its own, right of the diagram. + // + // 37 and not 39: two of the printed items wrap onto a second line and are folded + // into the item above by the paragraph rule, which is a different question from + // this one. Before the strips they were 6. + if items < 37 { + t.Errorf("the parts list came back as %d list items in its own column, want the 37 "+ + "measured — it was 6 while the column was welded to the diagram's callouts", + items) + } +} diff --git a/internal/doc/blocksstrips_test.go b/internal/doc/blocksstrips_test.go new file mode 100644 index 0000000..ba29de1 --- /dev/null +++ b/internal/doc/blocksstrips_test.go @@ -0,0 +1,221 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// The pages that have two columns and no second column. +// +// [doc.DetectColumns] answers "what are this page's text columns", and a Column is a +// published fact — language attribution reads it — so it insists on [minColumnRuns] +// runs before it names one. Reading order asks a narrower question and the same gate +// is wrong for it: a maintenance page with six runs in its right-hand column is still +// two columns to a reader, and read as one strip its two banners come back spliced. +// +// Every shape here is drawn from a page of one of the two manuals and is measured in +// blocksstrips_fixture_test.go against that page. + +// sparsePage is the sequential manual's page 530, to the shape that matters: two +// columns of a maintenance page, each a banner and a few lines, sharing baselines +// across a gutter at x=440-495. +// +// Six runs on the right and nine on the left, which is what defeats minColumnRuns=8, +// and 15 runs on the whole page, which is what defeats maxGutterCrossings=4 — at that +// density no x of the page is crossed by more than four runs, so the projection reads +// the right-hand half as one gutter. +func sparsePage(no int) *doc.PageRuns { return blockPage(no, sparseLines(0)...) } + +// sparseLines returns the page's lines, shifted down the page by dy. +func sparseLines(dy float64) []line { + out := []line{ + {y: 20, x: 65, w: 130, size: 15, weight: doc.WeightSemibold, bold: true, + text: "Mешок для сбора"}, + {y: 45, x: 65, w: 365, size: 14, text: "1. Снимите крышку отсека для пыли"}, + {y: 60, x: 65, w: 30, size: 14, text: "пыли."}, + {y: 200, x: 65, w: 355, size: 14, text: "Примечание. Потяните ручку вверх"}, + {y: 215, x: 66, w: 260, size: 14, text: "2. Очистите пыль и грязь с фильтра"}, + {y: 350, x: 65, w: 370, size: 14, text: "3. Установите новый мешок для сбора"}, + {y: 365, x: 65, w: 135, size: 14, text: "отсека для пыли на место."}, + {y: 385, x: 65, w: 300, size: 14, text: "4. Закройте крышку отсека для пыли."}, + {y: 400, x: 65, w: 180, size: 14, text: "и проверьте фиксацию."}, + + {y: 20, x: 504, w: 90, size: 15, weight: doc.WeightSemibold, bold: true, + text: "Основная щетка"}, + {y: 46, x: 497, w: 370, size: 14, text: "1. Надавите на зажимы защиты щетки"}, + {y: 61, x: 497, w: 130, size: 14, text: "достать щетку из робота."}, + {y: 300, x: 496, w: 348, size: 14, text: "2. Снимите крышки щетки с обоих"}, + {y: 315, x: 496, w: 340, size: 14, text: "рисунке. Для удаления запутавшихся"}, + } + for i := range out { + out[i].y += dy + } + return out +} + +func texts(blocks []doc.Block) []string { + out := make([]string, len(blocks)) + for i := range blocks { + out[i] = blocks[i].Text + } + return out +} + +// TestASparsePageIsNotOneColumn is the defect, stated as the reading it produced. Two +// section banners on one baseline either side of a gutter were one block reading +// "Mешок для сбора Основная щетка", and the two columns' step 1 was one sentence. +func TestASparsePageIsNotOneColumn(t *testing.T) { + page := sparsePage(530) + + // The premise, asserted rather than assumed: the column detector really does + // report one column here, so nothing downstream can lean on it. + if got := doc.DetectColumns(page.Runs, page.Width, page.Height); len(got.Columns) != 1 { + t.Fatalf("DetectColumns found %d columns on the sparse page, want 1 — this test "+ + "pins what happens when it finds one, and the page no longer does that: %s", + len(got.Columns), got.Note) + } + + got := doc.RegionBlocks(page, &doc.Region{Page: 530, X0: 0, X1: page.Width, Lang: "ru"}, nil, nil) + for i := range got { + if strings.Contains(got[i].Text, "сбора") && strings.Contains(got[i].Text, "Основная") { + t.Errorf("block %d welds the two columns' banners: %q", i, got[i].Text) + } + if strings.Contains(got[i].Text, "Снимите крышку") && strings.Contains(got[i].Text, "Надавите") { + t.Errorf("block %d welds the two columns' first step: %q", i, got[i].Text) + } + } + + // And the whole left column is read before any of the right one, which is the + // positive form of the same claim. + lastLeft, firstRight := -1, len(got) + for i := range got { + switch { + case got[i].X1 < 460 && i > lastLeft: + lastLeft = i + case got[i].X0 > 460 && i < firstRight: + firstRight = i + } + } + if lastLeft > firstRight { + t.Errorf("the columns interleave: left-hand block %d is read after right-hand "+ + "block %d\n%s", lastLeft, firstRight, strings.Join(texts(got), "\n")) + } +} + +// TestAStripNeedsAnEmptyCorridor is the other half of the bound, and it is what keeps +// the fallback from cutting a page wherever the text happens to be thin. The gutter of +// a sparse page is empty; the space inside a spec table's row is not, because the rows +// above and below reach across it. +func TestAStripNeedsAnEmptyCorridor(t *testing.T) { + // A specification table set as one column: a label at x=65 and a value at x=380, + // 216 units apart — the widest within-column gap either manual prints, from the + // sequential manual's "Model … RLL77SE". Nothing crosses that space either, until + // one row is set to the full measure, which is what a real spec page does. + var lines []line + for i := 0; i < 6; i++ { + y := 20 + float64(i)*20 + lines = append(lines, + line{y: y, x: 65, w: 120, size: 14, text: "Modell" + itoa(i)}, + line{y: y, x: 380, w: 90, size: 14, text: "RLL77SE" + itoa(i)}) + } + lines = append(lines, line{y: 160, x: 65, w: 405, size: 14, + text: "Bei normalem Gebrauch ist zwischen der Antenne und dem Koerper"}) + + page := blockPage(21, lines...) + got := doc.RegionBlocks(page, wholePage(21), nil, nil) + + for i := range got { + if strings.Contains(got[i].Text, "Modell0") && !strings.Contains(got[i].Text, "RLL77SE0") { + t.Errorf("block %d split a table row at its own column divider: %q", i, got[i].Text) + } + } +} + +// TestABannerAcrossTheMeasureIsNotSplit guards the shape the fallback must leave +// alone: a heading printed across both columns. It is one run, so no corridor lies +// inside it, and it also fills the corridor for every line beside it. +func TestABannerAcrossTheMeasureIsNotSplit(t *testing.T) { + lines := []line{{y: 20, x: 65, w: 760, size: 21, weight: doc.WeightSemibold, bold: true, + text: "Plановое обслуживание des ganzen Bogens"}} + page := blockPage(63, append(lines, sparseLines(60)...)...) + + got := doc.RegionBlocks(page, &doc.Region{Page: 63, X0: 0, X1: page.Width, Lang: "ru"}, nil, nil) + if len(got) == 0 { + t.Fatal("no blocks") + } + if got[0].Text != "Plановое обслуживание des ganzen Bogens" { + t.Errorf("the banner is %q, want it whole and read first\n%s", + got[0].Text, strings.Join(texts(got), "\n")) + } +} + +// TestDenseColumnsAreUnchanged is the column manual's page 62, the whole-page German +// region of two columns that conversion.md names. The detector answers it and the +// fallback must never run. +func TestDenseColumnsAreUnchanged(t *testing.T) { + var lines []line + for i := 0; i < 8; i++ { + y := 20 + float64(i)*18 + lines = append(lines, + line{y: y, x: 43, w: 400, size: 14, text: "links Zeile " + itoa(i)}, + line{y: y + 2, x: 463, w: 400, size: 14, text: "rechts Zeile " + itoa(i)}) + } + page := blockPage(62, lines...) + if got := doc.DetectColumns(page.Runs, page.Width, page.Height); len(got.Columns) != 2 { + t.Fatalf("DetectColumns found %d columns, want 2: %s", len(got.Columns), got.Note) + } + got := doc.RegionBlocks(page, wholePage(62), nil, nil) + if len(got) != 2 { + t.Fatalf("got %d blocks, want one paragraph per column:\n%s", + len(got), strings.Join(texts(got), "\n")) + } + if !strings.HasPrefix(got[0].Text, "links") || strings.Contains(got[0].Text, "rechts") { + t.Errorf("the first block is %q, want the left column whole", got[0].Text) + } +} + +// TestRightToLeftStripsAreReadRightToLeft is the sequential manual's page 216: an +// Arabic disposal page whose warning is in the right column and whose numbered +// removal guide is in the left. Read left first, the reader is handed step 1 before +// the paragraph that introduces it. +func TestRightToLeftStripsAreReadRightToLeft(t *testing.T) { + lines := []line{ + {y: 20, x: 634, w: 230, size: 17, weight: doc.WeightSemibold, bold: true, + text: "التخلص من البطارية"}, + {y: 60, x: 609, w: 255, size: 14, text: "تحتوي بطارية الليثيوم أيون المدمجة"}, + {y: 78, x: 664, w: 200, size: 14, text: "يجب إزالة البطارية من الجهاز"}, + {y: 96, x: 640, w: 224, size: 14, text: "يجب فصل الجهاز عن مصدر التيار"}, + {y: 114, x: 699, w: 165, size: 14, text: "يجب التخلص من البطارية بأمان"}, + + {y: 60, x: 430, w: 157, size: 15, weight: doc.WeightSemibold, bold: true, + text: "دليل الإزالة"}, + {y: 84, x: 340, w: 247, size: 14, text: "1. اقلب الروبوت واستخدم أداة مناسبة"}, + {y: 120, x: 351, w: 236, size: 14, text: "2. افصل الأطراف بين البطارية واللوحة"}, + } + page := blockPage(216, lines...) + got := doc.RegionBlocks(page, &doc.Region{Page: 216, X0: 0, X1: page.Width, Lang: "ar"}, nil, nil) + if len(got) < 2 { + t.Fatalf("got %d blocks", len(got)) + } + // Asserted on the geometry rather than the words: bidi.go stores a right-to-left + // line in logical order and this page's runs are built in it, so the text comes + // back reversed and comparing strings here would be testing bidi.go instead. + if got[0].X0 < 600 { + t.Errorf("the first block is at x=%.0f-%.0f, want the RIGHT column of an Arabic "+ + "page\n%s", got[0].X0, got[0].X1, strings.Join(texts(got), "\n")) + } + if last := &got[len(got)-1]; last.X0 > 600 { + t.Errorf("the last block is at x=%.0f-%.0f, want the LEFT column last", + last.X0, last.X1) + } + + // The same page in a left-to-right language reads the other way round, which is + // what says the direction comes from the region and not from the geometry. + ltr := doc.RegionBlocks(page, &doc.Region{Page: 216, X0: 0, X1: page.Width, Lang: "de"}, nil, nil) + if len(ltr) == 0 || ltr[0].X0 > 600 { + t.Errorf("a left-to-right region did not read its LEFT column first\n%s", + strings.Join(texts(ltr), "\n")) + } +} diff --git a/internal/doc/blockstable_test.go b/internal/doc/blockstable_test.go new file mode 100644 index 0000000..6c49ad9 --- /dev/null +++ b/internal/doc/blockstable_test.go @@ -0,0 +1,262 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for the table walk. Every shape here is the column manual's page +// 57 reduced to what makes the case, because that is the page +// docs/design/conversion.md measured this join against: two side-by-side +// troubleshooting tables of two cell columns each, a header row printed above a top +// border the document does not draw, and a running head across the top. +// +// The page builders and the region helper come from blocks_test.go, so a table page +// and an ordinary one are built the same way and can be compared. + +// trouble builds one two-column troubleshooting table: dividers at x0, mid and x1, +// a row per fault, and a full-width spanning cell at the top for the section label, +// which is what page 57 prints as "Allgemein (alle Funktionen)". +func trouble(x0, mid, x1, y0 float64, rows int) doc.RuledTable { + t := doc.RuledTable{ + Box: doc.CellRect{X0: x0, Y0: y0, X1: x1, Y1: y0 + float64(rows+1)*40}, + Rows: rows + 1, Cols: 2, + } + t.Cells = append(t.Cells, doc.RuledCell{Row: 0, Col: 0, ColSpan: 2, + Rect: doc.CellRect{X0: x0, Y0: y0, X1: x1, Y1: y0 + 40}}) + for r := 1; r <= rows; r++ { + top := y0 + float64(r)*40 + t.Cells = append(t.Cells, + doc.RuledCell{Row: r, Col: 0, ColSpan: 1, + Rect: doc.CellRect{X0: x0, Y0: top, X1: mid, Y1: top + 40}}, + doc.RuledCell{Row: r, Col: 1, ColSpan: 1, + Rect: doc.CellRect{X0: mid, Y0: top, X1: x1, Y1: top + 40}}) + } + return t +} + +// troubleRows is how many fault rows the built table has. Eight, because that is +// [doc] minColumnRuns: below it a cell column is not enough runs to be read as a text +// column at all, and the whole point of the first test is that it IS read as one when +// the ruled lines are not available. Page 57's tables have seven rows. +const troubleRows = 8 + +// troublePage is page 57's shape: a running head, a header row above the table's +// undrawn top border, the spanning label cell, and a question and an answer per row. +func troublePage(no int) (*doc.PageRuns, doc.RuledTable) { + table := trouble(30, 170, 430, 100, troubleRows) + lines := []line{ + {y: 16, x: 40, w: 140, size: 17, weight: doc.WeightBold, text: "Fehlerbehebung"}, + {y: 70, x: 32, w: 120, size: 11, text: "Aufgetretene Störungen"}, + {y: 70, x: 175, w: 100, size: 11, text: "Grund / Abhilfe"}, + {y: 108, x: 34, w: 200, size: 11, text: "Allgemein (alle Funktionen)"}, + } + for r := 1; r <= troubleRows; r++ { + y := float64(108 + r*40) + lines = append(lines, + line{y: y, x: 34, w: 120, size: 11, text: "Fehler " + itoa(r)}, + line{y: y, x: 174, w: 240, size: 11, text: "Abhilfe " + itoa(r)}) + } + return blockPage(no, lines...), table +} + +// TestRegionBlocksReadATableAcrossItsRows is the fix for the limitation blocks.go +// used to record. Without the ruled lines the two cell columns are two text columns, +// so both questions read before both answers; with them each row is read across, and +// a question is beside its own remedy. +func TestRegionBlocksReadATableAcrossItsRows(t *testing.T) { + page, table := troublePage(57) + + down := blockTexts(doc.RegionBlocks(page, wholePage(57), nil, nil)) + if strings.Index(down, "Fehler "+itoa(troubleRows)) > strings.Index(down, "Abhilfe 1") { + t.Fatalf("without ruled lines this page is meant to read down every question and "+ + "then down every answer, which is the limitation being fixed; it read: %s", down) + } + + got := doc.RegionBlocks(page, wholePage(57), []doc.RuledTable{table}, nil) + var cells []string + for i := range got { + if got[i].Kind != doc.BlockTable { + continue + } + cells = append(cells, got[i].Text) + if !strings.Contains(got[i].Note, "row ") { + t.Errorf("table block %d does not say where in the grid it sits: %q", + got[i].Index, got[i].Note) + } + } + want := []string{"Allgemein (alle Funktionen)"} + for r := 1; r <= troubleRows; r++ { + want = append(want, "Fehler "+itoa(r), "Abhilfe "+itoa(r)) + } + if strings.Join(cells, "|") != strings.Join(want, "|") { + t.Errorf("table cells read\n %q\nwant row-major\n %q", cells, want) + } +} + +// TestRegionBlocksKeepAHeadingPrintedAcrossATableExactlyOnce is the constraint +// conversion.md states twice, because both ways of getting it wrong are tempting: a +// heading printed across a table must not be placed in a cell, and banner blocks +// must not be suppressed wherever a table covers the region. Either makes it vanish. +// +// The running head and the header row are both such text: they belong to no cell, so +// they read with the prose, before the table, once each. +func TestRegionBlocksKeepAHeadingPrintedAcrossATableExactlyOnce(t *testing.T) { + page, table := troublePage(57) + got := doc.RegionBlocks(page, wholePage(57), []doc.RuledTable{table}, nil) + + seen, firstTable := 0, len(got) + for i := range got { + if strings.Contains(got[i].Text, "Fehlerbehebung") { + seen++ + if got[i].Kind == doc.BlockTable { + t.Errorf("the heading printed across the table came back as a table cell: %q", + got[i].Text) + } + } + if got[i].Kind == doc.BlockTable && i < firstTable { + firstTable = i + } + } + if seen != 1 { + t.Errorf("the heading appears %d times, want exactly once: %s", seen, blockTexts(got)) + } + if firstTable == 0 { + t.Errorf("the table reads before the heading printed above it: %s", blockTexts(got)) + } + // The header row above the table's undrawn top border is not lost either. + if !strings.Contains(blockTexts(got), "Aufgetretene Störungen") { + t.Errorf("the header row above the table's top border was lost: %s", blockTexts(got)) + } +} + +// TestRegionBlocksClipATableToTheRegion is the constraint that a table may span +// regions in DIFFERENT languages, and it is page 57 exactly as it stood before the +// cell dividers were recognised: a Finnish region on the question cells and a German +// one on the answers, with one table spanning both. +// +// Assuming one table sits inside one region would pull the Finnish column into a +// German conversion. The join is geometric for the reason conversion.md gives — a +// block keys on its region's left edge and a table area has none to key on — so what +// clips the table is that a cell is only ever offered the region's own runs. +func TestRegionBlocksClipATableToTheRegion(t *testing.T) { + page, table := troublePage(57) + tables := []doc.RuledTable{table} + + left := doc.RegionBlocks(page, &doc.Region{Page: 57, X0: 30, X1: 160, Lang: "fi"}, tables, nil) + right := doc.RegionBlocks(page, &doc.Region{Page: 57, X0: 170, X1: 430, Lang: "de"}, tables, nil) + if len(left) == 0 || len(right) == 0 { + t.Fatalf("got %d blocks left and %d right; both halves of the table must be read", + len(left), len(right)) + } + for _, side := range []struct { + name string + blocks []doc.Block + x0, x1 float64 + wants, not string + }{ + {"the Finnish region", left, 30, 160, "Fehler 1", "Abhilfe 1"}, + {"the German region", right, 170, 430, "Abhilfe 1", "Fehler 1"}, + } { + for i := range side.blocks { + b := &side.blocks[i] + if b.X0 < side.x0-1 || b.X1 > side.x1+1 { + t.Errorf("%s: block %d spans x=%.1f-%.1f, outside x=%.0f-%.0f: %q", + side.name, b.Index, b.X0, b.X1, side.x0, side.x1, b.Text) + } + } + texts := blockTexts(side.blocks) + if !strings.Contains(texts, side.wants) { + t.Errorf("%s does not hold %q: %s", side.name, side.wants, texts) + } + if strings.Contains(texts, side.not) { + t.Errorf("%s holds %q, which is the cell column beside it: %s", + side.name, side.not, texts) + } + } +} + +// TestRegionBlocksWithoutRuledLinesAreUnchanged is the compatibility property that +// matters most here: pdftocairo is optional, so no tables must produce exactly what +// the release before this one produced, on a page that draws a table and on one that +// does not. +func TestRegionBlocksWithoutRuledLinesAreUnchanged(t *testing.T) { + tabled, _ := troublePage(57) + plain := blockPage(62, + line{y: 40, x: 40, w: 120, size: 17, weight: doc.WeightBold, text: "Technische Daten"}, + line{y: 80, x: 40, w: 400, size: 11, text: "Spannungsversorgung: 230 V, 50 Hz"}, + line{y: 96, x: 40, w: 400, size: 11, text: "Leistungsaufnahme: siehe Typenschild"}, + ) + for _, p := range []*doc.PageRuns{tabled, plain} { + none := doc.RegionBlocks(p, wholePage(p.No), nil, nil) + empty := doc.RegionBlocks(p, wholePage(p.No), []doc.RuledTable{}, nil) + if blockTexts(none) != blockTexts(empty) { + t.Errorf("page %d reads differently for nil tables and no tables:\n %s\n %s", + p.No, blockTexts(none), blockTexts(empty)) + } + for i := range none { + if none[i].Kind == doc.BlockTable { + t.Errorf("page %d produced a table block with no ruled lines read", p.No) + } + } + } +} + +// TestRegionBlocksReadNoTextTwice is what keeps the reader inside what the gate +// charged for. Both walks draw from one filtered run set, so a cell and a block can +// never show the same text and neither can show text the other could not see — the +// region's character count comes through that same filter. +func TestRegionBlocksReadNoTextTwice(t *testing.T) { + page, table := troublePage(57) + // One of the production slugs usableRuns exists to drop, placed inside a cell. + // This manual's illustrations are placed PDFs that each brought an InDesign + // filename along, scaled down with the artwork, and 522 of them are in its text + // layer. A table walk with a filter of its own would put this in the reader. + page.Runs = append(page.Runs, doc.TextRun{ + X: 200, Y: 150, Width: 60, Height: 3, Text: "Amfibia_57.indd", + Font: doc.Font{Size: 3, Family: "Test-Face"}, + }) + + got := doc.RegionBlocks(page, wholePage(57), []doc.RuledTable{table}, nil) + if strings.Contains(blockTexts(got), "Amfibia") { + t.Errorf("a sub-legible production slug reached a table cell, so the table walk "+ + "is not reading through the region's own filter: %s", blockTexts(got)) + } + page.Runs = page.Runs[:len(page.Runs)-1] + + // The page's own words against the blocks' words, as multisets. Nothing may be + // counted twice and nothing may be missing, which is both halves of the property + // in one comparison. + onPage := map[string]int{} + for i := range page.Runs { + for _, word := range strings.Fields(page.Runs[i].Text) { + onPage[word]++ + } + } + inBlocks := map[string]int{} + for i := range got { + for _, word := range strings.Fields(got[i].Text) { + inBlocks[word]++ + } + } + for word, n := range onPage { + if inBlocks[word] != n { + t.Errorf("%q is printed %d times and read %d times", word, n, inBlocks[word]) + } + } + for word, n := range inBlocks { + if onPage[word] != n { + t.Errorf("%q is read %d times and printed %d times", word, n, onPage[word]) + } + } +} + +func blockTexts(blocks []doc.Block) string { + out := make([]string, 0, len(blocks)) + for i := range blocks { + out = append(out, string(blocks[i].Kind)+":"+blocks[i].Text) + } + return strings.Join(out, " | ") +} diff --git a/internal/doc/callouts.go b/internal/doc/callouts.go new file mode 100644 index 0000000..3df6ecf --- /dev/null +++ b/internal/doc/callouts.go @@ -0,0 +1,242 @@ +package doc + +import ( + "math" + "sort" +) + +// Callouts is which runs of a document are a figure's callout labels, and it exists +// so that a label the picture already prints is not ALSO printed in the middle of the +// prose. +// +// # Why this is needed at all +// +// The crop is the band the page laid a drawing and its labels out in — see +// [labelBand] — so the picture contains its own callouts. The runs those labels came +// from are ordinary region text, so without this they are also grouped into blocks, +// and page 521 of the sequential manual really does read `Датчики перепада высоты` as +// a floating paragraph in the middle of the section. Every label would arrive twice, +// once as pixels and once as prose. +// +// This is why the pass survived the reversal that removed the reader's label layer. +// It was written when the labels were drawn beside the crop, and the reason it is +// needed did not change when they stopped being: what removes a run from the flow is +// that something else shows it, not which thing. +// +// # Why at run level, which was measured and not assumed +// +// The obvious cheap version is to drop the finished BLOCK whose text equals a label. +// It does not work, and the measurement is not marginal. Over the sequential manual's +// Russian scope, of the 89 labels carried on converted pages: +// +// 23 are a block of their own -- a block filter would be exact +// 66 are merged into a bigger block -- a block filter is wrong either way +// 0 are in no block at all +// +// Page 522 is where it is worst: `Бак для отработанной воды`, `Ручка фильтра` and +// fourteen more arrive inside larger paragraphs, because a diagram's label column sets +// its lines at the body pitch and the paragraph rule has nothing to separate them by. +// Dropping those blocks would delete the neighbouring content with them; keeping them +// leaves 66 duplicates. So the run leaves before anything is grouped, which is exactly +// the reasoning [splitFurniture] records for the language tab — and for the same +// underlying reason, that the thing to remove is not always a block. +// +// # This is NOT furniture, and the difference is coverage +// +// Both types answer "does this run join the reading flow", at the same seam, and it +// would be easy to fold this into [Furniture]. It must not be, because +// `verify.checkCoverage` deliberately does NOT count furniture: furniture is +// DISCARDED, so counting it would hide a rule that wrongly claimed a paragraph. +// +// A callout label is not discarded. It is RELOCATED — out of the flow and onto the +// figure, where a reader still sees it. So its characters must stay in the coverage +// numerator, or coverage would fall by exactly the labels and there would be no way +// left to tell relocation from loss. [Block.Callout] is therefore counted by coverage +// and [Block.Furniture] is not, which is why they are two flags and not one. +type Callouts struct { + // at is page -> the label runs on it, by position and text. Matched on those rather + // than on an index into the page's runs, for the reason [Furniture.notes] gives: the + // runs [RegionBlocks] asks about are copies twice removed, because usableRuns and + // runsInBox each return a new slice. + at map[int][]calloutAt + + // total is how many label runs were claimed over the document, counted once per + // distinct run. + total int +} + +// calloutAt is one claimed run's position and text. +// +// A LIST SCANNED WITH A TOLERANCE, NOT A ROUNDED MAP KEY. The reason is now history +// and is kept because the history is the argument for not changing it back. [Mark] +// used to RECOVER a run's coordinate by multiplying the label's fraction of the crop +// back out, which lands within one ULP of the original: 55 came back as +// 54.999999999999986. Rounding does not absorb that. It AMPLIFIES it whenever the true +// value sits on a .5 boundary, which at a tolerance of 2 is every even coordinate — +// 55/2 is exactly 27.5 and rounds to 28, while 54.999999999999986/2 is 27.4999… and +// rounds to 27. Two buckets, one run, and the label silently stayed in the prose. +// +// Nothing is reconstructed now: the position on [Figure.LabelBoxes] is the run's own, +// so exact equality would work. The scan is kept because it costs nothing to defend — +// the nearest two runs on any page of either manual are further apart than 2 units, so +// a real neighbour cannot reach it — and because it is the shape that survives the +// next caller who does compute a position rather than carry one. +type calloutAt struct { + x, y float64 + text string +} + +// calloutTolerance is how close a run must be to a claimed one to be it, in the +// 1.5-scaled space. +// +// It is absorbing float arithmetic and nothing else — the run being looked up and the +// run that was claimed are the same measurement, not two readings of one page — so this +// could be far tighter than furniture's line tolerance. It is kept at the same value +// because there is no case in either document within 2 units of a wrong answer: the +// nearest two runs on any page of either manual are further apart than that, and a +// tolerance that cannot be reached by a real neighbour is one fewer number to defend. +const calloutTolerance = furnitureYTolerance + +// Total is how many label runs were claimed over the document. A nil receiver answers +// 0, which is what makes it safe to report on a conversion that found no figures — the +// same contract, and the same reason, as [Furniture.Total]. +func (c *Callouts) Total() int { + if c == nil { + return 0 + } + return c.total +} + +// IsLabel reports whether a run on a page is a figure's callout label. +// +// A nil receiver answers no for everything, which is what makes [RegionBlocks] work +// unchanged for the many callers that have one page and no figures — the same +// contract [Furniture.Note] has and for the same reason. +func (c *Callouts) IsLabel(page int, r *TextRun) bool { + if c == nil { + return false + } + text := furnitureText(r.Text) + for _, a := range c.at[page] { + if a.text == text && + math.Abs(a.x-r.X) <= calloutTolerance && + math.Abs(a.y-r.Y) <= calloutTolerance { + return true + } + } + return false +} + +// Mark records one figure's labels as being out of the flow. +// +// The position is the claimed run's own, carried on [Figure.LabelBoxes] and never +// serialised. It used to be RECONSTRUCTED, by multiplying the label's fraction of the +// crop back out, and that reconstruction is what [calloutAt] documents a bug about: it +// landed within one float ULP of the original, 55 coming back as 54.999999999999986, +// which a rounded bucket amplified into two different buckets and a label that +// silently stayed in the prose. Nothing is reconstructed now, so that hazard is gone +// at its source rather than absorbed. +func (c *Callouts) Mark(page int, f *Figure) { + if c == nil || len(f.Labels) == 0 || len(f.LabelBoxes) != len(f.Labels) { + return + } + if c.at == nil { + c.at = make(map[int][]calloutAt, 8) + } + for i, box := range f.LabelBoxes { + a := calloutAt{x: box.X0, y: box.Y0, text: furnitureText(f.Labels[i])} + // Two figures on one page can claim the same run — a label printed between two + // drawings is in both corridors — and it is one printed run either way. + if c.claimed(page, a) { + continue + } + c.at[page] = append(c.at[page], a) + c.total++ + } +} + +// claimed reports whether this exact run has already been recorded on the page. +func (c *Callouts) claimed(page int, a calloutAt) bool { + for _, seen := range c.at[page] { + if seen.text == a.text && + math.Abs(seen.x-a.x) <= calloutTolerance && + math.Abs(seen.y-a.y) <= calloutTolerance { + return true + } + } + return false +} + +// splitCallouts divides a region's runs into those that stay in the flow and those a +// figure's labels claimed. +// +// Both slices keep the order they arrived in, and this runs before a pitch, a body +// face or a line is measured, for the reason in this file's header. +func splitCallouts(runs []TextRun, page int, c *Callouts) (flow, labels []TextRun) { + if c == nil { + return runs, nil + } + if _, onPage := c.at[page]; !onPage { + return runs, nil + } + flow = make([]TextRun, 0, len(runs)) + for i := range runs { + if c.IsLabel(page, &runs[i]) { + labels = append(labels, runs[i]) + continue + } + flow = append(flow, runs[i]) + } + return flow, labels +} + +// calloutBlocks turns a region's callout runs into blocks. +// +// One block per printed line, which is what a label is: [figureLabels] claims runs, +// and a wrapped label's second line is its own claim. Nothing here rejoins them, +// because the figure already carries them as separate labels and a block that glued +// two of them would not match what the reader draws. +// +// Every block is a [BlockParagraph] whatever it is set in, on [furnitureBlocks]'s +// reasoning: a kind is a reading decision, and these blocks are not read — they exist +// so that `verify.checkCoverage` can still account for the characters. Right-to-left +// text is repaired here for the same reason it is repaired there, and it matters more +// than it looks: the sequential manual's Hebrew and Arabic sections carry labels, and +// storing them backwards would be writing them down wrong even where nothing reads +// them. +func calloutBlocks(runs []TextRun, r *Region, from int) []Block { + if len(runs) == 0 { + return nil + } + ordered := make([]TextRun, len(runs)) + copy(ordered, runs) + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].Y != ordered[j].Y { + return ordered[i].Y < ordered[j].Y + } + return ordered[i].X < ordered[j].X + }) + + rtlRegion := IsRightToLeftLanguage(r.Lang) + out := make([]Block, 0, len(ordered)) + for i := range ordered { + run := &ordered[i] + text := run.Text + if lineIsRightToLeft(ordered[i:i+1], rtlRegion) { + text = joinRunsRightToLeft(ordered[i : i+1]) + } + clean := collapseSpaces(text) + if clean == "" { + continue + } + out = append(out, Block{ + Page: r.Page, RegionX0: r.X0, Index: from + len(out), + Kind: BlockParagraph, Lang: r.Lang, Callout: true, + Note: "a figure's callout label, drawn beside the picture", + Text: clean, Lines: 1, + X0: run.X, X1: run.right(), Y0: run.Y, Y1: run.bottom(), + Chars: len([]rune(clean)), + }) + } + return out +} diff --git a/internal/doc/callouts_test.go b/internal/doc/callouts_test.go new file mode 100644 index 0000000..c05a5eb --- /dev/null +++ b/internal/doc/callouts_test.go @@ -0,0 +1,223 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for taking a figure's callout labels out of the block flow. No PDF +// and no poppler: the runs are built here, so the rule can be read against the +// reasoning in callouts.go, and the fixture tests hold the same rule against the two +// real manuals. +// +// The arrangement below is page 522 of the sequential manual reduced to its essentials +// — a label column set at the body pitch, immediately under a paragraph — because that +// is the shape a block-level filter cannot handle: measured over that document, 66 of +// 89 labels arrive INSIDE a bigger block, so dropping the block that contains a label +// would delete the paragraph with it. + +// calloutsFor marks the given runs of a page as one figure's labels, by building the +// figure the way findFigures would and handing it to Mark. +// +// The positions go in as the runs' own page coordinates, which is what +// [doc.Figure.LabelBoxes] carries and what Mark reads. +func calloutsFor(t *testing.T, page int, crop doc.CellRect, runs ...doc.TextRun) *doc.Callouts { + t.Helper() + fig := doc.Figure{Page: page, Rect: crop} + for i := range runs { + r := &runs[i] + fig.Labels = append(fig.Labels, r.Text) + fig.LabelBoxes = append(fig.LabelBoxes, doc.CellRect{ + X0: r.X, Y0: r.Y, X1: r.X + r.Width, Y1: r.Y + r.Height, + }) + } + c := &doc.Callouts{} + c.Mark(page, &fig) + return c +} + +// TestACalloutLabelLeavesTheBlockFlow is the whole of decision two: a label the reader +// draws beside the picture must not also be printed in the prose. +// +// The two labels here are set at the body pitch immediately below a paragraph, so the +// paragraph rule joins them to it — which is exactly why this has to happen at RUN +// level. The assertion is therefore in two halves: the labels are gone from the +// content, AND the paragraph that was glued to them is still whole. +func TestACalloutLabelLeavesTheBlockFlow(t *testing.T) { + const ( + x = 55.0 + pitch = 22.5 + ) + // A paragraph, then two label lines at the same pitch and the same left edge. + lines := []line{ + {y: 95, x: x, w: 500, size: 11, text: "Die Verpackung schuetzt das Geraet vor Transportschaeden."}, + {y: 95 + pitch, x: x, w: 480, size: 11, text: "Sie besteht aus wiederverwertbaren Materialien."}, + {y: 95 + 2*pitch, x: x, w: 90, size: 11, text: "Wassertank"}, + {y: 95 + 3*pitch, x: x, w: 70, size: 11, text: "Filtergriff"}, + } + page := blockPage(522, lines...) + region := wholePage(522) + + // Without the callouts, all four lines are content and the labels are glued in. + before := doc.RegionBlocks(page, region, nil, nil) + beforeText := blockText(before) + if !strings.Contains(beforeText, "Wassertank") || !strings.Contains(beforeText, "Filtergriff") { + t.Fatalf("the labels are not in the flow to begin with, so this test proves "+ + "nothing: %q", beforeText) + } + // The premise of the whole design: the label is not a block of its own. + glued := false + for i := range before { + if strings.Contains(before[i].Text, "Wassertank") && len(before[i].Text) > len("Wassertank")+4 { + glued = true + } + } + if !glued { + t.Fatalf("the label arrived as a block of its own, so this arrangement does not "+ + "exercise the case a block-level filter fails on: %q", beforeText) + } + + // With them, the labels are out of the content and the paragraph survives. + callouts := calloutsFor(t, 522, doc.CellRect{X0: 300, Y0: 100, X1: 500, Y1: 300}, + doc.TextRun{X: x, Y: 95 + 2*pitch, Width: 90, Height: 16, Text: "Wassertank"}, + doc.TextRun{X: x, Y: 95 + 3*pitch, Width: 70, Height: 16, Text: "Filtergriff"}, + ) + if callouts.Total() != 2 { + t.Fatalf("marked %d labels, expected 2", callouts.Total()) + } + after := doc.RegionBlocks(page, region, nil, nil, doc.WithCallouts(callouts)) + + var content, callout []doc.Block + for i := range after { + if after[i].Callout { + callout = append(callout, after[i]) + continue + } + content = append(content, after[i]) + } + + contentText := blockText(content) + for _, label := range []string{"Wassertank", "Filtergriff"} { + if strings.Contains(contentText, label) { + t.Errorf("the label %q is still in the prose; the reader draws it beside the "+ + "picture, so it would print twice", label) + } + } + // The paragraph the labels were glued to is untouched. This is the half a + // block-level filter gets wrong. + if !strings.Contains(contentText, "Transportschaeden") || + !strings.Contains(contentText, "wiederverwertbaren Materialien") { + t.Errorf("the paragraph the labels were glued to lost text: %q", contentText) + } + + // The labels are still THERE, marked, one block per printed line — which is what + // lets verify.checkCoverage account for their characters. + if len(callout) != 2 { + t.Fatalf("got %d callout blocks, expected one per printed label: %+v", + len(callout), callout) + } + if callout[0].Text != "Wassertank" || callout[1].Text != "Filtergriff" { + t.Errorf("callout blocks = %q and %q, expected the two labels", + callout[0].Text, callout[1].Text) + } + for i := range callout { + if callout[i].Furniture { + t.Errorf("callout block %d is also marked furniture; coverage skips furniture "+ + "and must NOT skip a label, because a label is relocated and not "+ + "discarded", i) + } + if callout[i].Note == "" { + t.Errorf("callout block %d carries no note saying why it left the flow", i) + } + } + // Content indices stay contiguous from 0, so "paragraph 2 of this region" still + // means the second thing a reader sees. + for i := range content { + if content[i].Index != i { + t.Errorf("content block %d has index %d; the callouts must come after the "+ + "content, not be interleaved with it", i, content[i].Index) + } + } +} + +// TestNoCalloutsIsTodaysReadingExactly is the property every optional pass here has to +// have: a caller that passes nothing gets the reading that shipped before this existed. +// A page cannot answer the question on its own — a label is claimed from a drawing's +// geometry — so every caller holding one page and no figures relies on this. +func TestNoCalloutsIsTodaysReadingExactly(t *testing.T) { + page := blockPage(522, bodyLines(95, 22.5, 6, "Ein Satz ueber die Verpackung.")...) + region := wholePage(522) + + plain := doc.RegionBlocks(page, region, nil, nil) + withNil := doc.RegionBlocks(page, region, nil, nil, doc.WithCallouts(nil)) + empty := doc.RegionBlocks(page, region, nil, nil, doc.WithCallouts(&doc.Callouts{})) + + if blockText(plain) != blockText(withNil) || blockText(plain) != blockText(empty) { + t.Errorf("passing no callouts changed the reading:\n plain %q\n nil %q\n empty %q", + blockText(plain), blockText(withNil), blockText(empty)) + } + if len(plain) != len(withNil) || len(plain) != len(empty) { + t.Errorf("block counts %d, %d, %d; the three must agree", + len(plain), len(withNil), len(empty)) + } + for i := range plain { + if plain[i].Callout { + t.Errorf("block %d is marked a callout with no callouts passed", i) + } + } +} + +// TestACalloutKeyIsPositionalNotTextual pins the key. A diagram's callouts are +// numerous, short and frequently identical — page 11 of the columns manual prints the +// digits 1 to 39 as separate runs — so a key on the text alone would mark every run +// reading "1" once one of them was claimed. furnitureKey can afford to omit x because +// furniture is one run near the top of a page; this cannot. +func TestACalloutKeyIsPositionalNotTextual(t *testing.T) { + const y = 140.0 + // Two runs reading "1" on the same baseline: one is a figure's callout, the other + // is a list marker in the text column. + claimed := doc.TextRun{X: 400, Y: y, Width: 6, Height: 16, Text: "1"} + other := doc.TextRun{X: 55, Y: y, Width: 6, Height: 16, Text: "1"} + + callouts := calloutsFor(t, 522, doc.CellRect{X0: 200, Y0: 100, X1: 380, Y1: 300}, claimed) + if !callouts.IsLabel(522, &claimed) { + t.Error("the claimed run is not recognised as a label") + } + if callouts.IsLabel(522, &other) { + t.Error("a different run with the same text on the same baseline was marked a " + + "label; the key must carry the column as well as the line") + } + // And nothing on another page is marked. + if callouts.IsLabel(523, &claimed) { + t.Error("the run was marked a label on a different page") + } +} + +// TestANilCalloutsAnswersNoForEverything covers the contract that keeps RegionBlocks +// working for every caller that has one page and not a document, which is most of this +// package's tests. +func TestANilCalloutsAnswersNoForEverything(t *testing.T) { + var c *doc.Callouts + r := doc.TextRun{X: 55, Y: 95, Width: 40, Height: 16, Text: "Wassertank"} + if c.IsLabel(522, &r) { + t.Error("a nil Callouts claimed a run") + } + if c.Total() != 0 { + t.Errorf("a nil Callouts reports %d claims", c.Total()) + } + // Mark on a nil receiver is a no-op rather than a panic, so a caller that found no + // figures does not have to branch. + c.Mark(522, &doc.Figure{Page: 522}) +} + +// blockText joins a slice of blocks, for the assertions that are about presence rather +// than about grouping. +func blockText(blocks []doc.Block) string { + parts := make([]string, 0, len(blocks)) + for i := range blocks { + parts = append(parts, blocks[i].Text) + } + return strings.Join(parts, " | ") +} diff --git a/internal/doc/clip.go b/internal/doc/clip.go new file mode 100644 index 0000000..3bf99c2 --- /dev/null +++ b/internal/doc/clip.go @@ -0,0 +1,346 @@ +package doc + +import ( + "encoding/xml" + "errors" + "io" + "math" +) + +// A clip is what makes a drawn shape's box the shape a reader sees. +// +// rules.go's walker reads a path's geometric extent, and that is not what the +// page paints: cairo writes `clip-path` on the group holding the artwork, and a +// drawing whose strokes run past its frame is cut back to the frame before +// anything reaches the paper. Ignoring it was measured and it is the largest +// visible defect in the conversion — 22 of the columns manual's 46 figures and 74 +// of the sequential manual's 163 arrived cut off by their own crop, neighbouring +// drawings merged into one figure, and a figure reached over the text beside it. +// This file is what [inkWalker] and [ruleWalker] consult so that a shape's box is +// its *visible* extent. +// +// Four properties of the SVG shape this code, and each is a decision rather than +// a detail. +// +// **A clip is a reference, and clips nest.** `clip-path="url(#clip-9)"` names a +// element elsewhere in the file, and an element is clipped by its own +// clip *and* by every clip on an ancestor. The effective clip is therefore the +// intersection, which is what [clipBox.intersect] accumulates as the walk +// descends. Cairo nests exactly two deep on both fixtures — a coarse integer +// window outside a tight one — and both are read. +// +// **The clip's bounding box is used, not the clip.** A may hold any +// shape, and clip-12 of the columns manual's page 16 is a Bézier ellipse. The box +// is the honest simplification: intersecting with it can only ever make a +// figure's box SMALLER than the unclipped extent and never wrongly larger, so the +// worst it can do is leave some of the old over-reach in place. It cannot cut +// away something the page paints. What it does not do is find the empty corners +// of a non-rectangular clip — a figure clipped to a circle keeps its bounding +// square, which is what a reader would crop by hand anyway. +// +// **A curve's control points are inside the box on purpose.** [subpaths] +// flattens a curve to its endpoints, which is right for a rule and wrong here: a +// clip's box built from endpoints alone can be smaller than the region the clip +// admits, and a clip that is too small cuts a real drawing. A Bézier lies inside +// the hull of its control points, so including them can only overstate the clip, +// which is the direction that cannot lose ink. [pathExtent] is that reading, and +// it is why this file does not simply call [subpaths]. +// +// **A clip that cannot be read is no clip at all.** An unresolvable reference, a +// with `clipPathUnits="objectBoundingBox"` — which needs a bounding box +// this walker does not have — or one holding no geometry leaves the shape +// unclipped. That is the old behaviour, which is wrong in a known and recorded +// direction, rather than a guess that could erase a picture. +// +// The compositing-group trap rules.go's header records applies here in full and +// is the reason no clip is resolved at parse time. A is stored in the +// coordinates of whatever referenced it, and cairo hoists content into +// with an offsetting transform at the use site — so the same resolves +// to two different page rectangles depending on which reference pulled it in. +// The definition is therefore kept in its own user space and composed with the +// walker's current matrix at the moment of use, exactly as the shapes already are. + +// clipDef is one 's extent in its own user space, with the element's +// own transform already applied so that the stored rectangle is in the +// coordinates of whatever references it. +type clipDef struct { + rect CellRect + ok bool +} + +// clipBox is the effective clip at a point in the walk, in the same output space +// as [Ink.Rect] — that is, after the current matrix and [svgPointScale]. +// +// The zero value is "no clip", which is what the top of the page is. +type clipBox struct { + rect CellRect + set bool +} + +// intersect adds one more clip to the effective one. +func (c clipBox) intersect(r CellRect) clipBox { + if !c.set { + return clipBox{rect: r, set: true} + } + return clipBox{set: true, rect: CellRect{ + X0: math.Max(c.rect.X0, r.X0), Y0: math.Max(c.rect.Y0, r.Y0), + X1: math.Min(c.rect.X1, r.X1), Y1: math.Min(c.rect.Y1, r.Y1), + }} +} + +// empty reports a clip that admits nothing, so every shape under it is invisible +// and the subtree can be abandoned. +func (c clipBox) empty() bool { + return c.set && (c.rect.X1 <= c.rect.X0 || c.rect.Y1 <= c.rect.Y0) +} + +// apply cuts a shape's box back to what the clip admits, reporting whether +// anything is left to paint. +// +// A degenerate shape is the case that needs stating: a hairline rule has zero +// height, so an area test would reject it. The comparison is therefore on each +// axis independently and a zero-extent axis survives as long as it lies inside +// the clip, which is the same question asked of a shape with no thickness that +// [verify.overlap1D] answers the same way. +func (c clipBox) apply(r CellRect) (CellRect, bool) { + if !c.set { + return r, true + } + out := CellRect{ + X0: math.Max(r.X0, c.rect.X0), Y0: math.Max(r.Y0, c.rect.Y0), + X1: math.Min(r.X1, c.rect.X1), Y1: math.Min(r.Y1, c.rect.Y1), + } + if out.X1 < out.X0 || out.Y1 < out.Y0 { + return CellRect{}, false + } + return out, true +} + +// clipAt resolves a clip-path attribute value under the current matrix, giving +// the rectangle it admits in output space. +// +// The four corners of the definition's box are transformed rather than its +// opposite pair, because a matrix with rotation would otherwise produce a +// rectangle that is not the box of the transformed shape. Cairo writes only +// scales, translations and axis flips on these fixtures, for which the two agree +// exactly; under a real rotation this overstates the clip, which is the direction +// that cannot cut a drawing away. +func (d *svgDoc) clipAt(attr string, m matrix) (CellRect, bool) { + ref, ok := refID(attr) + if !ok { + return CellRect{}, false + } + def, ok := d.clips[ref] + if !ok || !def.ok { + return CellRect{}, false + } + r := def.rect + minX, minY := math.Inf(1), math.Inf(1) + maxX, maxY := math.Inf(-1), math.Inf(-1) + for _, p := range [4]point{ + {r.X0, r.Y0}, {r.X1, r.Y0}, {r.X1, r.Y1}, {r.X0, r.Y1}, + } { + x, y := m.apply(p.x, p.y) + x, y = x*svgPointScale, y*svgPointScale + minX, maxX = math.Min(minX, x), math.Max(maxX, x) + minY, maxY = math.Min(minY, y), math.Max(maxY, y) + } + return CellRect{X0: minX, Y0: minY, X1: maxX, Y1: maxY}, true +} + +// readClipPath consumes one element and returns the extent of the +// geometry inside it, in the coordinates of whatever references the clip. +// +// It reads the element from the stream instead of keeping its children in the +// tree, for the reason [svgNode] gives about attributes it does not use: page 42 +// of the columns manual carries 34,920 elements in 30 MB of SVG, and a +// rectangle per clip is a few hundred kilobytes where their subtrees are several +// megabytes. +// +// A transform on the itself and on any element inside it is composed, +// because a clip is geometry like any other and cairo is free to place it with a +// matrix. Nested groups are handled by the stack rather than assumed away. +func readClipPath(dec *xml.Decoder, start *xml.StartElement) (clipDef, error) { + var def clipDef + base := identity + for _, a := range start.Attr { + switch a.Name.Local { + case "transform": + base = parseTransform(a.Value) + case "clipPathUnits": + // The units are the object's own bounding box, which is the box this + // walker is trying to compute. Unresolvable rather than guessed: the + // element is still consumed, and the shape it clips stays unclipped. + if a.Value == "objectBoundingBox" { + return clipDef{}, dec.Skip() + } + } + } + + extend := func(m matrix, r CellRect) { + minX, minY := math.Inf(1), math.Inf(1) + maxX, maxY := math.Inf(-1), math.Inf(-1) + for _, p := range [4]point{ + {r.X0, r.Y0}, {r.X1, r.Y0}, {r.X1, r.Y1}, {r.X0, r.Y1}, + } { + x, y := m.apply(p.x, p.y) + minX, maxX = math.Min(minX, x), math.Max(maxX, x) + minY, maxY = math.Min(minY, y), math.Max(maxY, y) + } + box := CellRect{X0: minX, Y0: minY, X1: maxX, Y1: maxY} + if !def.ok { + def.rect, def.ok = box, true + return + } + def.rect = CellRect{ + X0: math.Min(def.rect.X0, box.X0), Y0: math.Min(def.rect.Y0, box.Y0), + X1: math.Max(def.rect.X1, box.X1), Y1: math.Max(def.rect.Y1, box.Y1), + } + } + + // Several shapes in one are a union, so their boxes are unioned — + // which is again the direction that overstates the clip rather than cutting + // something the page paints. + stack := []matrix{base} + for { + tok, err := dec.Token() + if err != nil { + if errors.Is(err, io.EOF) { + return def, nil + } + return clipDef{}, err + } + switch v := tok.(type) { + case xml.StartElement: + m := stack[len(stack)-1] + var d string + var x, y, w, h float64 + for _, a := range v.Attr { + switch a.Name.Local { + case "transform": + m = m.compose(parseTransform(a.Value)) + case "d": + d = a.Value + case "x": + x = parseFloat(a.Value) + case "y": + y = parseFloat(a.Value) + case "width": + w = parseFloat(a.Value) + case "height": + h = parseFloat(a.Value) + } + } + switch v.Name.Local { + case "path": + if box, ok := pathExtent(d); ok { + extend(m, box) + } + case "rect": + extend(m, CellRect{X0: x, Y0: y, X1: x + w, Y1: y + h}) + } + stack = append(stack, m) + case xml.EndElement: + if len(stack) <= 1 { + return def, nil + } + stack = stack[:len(stack)-1] + } + } +} + +// pathExtent is the box containing a path, control points included. +// +// Deliberately not [subpaths]: that flattens a curve to its endpoints, which +// gives a box a curve can bulge out of. Here the box must contain the whole path, +// because it becomes a clip and a clip that is too small cuts away real ink. A +// Bézier lies within the hull of its control points, so including them is +// sufficient rather than approximate. +// +// The one shape this cannot bound tightly is an elliptical arc, whose bulge is +// implied by radii rather than drawn with control points: only its endpoint is +// read, so an `A` command can understate the box. Cairo emits no arcs — both +// fixtures' 36,000 clip paths are lines and cubics — and stating it is cheaper +// than implementing an arc parameterisation nothing here produces. +func pathExtent(d string) (CellRect, bool) { + toks := tokenizePath(d) + minX, minY := math.Inf(1), math.Inf(1) + maxX, maxY := math.Inf(-1), math.Inf(-1) + found := false + add := func(p point) { + minX, maxX = math.Min(minX, p.x), math.Max(maxX, p.x) + minY, maxY = math.Min(minY, p.y), math.Max(maxY, p.y) + found = true + } + + var pt, start point + cmd := byte('M') + for i := 0; i < len(toks); { + if toks[i].isCmd { + cmd = toks[i].cmd + i++ + if cmd == 'Z' || cmd == 'z' { + pt = start + } + continue + } + var nums []float64 + for i < len(toks) && !toks[i].isCmd { + nums = append(nums, toks[i].num) + i++ + } + upper := cmd &^ 0x20 + k := commandArity(upper) + rel := cmd >= 'a' + for j := 0; j+k <= len(nums); j += k { + a := nums[j : j+k] + switch upper { + case 'H': + if rel { + pt = point{pt.x + a[0], pt.y} + } else { + pt = point{a[0], pt.y} + } + add(pt) + case 'V': + if rel { + pt = point{pt.x, pt.y + a[0]} + } else { + pt = point{pt.x, a[0]} + } + add(pt) + case 'A': + // Only the endpoint is a coordinate; the leading five arguments are + // radii and flags. See the note above about what that costs. + next := point{a[5], a[6]} + if rel { + next = point{pt.x + next.x, pt.y + next.y} + } + pt = next + add(pt) + default: + // Every coordinate pair of the command, so a curve's control points + // are in the box. A relative command's pairs are all relative to the + // point the command started at, which is why pt moves only once, on + // the last pair. + var last point + for p := 0; p+1 < k; p += 2 { + q := point{a[p], a[p+1]} + if rel { + q = point{pt.x + q.x, pt.y + q.y} + } + add(q) + last = q + } + if upper == 'M' && j == 0 { + start = last + } + pt = last + } + } + } + if !found { + return CellRect{}, false + } + return CellRect{X0: minX, Y0: minY, X1: maxX, Y1: maxY}, true +} diff --git a/internal/doc/clip_internal_test.go b/internal/doc/clip_internal_test.go new file mode 100644 index 0000000..02839f1 --- /dev/null +++ b/internal/doc/clip_internal_test.go @@ -0,0 +1,204 @@ +package doc + +import ( + "math" + "testing" +) + +// Unit tests for the clip reader. No poppler and no PDF: figures_fixture_test.go +// drives the real tool against the real manuals. +// +// clipSVG is written to hold the four things that can go wrong, each with numbers +// far enough apart that a wrong answer is a different coordinate rather than a +// different count: +// +// group-1 a clip defined in and applied inside a hoisted compositing +// group, which is the trap rules.go's header records. The two +// translations cancel exactly, as they do in cairo's output, so what +// this case pins is that the clip is found and applied through the +// reference at all: walking from the top instead would shift the +// clip and the rule together by (20, 10). +// scaled a clip on a group whose own transform does NOT cancel, which is what +// pins the composition itself. The clip admits x=10-30 in the group's +// user space, and that space is scaled by two and shifted, so the rule +// survives from x=20 to 60 in page units. Resolving the clip in page +// space instead — the second of the two wrong answers rules.go records — +// gives x=10-30 and a rule half the length in the wrong place. +// nested two clips, one inside the other, on a rule that spans the page. The +// effective clip is the intersection: reading only the inner one gives +// x=50-150 and only the outer one x=20-120, where the answer is 50-120. +// curved a clip whose edge is a Bézier that bulges 20 units past its +// endpoints. The rule it clips lies inside the bulge and outside the +// endpoints, so a clip box built by flattening the curve — which is +// what [subpaths] would give — drops the rule entirely. +// gone a rule drawn wholly outside its clip, which paints nothing and must +// not be recorded at all. +const clipSVG = ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +` + +// TestClipCutsAShapeToWhatIsPainted is the whole point of clip.go: an ink box is +// the visible extent, not the path's own. +func TestClipCutsAShapeToWhatIsPainted(t *testing.T) { + ink, err := parseInk([]byte(clipSVG)) + if err != nil { + t.Fatalf("parseInk: %v", err) + } + // Every rectangle is in output units, which are the SVG's points times + // [svgPointScale]. + want := []CellRect{ + // The hoisted group: the rule runs x=10-70 and the clip admits 0-40, so it + // is painted from 10 to 40. Its two translations cancel, so this pins that + // the clip is followed into at all rather than how it is composed — + // the scaled case below pins that. + {X0: 15, Y0: 30, X1: 60, Y1: 30}, + // Two nested clips intersect to x=50-120. + {X0: 75, Y0: 90, X1: 180, Y1: 90}, + // The clip is read in the group's own space: x=10-30 there is x=20-60 on the + // page under translate(0,10) scale(2,1). Read in page space it would be + // x=10-30, which is 15-45 in output units. + {X0: 30, Y0: 105, X1: 90, Y1: 105}, + // Inside the Bézier's bulge, so x=10-90 survives. + {X0: 15, Y0: 135, X1: 135, Y1: 135}, + } + if len(ink) != len(want) { + t.Fatalf("%d shapes, want %d: %v", len(ink), len(want), ink) + } + for i := range want { + got := ink[i].Rect + if !sameRect(got, want[i]) { + t.Errorf("shape %d is %v, want %v", i, got, want[i]) + } + } +} + +// TestARuleIsRecordedAtTheLengthItIsPainted is the same reading from the table +// side, because both walkers share the clip and a cell boundary is read off where +// a rule ends. +func TestARuleIsRecordedAtTheLengthItIsPainted(t *testing.T) { + rules, err := parseRules([]byte(clipSVG)) + if err != nil { + t.Fatalf("parseRules: %v", err) + } + want := []Rule{ + {Dir: Horizontal, At: 30, Start: 15, End: 60}, + {Dir: Horizontal, At: 90, Start: 75, End: 180}, + {Dir: Horizontal, At: 105, Start: 30, End: 90}, + {Dir: Horizontal, At: 135, Start: 15, End: 135}, + } + if len(rules) != len(want) { + t.Fatalf("%d rules, want %d: %v", len(rules), len(want), rules) + } + for i := range want { + r := rules[i] + if r.Dir != want[i].Dir || math.Abs(r.At-want[i].At) > 0.01 || + math.Abs(r.Start-want[i].Start) > 0.01 || math.Abs(r.End-want[i].End) > 0.01 { + t.Errorf("rule %d is %s at=%.2f %.2f-%.2f, want %s at=%.2f %.2f-%.2f", + i, r.Dir, r.At, r.Start, r.End, + want[i].Dir, want[i].At, want[i].Start, want[i].End) + } + } +} + +// TestAnUnreadableClipLeavesTheShapeAlone is the stance clip.go's header takes: +// a clip that cannot be resolved is no clip, because the old behaviour is wrong +// in a recorded direction where a guess could erase a picture. +func TestAnUnreadableClipLeavesTheShapeAlone(t *testing.T) { + for _, tc := range []struct{ name, clip string }{ + {"a reference to nothing", ``}, + {"units this walker cannot resolve", + ``}, + {"an empty clipPath", ``}, + } { + t.Run(tc.name, func(t *testing.T) { + svg := ` + + + + +` + tc.clip + ` + + +` + ink, err := parseInk([]byte(svg)) + if err != nil { + t.Fatalf("parseInk: %v", err) + } + if len(ink) != 1 { + t.Fatalf("%d shapes, want 1: %v", len(ink), ink) + } + if want := (CellRect{X0: 15, Y0: 15, X1: 135, Y1: 15}); !sameRect(ink[0].Rect, want) { + t.Errorf("shape is %v, want the unclipped %v", ink[0].Rect, want) + } + }) + } +} + +// TestPathExtentHoldsTheWholeCurve is why this file does not call [subpaths]: a +// clip box must contain the path, and a curve leaves its endpoints' box. +func TestPathExtentHoldsTheWholeCurve(t *testing.T) { + // The same cubic the clip fixture uses. Its endpoints are at y=80 and its + // control points at y=100, so the curve reaches below 80 and the box must too. + box, ok := pathExtent("M 10 80 C 10 100, 90 100, 90 80 Z M 10 80 ") + if !ok { + t.Fatal("no extent read") + } + if want := (CellRect{X0: 10, Y0: 80, X1: 90, Y1: 100}); !sameRect(box, want) { + t.Errorf("extent is %v, want %v", box, want) + } + + // Relative commands take every control point from the point the command + // started at, not from the previous pair. + box, ok = pathExtent("m 10 10 c 0 20, 40 20, 40 0") + if !ok { + t.Fatal("no extent read") + } + if want := (CellRect{X0: 10, Y0: 10, X1: 50, Y1: 30}); !sameRect(box, want) { + t.Errorf("relative extent is %v, want %v", box, want) + } +} diff --git a/internal/doc/columnlang.go b/internal/doc/columnlang.go new file mode 100644 index 0000000..6ea3acd --- /dev/null +++ b/internal/doc/columnlang.go @@ -0,0 +1,142 @@ +package doc + +import ( + "fmt" + "sort" + "strings" +) + +// ColumnLanguage is what one text column of a page turned out to be. +// +// The unit is the column and not the page, because on a manual whose languages +// run in parallel a page holds several. Everything about naming a language has +// to be asked per column: a tag printed on the page belongs to one of them, and +// the alphabet evidence of one column says nothing about its neighbour. +type ColumnLanguage struct { + Column Column `json:"column"` + // Code is the label as printed, which need not be a valid tag: real manuals + // print D, RUS, UA and KAZ. + Code string `json:"code,omitempty"` + // Lang is the BCP-47 tag, empty when nothing was established. + Lang string `json:"lang,omitempty"` + // Source is the signal that named it, empty when none could. + Source Source `json:"source,omitempty"` + // Conflict marks a column whose printed tag and whose alphabet disagree. + // Recorded, never resolved silently. + Conflict bool `json:"conflict"` + // Note says in checkable terms how the column was read. + Note string `json:"note,omitempty"` +} + +// topRunsForTag is how many of a column's leading runs are searched for its +// printed tag. +// +// Not one: a right-to-left column puts its heading before the tab in reading +// order, and on the measured manual the tag sits on the second line. Not many +// either, since the further down the column the search goes the more ordinary +// words it meets. Five covers a heading, a subheading and the tab. +const topRunsForTag = 5 + +// ColumnLanguages names each column of a page. +// +// knownCodes is the vocabulary the document's own contents table declares, and +// it is what makes a single-letter tag usable: "D" is German on a manual whose +// index lists D, and a list marker everywhere else. Pass nil when no index was +// parsed — single letters are then believed only if the column's own alphabet +// agrees with them. +func ColumnLanguages(runs []TextRun, cols []Column, knownCodes map[string]bool) []ColumnLanguage { + out := make([]ColumnLanguage, 0, len(cols)) + for i := range cols { + out = append(out, nameColumn(runs, &cols[i], knownCodes)) + } + return out +} + +// nameColumn decides one column's language from its tag and its alphabet. +func nameColumn(runs []TextRun, col *Column, knownCodes map[string]bool) ColumnLanguage { + inside := runsInColumn(runs, col) + result := ColumnLanguage{Column: *col} + + tagCode, tagLang := columnTag(inside, knownCodes) + + var text strings.Builder + for i := range inside { + text.WriteString(inside[i].Text) + text.WriteByte(' ') + } + rep := MatchRepertoire(text.String()) + repLang, repNamed := rep.Language() + + switch { + case tagLang != "" && repNamed && SameLanguage(tagLang, repLang): + // Both signals, agreeing. The strongest reading available: the document + // says so and its own letters bear it out. + result.Code, result.Lang, result.Source = tagCode, tagLang, SourcePageTag + result.Note = fmt.Sprintf("printed %s, and the column's alphabet agrees", tagCode) + + case tagLang != "" && repNamed: + // Both signals, disagreeing. Prefer the printed tag — it is the document + // asserting its own language — but record the conflict, because one of + // them is wrong and the user is better placed to say which. + result.Code, result.Lang, result.Source = tagCode, tagLang, SourcePageTag + result.Conflict = true + result.Note = fmt.Sprintf("printed %s, but the column's alphabet reads as %s", + tagCode, DisplayName(repLang)) + + case tagLang != "": + result.Code, result.Lang, result.Source = tagCode, tagLang, SourcePageTag + result.Note = fmt.Sprintf("printed %s; no distinctive letters to corroborate it", tagCode) + + case repNamed: + // No tag. Only three of the measured manual's five languages print one, + // so this is the common case rather than a fallback. + result.Lang, result.Source = repLang, SourceRepertoire + result.Code = strings.ToUpper(BaseLanguage(repLang)) + result.Note = rep.Note + + case len(rep.Tied()) > 1: + // The alphabet narrowed it and cannot finish. Naming one would be a coin + // toss; saying which two it is between is useful. + result.Note = fmt.Sprintf("alphabet fits %s equally", strings.Join(rep.Tied(), " and ")) + + default: + result.Note = "no printed tag and no distinctive letters" + } + return result +} + +// runsInColumn returns the runs belonging to a column, in reading order. +// +// Membership is [runsInBox], shared with the region reader so that a region's +// characters are counted over exactly the runs its language was read from. The +// sort is what this adds: the printed tag is looked for in a column's leading +// runs, and "leading" means down the page rather than in the order the tool +// happened to emit them. +func runsInColumn(runs []TextRun, col *Column) []TextRun { + inside := runsInBox(runs, col.Min, col.Max) + sort.SliceStable(inside, func(a, b int) bool { return inside[a].Y < inside[b].Y }) + return inside +} + +// columnTag finds a language code printed within a column. +func columnTag(inside []TextRun, knownCodes map[string]bool) (code, lang string) { + limit := min(topRunsForTag, len(inside)) + for i := range inside[:limit] { + token := strings.TrimSpace(stripFormatting(inside[i].Text)) + if token == "" || len([]rune(token)) > maxRunesInCodeLine { + continue + } + if !looksLikeLanguageCode(token) || !PlausibleCodeToken(token) { + continue + } + // A single letter is the most ambiguous token a page carries, so it is + // taken only where the document's own index lists it. + if singleLetterNeedsSupport(token) && !knownCodes[strings.ToUpper(token)] { + continue + } + if normalised, ok := NormalizeCode(token); ok { + return strings.ToUpper(token), normalised + } + } + return "", "" +} diff --git a/internal/doc/columnlang_test.go b/internal/doc/columnlang_test.go new file mode 100644 index 0000000..b93187c --- /dev/null +++ b/internal/doc/columnlang_test.go @@ -0,0 +1,186 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for naming a column's language. No PDF, no poppler: runs are +// built here. The real-document acceptance lives against the fixtures. + +// runsAt builds a column's worth of text runs stacked down the page. +func runsAt(x float64, lines ...string) []doc.TextRun { + out := make([]doc.TextRun, 0, len(lines)) + for i, s := range lines { + out = append(out, doc.TextRun{ + X: x, Y: float64(20 + i*16), Width: 250, Height: 14, Text: s, + }) + } + return out +} + +func col(x0, x1 float64, runs int) doc.Column { + return doc.Column{Min: x0, Max: x1, Runs: runs} +} + +// german and polish are ordinary manual prose carrying each language's own +// letters, so the alphabet signal has something to work with. +const ( + german = "Gerät sorgfältig prüfen und für spätere Zwecke aufbewahren. Größe beachten." + polish = "Urządzenie należy sprawdzić i zachować instrukcję. Część zamienna dostępna." + ukr = "Пристрій слід перевірити та зберегти інструкцію. Її потрібно вивчити." + rus = "Прибор следует проверить и сохранить инструкцию. Её нужно изучить." +) + +func TestColumnNamedByTagAndAlphabetAgreeing(t *testing.T) { + runs := runsAt(30, "PL", polish, polish) + got := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 3)}, nil) + + if len(got) != 1 { + t.Fatalf("expected 1 column, got %d", len(got)) + } + if got[0].Lang != "pl" || got[0].Source != doc.SourcePageTag { + t.Errorf("got %q via %q, want pl via page-tag", got[0].Lang, got[0].Source) + } + if got[0].Conflict { + t.Error("agreement was recorded as a conflict") + } + if !strings.Contains(got[0].Note, "agrees") { + t.Errorf("note should record the corroboration, got %q", got[0].Note) + } +} + +func TestColumnTagAndAlphabetDisagreeingIsRecorded(t *testing.T) { + // The tag says Polish, the letters are unmistakably German. One of them is + // wrong and the design forbids resolving that silently. + runs := runsAt(30, "PL", german, german) + got := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 3)}, nil) + + if !got[0].Conflict { + t.Error("a tag contradicted by the alphabet must be flagged") + } + if got[0].Lang != "pl" { + t.Errorf("lang = %q; the printed tag should still win, being the document's own claim", got[0].Lang) + } + if !strings.Contains(got[0].Note, "German") { + t.Errorf("the note must name what the alphabet read instead, got %q", got[0].Note) + } +} + +func TestColumnNamedByAlphabetAloneWhenNoTag(t *testing.T) { + // The common case rather than a fallback: only three of the measured + // manual's five languages print a tag at all. + runs := runsAt(30, ukr, ukr, ukr) + got := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 3)}, nil) + + if got[0].Lang != "uk" || got[0].Source != doc.SourceRepertoire { + t.Errorf("got %q via %q, want uk via repertoire", got[0].Lang, got[0].Source) + } +} + +func TestNeighbouringColumnsAreNamedIndependently(t *testing.T) { + // The whole point of working per column. A page's two columns must not + // contaminate one another, and reading the page as one blob would name at + // most one of them. + var runs []doc.TextRun + runs = append(runs, runsAt(30, "D", german, german)...) + runs = append(runs, runsAt(320, rus, rus, rus)...) + + got := doc.ColumnLanguages(runs, + []doc.Column{col(30, 280, 3), col(320, 570, 3)}, + map[string]bool{"D": true}) + + if len(got) != 2 { + t.Fatalf("expected 2 columns, got %d", len(got)) + } + if got[0].Lang != "de" { + t.Errorf("left column = %q, want de", got[0].Lang) + } + if got[1].Lang != "ru" { + t.Errorf("right column = %q, want ru", got[1].Lang) + } +} + +func TestSingleLetterTagNeedsTheIndexVocabulary(t *testing.T) { + // "D" is German on a manual whose contents table lists D, and a figure label + // everywhere else. Without the vocabulary the column falls back to its + // alphabet, which here happens to agree — the point is which signal was used. + runs := runsAt(30, "D", german, german) + + with := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 3)}, map[string]bool{"D": true}) + if with[0].Source != doc.SourcePageTag { + t.Errorf("with the vocabulary the tag should be used, got %q", with[0].Source) + } + + without := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 3)}, nil) + if without[0].Source == doc.SourcePageTag { + t.Error("without the vocabulary a single letter must not be taken as a tag") + } + if without[0].Lang != "de" { + t.Errorf("the alphabet should still name it de, got %q", without[0].Lang) + } +} + +func TestTagIsFoundBelowTheColumnHeading(t *testing.T) { + // A right-to-left column puts its heading before the tab in reading order, + // so the tag is not the first run. Searching only the first would lose it. + runs := runsAt(30, "Sicherheitshinweise", "D", german) + got := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 3)}, map[string]bool{"D": true}) + + if got[0].Source != doc.SourcePageTag || got[0].Code != "D" { + t.Errorf("got %q via %q, want the D printed on the second line", got[0].Code, got[0].Source) + } +} + +func TestUnnameableColumnSaysSo(t *testing.T) { + // English has no distinctive letters, so with no tag there is nothing to go + // on. Saying nothing is the correct answer and it must carry a reason. + runs := runsAt(30, "Please read these instructions before use and keep them safe.") + got := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 1)}, nil) + + if got[0].Lang != "" { + t.Errorf("named %q from text with no distinctive letters", got[0].Lang) + } + if got[0].Note == "" { + t.Error("an unnamed column must explain why") + } +} + +func TestIndistinguishableLanguagesReportTheTie(t *testing.T) { + // Danish and Norwegian share their whole repertoire. Naming one would be a + // coin toss; naming both is useful. + runs := runsAt(30, "Læs denne vejledning grundigt før brug og gem den på et sikkert sted.") + got := doc.ColumnLanguages(runs, []doc.Column{col(30, 280, 1)}, nil) + + if got[0].Lang != "" { + t.Errorf("picked %q from an ambiguous alphabet", got[0].Lang) + } + if !strings.Contains(got[0].Note, "equally") { + t.Errorf("the note should name the tie, got %q", got[0].Note) + } +} + +func TestRunsSpanningTwoColumnsBelongToNeither(t *testing.T) { + // A heading set across the measure is not evidence for either column. + var runs []doc.TextRun + runs = append(runs, doc.TextRun{X: 30, Y: 5, Width: 540, Height: 20, Text: "Wskazówki " + polish}) + runs = append(runs, runsAt(30, german, german)...) + runs = append(runs, runsAt(320, german, german)...) + + got := doc.ColumnLanguages(runs, + []doc.Column{col(30, 280, 2), col(320, 570, 2)}, nil) + + for i, c := range got { + if c.Lang != "de" { + t.Errorf("column %d = %q, want de — the spanning Polish heading leaked in", i, c.Lang) + } + } +} + +func TestNoColumnsGivesNoAnswers(t *testing.T) { + if got := doc.ColumnLanguages(nil, nil, nil); len(got) != 0 { + t.Errorf("got %d answers for no columns", len(got)) + } +} diff --git a/internal/doc/columns.go b/internal/doc/columns.go new file mode 100644 index 0000000..ce0c252 --- /dev/null +++ b/internal/doc/columns.go @@ -0,0 +1,696 @@ +package doc + +import ( + "fmt" + "math" + "sort" + "strings" +) + +// Text columns are found by projecting text runs onto the x-axis and looking +// for the bands that almost nothing crosses. +// +// Two simpler things were tried first and both fail on a real manual, which is +// why this is more than a one-liner. +// +// **Binary coverage fails.** Splitting wherever no run at all covers an x gets +// 63 of 68 pages of the measured fixture right and merges the rest. On its +// page 63 a single section heading runs across the top of both columns; on its +// page 68 a banner heading crosses all three. One run out of eighty is enough +// to weld two columns together for ever, and no choice of gap width repairs it. +// +// **Left-alignment peaks fail**, the other way, by over-splitting. A parts list +// set with a hanging indent puts markers at x=30 and their text at x=60, so a +// three-column page offers six peaks; a table nested inside a column adds a +// seventh at x=192. No "merge peaks closer than N" rule separates a 30-unit +// hanging indent from a 162-unit table indent and still keeps two real columns +// 280 units apart. +// +// What works is counting, not testing for emptiness: for each x, how many runs +// cross it. A gutter is then a band that *few* runs cross rather than none, +// which absorbs the spanning heading, and it is a page-wide statistic rather +// than a local one, which is what makes it immune to indents. Measured on the +// fixture's page 13, the hanging indent at x=43-59 is crossed by 10 runs — +// the lines of that same column that are set to the full measure — while its +// real gutter at x=290-309 is crossed by none. On page 63 the nested table's +// gap at x=186-191 is crossed by 17 runs, the paragraphs printed above the +// table. Neither can be mistaken for a column boundary once you count. +// +// See docs/design/layouts.md for why column geometry is needed at all: in a +// parallel-columns manual a column is a language, and a page is not. + +// Bounds on what counts as a text run, a gutter and a column. +// +// Every one of these is measured against the Thomas DryBox Amfibia fixture +// (68 pages, testdata/fixtures/thomas-drybox-amfibia.json), read through +// `pdftohtml -xml`, whose 892-unit page space matches a `pdftoppm -r 108` +// raster 1:1 so that a detected box can be drawn on the page and looked at. +const ( + // minRunHeightFraction is how short a run may be, against the page's own + // median run height, and still count as text. + // + // This is the artifact filter, and it is not optional. The fixture's text + // layer carries an InDesign filename slug and an export timestamp 260 times + // each across 67 of its 68 pages — 8% of all runs — because most of its + // illustrations are placed PDFs that each brought their own slug along, + // scaled down with the artwork. Several sit in a gutter. + // + // Height separates them cleanly where their text does not: 520 of the 522 + // artifact runs are 2 to 6 units tall against a body median of 17 to 21 + // (0.12 to 0.35 of it), while the shortest genuine text on any page is 9 + // (0.53). 0.4 sits in that gap. Measured effect: without it the right-hand + // column of 65 of the 68 pages stretches into the margin to swallow a slug. + // + // Repetition, the more obvious filter, is wrong in both directions. Keying + // on "appears many times on this page" misses the artifact entirely on the + // pages carrying only one placed graphic (two occurrences), and on the back + // page of service addresses it would delete 742 of 769 genuine runs, + // because "Robert Thomas" really is printed twelve times there. Keying on + // "appears on many pages" is worse: it also matches the printed language + // tags D, PL and UA, which are the most valuable signal in the document. + minRunHeightFraction = 0.4 + + // maxGutterCrossings is how many runs may cross a band and leave it still a + // gutter. It is a count of spanning furniture — a banner heading, a footer, + // a caption set across the measure — not a fraction of the page. + // + // Measured: the fixture's page 63 carries one such run (the section heading + // over both columns), page 68 two. 4 leaves headroom for a page with a + // heading, a footer and a caption. The eight ground-truth pages all come out + // right for any value from 2 to 7. + maxGutterCrossings = 4 + + // minGutterFraction is how wide a low-crossing band must be, as a fraction + // of page width, before it is believed to be a gutter rather than a chance + // alignment of word spaces. + // + // 1% is 8.9 units on the fixture's 892-unit page. The narrowest true gutter + // measured there is 13 units (page 68, between its second and third address + // columns); the eight ground-truth pages come out right for anything from 4 + // to 12 units. Held as a fraction, not a length, so it survives a page + // rendered at another resolution. + minGutterFraction = 0.01 + + // minColumnRuns is how many text runs a region needs before it is a column. + // + // Without it an exploded diagram becomes a page of columns: the fixture's + // page 12 scatters numbered callouts across two thirds of its width, in + // clusters of one to five runs, beside a single real text column of 94. + // docs/design/layouts.md records a four-run margin element already having + // been miscounted as a column once, and the fixture manifest settled on + // eight for the same reason. Nothing between 1 and 16 changes the answer on + // the eight ground-truth pages, so this is a guard against pages not yet + // seen rather than a tuned value. + minColumnRuns = 8 + + // minColumnWidthFraction is how wide a region must be, against the page, + // before it is a column rather than a strip. + // + // This guards a case the fixture only narrowly avoids. A list set with a + // hanging indent puts its markers in a strip of their own — x=30 to 42, with + // the text from x=60 — and the space between is a page-wide band no text + // crosses, which is exactly the definition of a gutter used here. On the + // fixture's page 13 ten lines of that column happen to be set to the full + // measure and cross it, so the strip never separates; a list without such + // lines would hand its bullets back as a column. + // + // Width settles it where crossings cannot. Measured on the fixture: the + // narrowest real column is 227 units, 25% of the page, and the narrowest + // cell column of its troubleshooting tables 116, or 13%; a marker strip is + // 13 units, 1.5%. 5% lies between them with a wide margin either way, and it + // changes no answer on the 68 measured pages. + minColumnWidthFraction = 0.05 + + // baselineToleranceFraction and minBaselineShare decide when a strip too + // narrow to be a column is really part of the column beside it. + // + // A list marker and the text it labels sit on one line — poppler reports + // both with the same top — and that shared baseline is what makes them one + // column. The projection cannot see it, because the space between a marker + // and its text is as empty as any gutter. So a narrow strip is folded into + // its neighbour when most of its runs line up with that neighbour's. + // + // The tolerance is small on purpose. Markers align exactly, so it needs to + // absorb rounding and nothing more; at 15% of the median run height it is + // about 2.5 units against a 21-unit line pitch, which leaves a stray figure + // callout roughly one chance in four of matching any given line by accident + // and almost none of matching four fifths of them. + baselineToleranceFraction = 0.15 + minBaselineShare = 0.8 + + // maxProjectionBuckets caps the projection array. Page width reaches this + // code from a caller's coordinates, and a nonsense value would otherwise + // turn into a nonsense allocation — the same reasoning as maxExtractedBytes + // in pdf.go. 100,000 is an A0 page at 300 dpi with room to spare. + maxProjectionBuckets = 100_000 +) + +// TextRun is one positioned run of text on a page. +// +// X and Y are its top-left corner in the same units as the page dimensions +// passed to [DetectColumns]; the detector never assumes what those units are, +// beyond needing enough of them across a page to resolve a gutter. Poppler's +// `pdftohtml -xml` reports exactly these four numbers per run, which is where +// the shape comes from. +type TextRun struct { + X float64 `json:"x"` + Y float64 `json:"y"` + Width float64 `json:"width"` + Height float64 `json:"height"` + Text string `json:"text"` + // Font is what the run is set in, and it is what will tell a heading from a + // paragraph. Nothing in this file reads it — the column detector is pure + // geometry — but it travels with the run because resolving it needs the + // document's font table, which only [ExtractRuns] sees. + // + // Its zero value means "not known", which is what a run built by hand in a + // test carries. That is why it comes last and why the five fields above are + // unchanged: every existing caller constructs those positionally or by name + // and must keep compiling and meaning the same thing. + Font Font `json:"font,omitzero"` +} + +func (r *TextRun) right() float64 { return r.X + r.Width } +func (r *TextRun) bottom() float64 { return r.Y + r.Height } + +// Column is one text column of a page. +type Column struct { + // Min and Max are the x-range the column's text actually occupies, not the + // band it was cut from: they are the leftmost and rightmost edges of the + // runs assigned to it. A caller clipping text to this range gets the + // column's own words and no others. + Min float64 `json:"min"` + Max float64 `json:"max"` + // Runs is how many text runs the column holds. It is the density evidence: + // a column of five runs is margin furniture, not a column. + Runs int `json:"runs"` + // Note says in checkable terms why this is a column. + Note string `json:"note,omitempty"` +} + +// Width is the column's horizontal extent. +func (c *Column) Width() float64 { return c.Max - c.Min } + +// Gutter is a band that separates two columns. +type Gutter struct { + // Min and Max are the band's x-range. + Min float64 `json:"min"` + Max float64 `json:"max"` + // Crossings is how many runs cross it — the spanning headings and footers + // that binary coverage mistakes for proof that the columns are one. + Crossings int `json:"crossings"` +} + +// ColumnLayout is what the detector concluded about one page. +// +// An empty Columns is a normal outcome, not a failure: a full-page photograph +// and a diagram with nothing but callouts both have no text column, and saying +// so is more useful than naming one. Note explains which it was. +type ColumnLayout struct { + // Columns are the page's text columns, left to right. + Columns []Column `json:"columns,omitempty"` + // Gutters are the bands the columns were cut at, left to right. + Gutters []Gutter `json:"gutters,omitempty"` + // Runs is how many runs survived filtering and were projected. + Runs int `json:"runs"` + // Dropped counts the runs excluded before projection, by reason. Present so + // that a page whose text vanished can be explained rather than guessed at. + Dropped DroppedRuns `json:"dropped"` + // Spanning is how many runs crossed a gutter and so belong to no single + // column — the headings and footers set across the measure. + Spanning int `json:"spanning"` + // Note says in checkable terms how the page was read. + Note string `json:"note,omitempty"` +} + +// DroppedRuns records why runs were excluded from the projection. +type DroppedRuns struct { + // Blank is runs with no visible text. + Blank int `json:"blank"` + // Rotated is runs with no horizontal extent. Poppler reports rotated text + // with width 0, which is a marginal note turned on its side — real text, + // but not evidence of a column, and it must not stretch one. + Rotated int `json:"rotated"` + // OffPage is runs lying outside the page box. These are not a curiosity: + // the fixture's back page carries seven lines of a superseded address list + // parked above the top edge, and counting them merges two of its three + // columns. + OffPage int `json:"offPage"` + // Small is runs too short to be body text — see [minRunHeightFraction]. + Small int `json:"small"` +} + +// Total is how many runs were dropped for all reasons. +func (d *DroppedRuns) Total() int { return d.Blank + d.Rotated + d.OffPage + d.Small } + +// DetectColumns finds the text columns of one page. +// +// The runs are one page's text with coordinates, in any consistent unit; +// pageWidth and pageHeight are that page's box in the same unit. Column widths +// are not assumed to be equal, and neither is their number, their pitch, nor +// that a page has any: all three vary within a single real manual, sometimes +// between facing pages. See docs/design/layouts.md. +// +// There is deliberately no confidence score. The honest evidence is countable — +// how many runs a column holds, how many crossed each gutter, how many runs +// were dropped and why — and a number in 0 to 1 synthesised from those would +// only hide them. Every field here is something a reader can check against the +// page. +func DetectColumns(runs []TextRun, pageWidth, pageHeight float64) ColumnLayout { + return detectColumns(runs, pageWidth, pageHeight, layoutGates) +} + +// columnGates are the two bounds that decide how forgiving the projection is: how +// many runs may cross a band and leave it a gutter, and how many runs a region needs +// before it is worth reporting. +// +// They are a parameter and not two constants because the same projection answers two +// different questions. See [layoutGates] and [readingGates]. +type columnGates struct { + maxCrossings int + minRuns int +} + +// layoutGates answer "what are this page's text columns" — a published fact that +// language attribution reads, so it tolerates a banner heading crossing a gutter and +// insists a column carry real text. Both numbers are measured; see +// [maxGutterCrossings] and [minColumnRuns]. +// +// readingGates answer the narrower question "in what order are these runs read", and +// both bounds go to their limit for a measured reason. +// +// Crossings goes to 0 because a *count* of crossings does not survive a sparse page. +// maxGutterCrossings=4 is 2% of a dense page's runs and 24% of the sequential manual's +// page 530, which carries 17; there, every x on the page is crossed by at most 4 runs, +// so the projection reports the whole right-hand half as one gutter and the page as +// one column. A band no run crosses at all cannot swallow text that way, and the +// reason binary coverage was rejected for [DetectColumns] — one spanning heading welds +// two columns for ever — is not a reason here, because welding two strips together is +// what already happens on this path and the worst a missed corridor can do is leave it. +// +// Runs goes to 1 because [minColumnRuns] is what fails on exactly these pages: page +// 530's right-hand column holds 6 runs and page 533's left-hand one holds 6, both +// under 8, so neither is called a column and the page is read as a single strip +// running across the gutter. A strip of one run is not a claim that the page has a +// column there; it is a claim that the run is not on the same line as the text on the +// other side of an empty corridor, which is true. +var ( + layoutGates = columnGates{maxCrossings: maxGutterCrossings, minRuns: minColumnRuns} + readingGates = columnGates{maxCrossings: 0, minRuns: 1} +) + +// readingStrips divides a region into the strips reading order runs down, for the +// pages [DetectColumns] cannot call. It is the fallback in [readingGroups] and +// nothing else may use it: these are not the page's columns. +func readingStrips(runs []TextRun, pageWidth, pageHeight float64) []Column { + return detectColumns(runs, pageWidth, pageHeight, readingGates).Columns +} + +func detectColumns(runs []TextRun, pageWidth, pageHeight float64, g columnGates) ColumnLayout { + var out ColumnLayout + + kept := usableRuns(runs, pageWidth, pageHeight, &out.Dropped) + out.Runs = len(kept) + if len(kept) == 0 { + out.Note = fmt.Sprintf("no usable text runs (%s)", dropSummary(&out.Dropped)) + return out + } + + buckets := int(math.Ceil(pageWidth)) + 1 + if buckets < 2 || buckets > maxProjectionBuckets { + out.Note = fmt.Sprintf("page width %g cannot be projected", pageWidth) + return out + } + + crossings := project(kept, buckets) + minGutter := minGutterFraction * pageWidth + gutters := findGutters(crossings, minGutter, g.maxCrossings) + + inkMin, inkMax := extent(kept) + regions := between(gutters, inkMin, inkMax) + + out.Columns, out.Spanning = assign(kept, regions, gutters, + minColumnWidthFraction*pageWidth, baselineToleranceFraction*medianHeight(kept), g.minRuns) + out.Gutters = keepInnerGutters(gutters, crossings, out.Columns) + out.Note = layoutNote(&out, minGutter, g) + return out +} + +// usableRuns drops everything that is not body text, recording why. +func usableRuns(runs []TextRun, pageWidth, pageHeight float64, dropped *DroppedRuns) []TextRun { + texted := make([]TextRun, 0, len(runs)) + for i := range runs { + if strings.TrimSpace(runs[i].Text) == "" { + dropped.Blank++ + continue + } + texted = append(texted, runs[i]) + } + if len(texted) == 0 { + return nil + } + + // The page's own median height is the reference, not a fixed size: a cover + // set in 34-unit type and a body page set in 17 must both be judged against + // what is normal for themselves. + minHeight := minRunHeightFraction * medianHeight(texted) + + kept := make([]TextRun, 0, len(texted)) + for i := range texted { + r := &texted[i] + switch { + case r.Width <= 0: + dropped.Rotated++ + case r.X < 0 || r.right() > pageWidth || r.Y < 0 || r.bottom() > pageHeight: + dropped.OffPage++ + case r.Height < minHeight: + dropped.Small++ + default: + kept = append(kept, *r) + } + } + return kept +} + +func medianHeight(runs []TextRun) float64 { + hs := make([]float64, len(runs)) + for i := range runs { + hs[i] = runs[i].Height + } + sort.Float64s(hs) + n := len(hs) + if n%2 == 1 { + return hs[n/2] + } + return (hs[n/2-1] + hs[n/2]) / 2 +} + +// project counts, for each x, how many runs cross it. +func project(runs []TextRun, buckets int) []int { + crossings := make([]int, buckets) + for i := range runs { + lo, hi := bucketRange(&runs[i], buckets) + for x := lo; x <= hi; x++ { + crossings[x]++ + } + } + return crossings +} + +func bucketRange(r *TextRun, buckets int) (lo, hi int) { + lo = int(math.Floor(r.X)) + hi = int(math.Floor(r.right())) + if lo < 0 { + lo = 0 + } + if hi > buckets-1 { + hi = buckets - 1 + } + return lo, hi +} + +// span is a half-open-free inclusive x-range in bucket coordinates. +type span struct{ lo, hi int } + +func (s span) width() int { return s.hi - s.lo + 1 } + +// findGutters returns the bands few enough runs cross, wide enough to believe. +func findGutters(crossings []int, minWidth float64, maxCrossings int) []span { + var out []span + start := -1 + for x := 0; x <= len(crossings); x++ { + low := x < len(crossings) && crossings[x] <= maxCrossings + switch { + case low && start < 0: + start = x + case !low && start >= 0: + if s := (span{start, x - 1}); float64(s.width()) >= minWidth { + out = append(out, s) + } + start = -1 + } + } + return out +} + +// between returns the regions left over once the gutters are removed, clipped +// to where there is ink. The clipping is what keeps a page's blank margins from +// being offered as columns. +func between(gutters []span, inkMin, inkMax float64) []span { + lo, hi := int(math.Floor(inkMin)), int(math.Floor(inkMax)) + var out []span + cur := lo + for _, g := range gutters { + if g.hi < cur || g.lo > hi { + continue + } + if g.lo > cur { + out = append(out, span{cur, g.lo - 1}) + } + cur = g.hi + 1 + } + if cur <= hi { + out = append(out, span{cur, hi}) + } + return out +} + +func extent(runs []TextRun) (lo, hi float64) { + lo, hi = math.Inf(1), math.Inf(-1) + for i := range runs { + lo = math.Min(lo, runs[i].X) + hi = math.Max(hi, runs[i].right()) + } + return lo, hi +} + +// assign puts each run in a region and turns the regions that earn it into +// columns. A run crossing a gutter belongs to no column and is counted instead: +// a heading printed across two columns is evidence about neither. +func assign(runs []TextRun, regions, gutters []span, minWidth, baselineTol float64, + minRuns int) (cols []Column, spanning int) { + members := make([][]int, len(regions)) + + for i := range runs { + r := &runs[i] + if crossesAny(r, gutters) { + spanning++ + continue + } + if k := regionOf(r, regions); k >= 0 { + members[k] = append(members[k], i) + } + } + + absorbStrips(runs, members, minWidth, baselineTol) + + for k := range regions { + mine := members[k] + if len(mine) < minRuns { + continue + } + lo, hi := math.Inf(1), math.Inf(-1) + for _, i := range mine { + lo = math.Min(lo, runs[i].X) + hi = math.Max(hi, runs[i].right()) + } + if hi-lo < minWidth { + continue + } + cols = append(cols, Column{ + Min: lo, Max: hi, Runs: len(mine), + Note: fmt.Sprintf("%d text runs between x=%.0f and x=%.0f", len(mine), lo, hi), + }) + } + return cols, spanning +} + +// absorbStrips folds a region too narrow to be a column into the column beside +// it, when the two sit on the same lines. +// +// This is the hanging indent. A parts list puts its markers at x=30 and their +// text at x=60, and the space between is page-wide and empty — a gutter by +// every test the projection can apply. The fixture's page 13 escapes only +// because ten lines of each column are set to the full measure and cross the +// indent; a list without such lines would otherwise lose its markers from the +// column's x-range, and a caller clipping to that range would lose the item +// numbers with them. +// +// A figure callout sitting beside a column is not absorbed, because it does not +// share the column's baselines. That is the whole distinction, and it is a +// property of the page rather than a threshold: a marker and its text are one +// line of one column, a callout and a column are not. +func absorbStrips(runs []TextRun, members [][]int, minWidth, baselineTol float64) { + for k := range members { + if len(members[k]) == 0 || spread(runs, members[k]) >= minWidth { + continue + } + best, bestShared := -1, 0 + for _, n := range []int{k - 1, k + 1} { + if n < 0 || n >= len(members) || len(members[n]) == 0 { + continue + } + if spread(runs, members[n]) < minWidth { + continue + } + shared := sharedBaselines(runs, members[k], members[n], baselineTol) + if shared > bestShared { + best, bestShared = n, shared + } + } + if best < 0 || float64(bestShared) < minBaselineShare*float64(len(members[k])) { + continue + } + members[best] = append(members[best], members[k]...) + members[k] = nil + } +} + +// spread is how wide the runs of a region reach. +func spread(runs []TextRun, idx []int) float64 { + lo, hi := math.Inf(1), math.Inf(-1) + for _, i := range idx { + lo = math.Min(lo, runs[i].X) + hi = math.Max(hi, runs[i].right()) + } + return hi - lo +} + +// sharedBaselines counts how many of a strip's runs sit on a line that the +// other region also occupies. +func sharedBaselines(runs []TextRun, strip, other []int, tol float64) int { + n := 0 + for _, i := range strip { + for _, j := range other { + if sameBaseline(runs[i].Y, runs[j].Y, tol) { + n++ + break + } + } + } + return n +} + +// sameBaseline is the single definition of "these runs sit on one line". +// +// Poppler reports every run of a line with the same top, so this has rounding to +// absorb and nothing more — see [baselineToleranceFraction] for the tolerance and +// what it was measured against. Shared with blocks.go, which folds runs into lines +// for a different purpose: a marker and its text are one line whether the question +// being asked is which column they belong to or which paragraph. +func sameBaseline(a, b, tol float64) bool { return math.Abs(a-b) <= tol } + +// crossesAny reports whether a run passes right over a gutter. Reaching into +// one is not crossing it: a column's longest lines routinely end inside the +// whitespace beside them. +func crossesAny(r *TextRun, gutters []span) bool { + for _, g := range gutters { + if r.X < float64(g.lo) && r.right() > float64(g.hi) { + return true + } + } + return false +} + +// regionOf places a run by its left edge, which is where its column is: a run +// starting inside a gutter is flowing into the column on its right. +func regionOf(r *TextRun, regions []span) int { + x := int(math.Floor(r.X)) + for k := range regions { + if x <= regions[k].hi { + if x >= regions[k].lo || r.right() >= float64(regions[k].lo) { + return k + } + return -1 + } + } + return -1 +} + +// keepInnerGutters reports only the gutters that actually separate two reported +// columns. The rest are margins and the blank space around a diagram, which are +// true of the page but say nothing about its columns. +func keepInnerGutters(gutters []span, crossings []int, cols []Column) []Gutter { + if len(cols) < 2 { + return nil + } + var out []Gutter + for _, g := range gutters { + if !separatesTwo(g, cols) { + continue + } + out = append(out, Gutter{ + Min: float64(g.lo), + Max: float64(g.hi), + Crossings: maxIn(crossings, g), + }) + } + return out +} + +func separatesTwo(g span, cols []Column) bool { + for i := 0; i+1 < len(cols); i++ { + if cols[i].Max <= float64(g.hi) && cols[i+1].Min >= float64(g.lo) { + return true + } + } + return false +} + +func maxIn(crossings []int, s span) int { + best := 0 + for x := s.lo; x <= s.hi && x < len(crossings); x++ { + if crossings[x] > best { + best = crossings[x] + } + } + return best +} + +// layoutNote renders the reasoning in the terms a reader can check against the +// page: how many columns, cut where, and what was set aside to see them. +func layoutNote(l *ColumnLayout, minGutter float64, g columnGates) string { + var b strings.Builder + switch len(l.Columns) { + case 0: + fmt.Fprintf(&b, "no region holds the %d text runs a column needs", g.minRuns) + case 1: + fmt.Fprintf(&b, "one text column, x=%.0f-%.0f", l.Columns[0].Min, l.Columns[0].Max) + default: + parts := make([]string, len(l.Columns)) + for i := range l.Columns { + parts[i] = fmt.Sprintf("%.0f-%.0f", l.Columns[i].Min, l.Columns[i].Max) + } + fmt.Fprintf(&b, "%d text columns at x=%s, cut at %d gutter(s) at least %.0f wide "+ + "that at most %d runs cross", + len(l.Columns), strings.Join(parts, ", "), len(l.Gutters), minGutter, g.maxCrossings) + } + if l.Spanning > 0 { + fmt.Fprintf(&b, "; %d run(s) span a gutter and belong to no column", l.Spanning) + } + if l.Dropped.Total() > 0 { + fmt.Fprintf(&b, "; ignored %s", dropSummary(&l.Dropped)) + } + return b.String() +} + +func dropSummary(d *DroppedRuns) string { + parts := make([]string, 0, 4) + for _, p := range []struct { + n int + what string + }{ + {d.Blank, "blank"}, + {d.Rotated, "rotated"}, + {d.OffPage, "off-page"}, + {d.Small, "sub-legible"}, + } { + if p.n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", p.n, p.what)) + } + } + if len(parts) == 0 { + return "nothing" + } + return strings.Join(parts, ", ") + " runs" +} diff --git a/internal/doc/columns_fixture_test.go b/internal/doc/columns_fixture_test.go new file mode 100644 index 0000000..5b6f55c --- /dev/null +++ b/internal/doc/columns_fixture_test.go @@ -0,0 +1,347 @@ +package doc_test + +import ( + "context" + "fmt" + "math" + "os" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" +) + +// These run the column pipeline against the real parallel-columns manual, from +// the PDF rather than from coordinates written by hand. +// +// That distinction is the point of the file. DetectColumns and ColumnLanguages +// were both developed against runs typed into a test, and the fixture's own +// per-page entries were produced by a script that no longer exists — so nothing +// until now checked that what poppler actually reports for this document is what +// those tests assumed. The eight pages a human compared against their rendered +// images are the ones asserted here; the rest of the manifest's pages were +// produced by the detector and holding it to those would be circular. See the +// provenance note in the manifest. + +// columnFixture loads the column manual and its ground truth. +func columnFixture(t *testing.T) (manifest *fixture.Manifest, path string) { + t.Helper() + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixture and run the real-document tests", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFInfo, extern.PDFToText, extern.PDFToHTML} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + + m, err := fixture.Load(fixturesDir, "thomas-drybox-amfibia") + if err != nil { + t.Fatalf("load manifest: %v", err) + } + cached, err := m.Fetch(context.Background()) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + return m, cached +} + +// extractColumnFixture reads the document both ways: positioned runs for the +// geometry, and plain text for the printed index whose vocabulary of codes is +// what makes a single-letter tag like "D" usable. +func extractColumnFixture(t *testing.T) (manifest *fixture.Manifest, pages []doc.PageRuns, knownCodes map[string]bool) { + t.Helper() + m, path := columnFixture(t) + + runs, err := doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + return m, runs, doc.IndexCodes(res.BySource[doc.SourceIndex]) +} + +// TestExtractedRunsMatchTheMeasuredDocument checks the coordinate space and the +// volume of text against facts recorded in the manifest, before anything is +// concluded from them. A change in either invalidates every threshold in +// columns.go, all of which are fractions of the page. +func TestExtractedRunsMatchTheMeasuredDocument(t *testing.T) { + m, pages, _ := extractColumnFixture(t) + + if len(pages) != m.Pages { + t.Fatalf("extracted %d pages, manifest says %d", len(pages), m.Pages) + } + if m.PageBox == nil { + t.Fatal("the manifest records no page box to check against") + } + + total := 0 + for i := range pages { + p := &pages[i] + total += len(p.Runs) + // Exact: the box is the PDF's page size scaled by 1.5, a property of the + // output format rather than of the document's typesetting. + if p.Width != m.PageBox.Width || p.Height != m.PageBox.Height { + t.Errorf("page %d box = %gx%g, manifest says %gx%g", + p.No, p.Width, p.Height, m.PageBox.Width, m.PageBox.Height) + break + } + } + + // Tolerant: how a version of poppler breaks a line into runs may shift, and a + // few runs either way changes no conclusion. An order-of-magnitude change means + // the tool is doing something else entirely. + if diff := abs(total - m.TextRuns); diff > m.TextRuns/20 { + t.Errorf("extracted %d text runs, manifest says %d (%d apart)", total, m.TextRuns, diff) + } + t.Logf("%d pages, %d runs, page box %gx%g", len(pages), total, pages[0].Width, pages[0].Height) +} + +// TestColumnGeometryMatchesTheVerifiedPages is the acceptance test for the +// detector on a real document: the eight pages checked by eye, column for column. +func TestColumnGeometryMatchesTheVerifiedPages(t *testing.T) { + m, pages, _ := extractColumnFixture(t) + + byNo := make(map[int]*doc.PageRuns, len(pages)) + for i := range pages { + byNo[pages[i].No] = &pages[i] + } + + verified := m.VerifiedPages() + if len(verified) == 0 { + t.Fatal("the manifest records no human-verified pages, so nothing here is ground truth") + } + + for _, want := range verified { + page, ok := byNo[want.Page] + if !ok { + t.Errorf("page %d is in the manifest but was not extracted", want.Page) + continue + } + layout := doc.DetectColumns(page.Runs, page.Width, page.Height) + + if len(layout.Columns) != want.Columns { + t.Errorf("page %d: found %d columns, the render shows %d — %s", + want.Page, len(layout.Columns), want.Columns, layout.Note) + continue + } + if layout.Spanning != want.Spanning { + t.Errorf("page %d: %d runs span a gutter, manifest says %d", + want.Page, layout.Spanning, want.Spanning) + } + for i, wantCol := range want.Cols { + got := layout.Columns[i] + // Column edges are the extent of the runs assigned to the column, so + // they are exact integers in this space. A tolerance of 1 absorbs the + // rounding in how the manifest recorded them. + if math.Abs(got.Min-float64(wantCol.X0)) > 1 || math.Abs(got.Max-float64(wantCol.X1)) > 1 { + t.Errorf("page %d column %d: x=%.0f-%.0f, manifest says %d-%d", + want.Page, i, got.Min, got.Max, wantCol.X0, wantCol.X1) + } + if got.Runs != wantCol.Runs { + t.Errorf("page %d column %d: %d runs, manifest says %d", + want.Page, i, got.Runs, wantCol.Runs) + } + } + } +} + +// TestColumnLanguagesMatchTheVerifiedPages checks the other half against the same +// eight pages: not only where the columns are, but what language each one is. +// +// This is what could not be checked before. The per-column languages were +// established against runs supplied by hand, so agreement with the manifest here +// is the first evidence that the signal works on what the tool actually reports. +func TestColumnLanguagesMatchTheVerifiedPages(t *testing.T) { + m, pages, knownCodes := extractColumnFixture(t) + + byNo := make(map[int]*doc.PageRuns, len(pages)) + for i := range pages { + byNo[pages[i].No] = &pages[i] + } + + named, total := 0, 0 + for _, want := range m.VerifiedPages() { + page, ok := byNo[want.Page] + if !ok { + continue + } + layout := doc.DetectColumns(page.Runs, page.Width, page.Height) + got := doc.ColumnLanguages(page.Runs, layout.Columns, knownCodes) + if len(got) != len(want.Cols) { + // Geometry is asserted by its own test; this one only reports what it + // could not line up. + t.Logf("page %d: %d columns against %d in the manifest, skipping languages", + want.Page, len(got), len(want.Cols)) + continue + } + + for i, wantCol := range want.Cols { + total++ + if wantCol.Lang == "" { + // The manifest records some columns as unestablished on purpose, + // including one the signal reads wrongly. Naming one of those is not a + // failure here, but claiming the manifest agreed would be. + t.Logf("page %d column %d: manifest records no language (%s); read as %q", + want.Page, i, wantCol.Note, got[i].Lang) + continue + } + if got[i].Lang == "" { + t.Errorf("page %d column %d: no language established, manifest says %s — %s", + want.Page, i, wantCol.Lang, got[i].Note) + continue + } + if !doc.SameLanguage(got[i].Lang, wantCol.Lang) { + t.Errorf("page %d column %d: read as %s, manifest says %s — %s", + want.Page, i, got[i].Lang, wantCol.Lang, got[i].Note) + continue + } + named++ + } + } + + if total == 0 { + t.Fatal("no column was compared") + } + t.Logf("%d of %d columns on the human-verified pages named correctly", named, total) +} + +// TestColumnLanguageAttributionIsRecorded pins which signal names each of the +// document's 169 columns, because the totals alone hide two things. +// +// First, the printed tab and the alphabet cover nearly the same columns here, so a +// change that destroys one of them barely moves the count. Reading the runs with +// Go's plain `,chardata` — which loses every styled run, and the printed tabs are +// styled — still names 166 of 169 columns, one fewer than reading them correctly. +// Only the attribution collapses, from 53 tag-named columns to none. A test on the +// total would have called that healthy. +// +// Second, and not a defect in this file: the 53 is short of what the printed tabs +// could give. columnTag believes a single-letter code only where the document's own +// contents table lists it, and IndexRuns cannot parse this manual's contents page — +// it yields the vocabulary [FAX GA NDE UA VIA Z], of which only UA is a language. +// So every German column's printed "D" is rejected for want of corroboration and +// falls back to its alphabet. Supplying the real vocabulary by hand raises tag +// naming to 79 and drops alphabet naming to 88, which is what the commit that +// introduced ColumnLanguages recorded — measured with a hand-supplied list rather +// than through the assembled pipeline. Fixing the index parser for this shape of +// contents page is separate work; this test records the true current reading so the +// gap is visible instead of inferred. +func TestColumnLanguageAttributionIsRecorded(t *testing.T) { + _, pages, knownCodes := extractColumnFixture(t) + + bySource := make(map[doc.Source]int, 4) + columns, named, conflicts := 0, 0, 0 + for i := range pages { + p := &pages[i] + layout := doc.DetectColumns(p.Runs, p.Width, p.Height) + for _, col := range doc.ColumnLanguages(p.Runs, layout.Columns, knownCodes) { + columns++ + if col.Lang != "" { + named++ + bySource[col.Source]++ + } + if col.Conflict { + conflicts++ + } + } + } + + t.Logf("%d columns, %d named (%d by printed tag, %d by alphabet), %d conflicting", + columns, named, bySource[doc.SourcePageTag], bySource[doc.SourceRepertoire], conflicts) + + if columns != 169 { + t.Errorf("found %d columns across the document, previously measured 169", columns) + } + if named < 165 { + t.Errorf("only %d of %d columns named; 167 were measured", named, columns) + } + // Both signals must keep contributing. Either one reaching zero is the failure + // the totals cannot show, and it is exactly what a regression in run extraction + // or in tag matching looks like. + if got := bySource[doc.SourcePageTag]; got < 40 { + t.Errorf("%d columns named by their printed tab, measured 53 — a collapse here "+ + "is invisible in the total, because the alphabet covers the same columns", got) + } + if got := bySource[doc.SourceRepertoire]; got < 90 { + t.Errorf("%d columns named by their alphabet, measured 114", got) + } +} + +// TestSectionedManualExtractsEveryPage reads the other manual — the sequential one +// — because improving one document by altering the other is the regression the +// design names, and extraction is where that would start. +// +// It deliberately makes no claim about how many columns its pages have. The first +// version of this test asserted they were single-column and failed: 199 of its 560 +// pages read as three columns and only 148 as one. Rendering pages 20 and 100 at +// `pdftoppm -r 108` settled it — both are two side-by-side troubleshooting tables, +// and the regions the detector returns are their cells, correctly located. On page +// 20 it returns only the two wide answer cells, because the narrow question cells +// hold fewer runs than minColumnRuns allows, which is that guard working. +// +// So the assumption was wrong and the code was right. The distribution is recorded +// in the manifest and logged here; what must not change on this manual is its +// language map, and that is asserted where the language map is built. +func TestSectionedManualExtractsEveryPage(t *testing.T) { + m, path := loadFixture(t) + if !extern.Available(extern.PDFToHTML) { + t.Skip("pdftohtml is not installed") + } + + pages, err := doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + if len(pages) != m.Pages { + t.Fatalf("extracted %d pages, manifest says %d", len(pages), m.Pages) + } + if m.PageBox != nil && pages[0].Width != m.PageBox.Width { + t.Errorf("page box width = %g, manifest says %g", pages[0].Width, m.PageBox.Width) + } + + total := 0 + counts := make(map[int]int, 6) + for i := range pages { + p := &pages[i] + total += len(p.Runs) + counts[len(doc.DetectColumns(p.Runs, p.Width, p.Height).Columns)]++ + } + + if diff := abs(total - m.TextRuns); m.TextRuns > 0 && diff > m.TextRuns/20 { + t.Errorf("extracted %d text runs, manifest says %d (%d apart)", total, m.TextRuns, diff) + } + + // Every page must yield its box and its number, whatever its layout. A page + // lost here shifts every later page number, and a page number is what the + // language map is keyed on. + for i := range pages { + if pages[i].No != i+1 { + t.Fatalf("page at index %d is numbered %d; numbering must follow the PDF", + i, pages[i].No) + } + } + + // Recorded, not asserted — see this test's own comment for why a claim here was + // wrong. The manifest holds the same distribution, so a change shows up as a + // disagreement with a written-down measurement rather than as a silent drift. + t.Logf("%d pages, %d runs; column counts: %s", len(pages), total, summarizeCounts(counts)) +} + +func summarizeCounts(counts map[int]int) string { + out := "" + for n := 0; n <= 6; n++ { + if counts[n] == 0 { + continue + } + if out != "" { + out += ", " + } + out += fmt.Sprintf("%d columns: %d pages", n, counts[n]) + } + return out +} diff --git a/internal/doc/columns_test.go b/internal/doc/columns_test.go new file mode 100644 index 0000000..829d689 --- /dev/null +++ b/internal/doc/columns_test.go @@ -0,0 +1,557 @@ +package doc + +import ( + "fmt" + "strings" + "testing" +) + +// The fixtures below are synthetic, but their geometry is not invented: every +// coordinate is taken from the Thomas DryBox Amfibia manual read through +// `pdftohtml -xml`, whose page is 892 by 850 units. Each named page reproduces +// one page of that document, including the traps — the hanging indent, the +// nested table, the spanning heading, the production artifacts and the +// off-page ghosts — so that the properties being asserted are the ones that +// actually broke earlier attempts, and so the suite stays hermetic. +// +// The expected answers are the human-verified column counts and starts +// recorded for those pages, checked against page images. + +const ( + testPageW = 892 + testPageH = 850 + testLineH = 17 // body text height throughout the fixture + testPitch = 21 // baseline-to-baseline + testBodyTop = 65 +) + +// textBlock stacks lines of body text at x, each set to the full measure +// except the last line of every paragraph, which is short — the taper that +// makes a column's right-hand crossing count fall away. +func textBlock(x, width, top float64, lines int) []TextRun { + out := make([]TextRun, 0, lines) + for i := range lines { + w := width + if i%7 == 6 { + w = width * 0.55 + } + out = append(out, TextRun{ + X: x, Y: top + float64(i)*testPitch, + Width: w, Height: testLineH, + Text: fmt.Sprintf("body line %d", i), + }) + } + return out +} + +// hangingList sets a numbered list the way the fixture's parts lists are set: +// a narrow marker at x, its text indented by hang, and every so often a +// sub-heading run at the outer margin set to the full measure. Those +// full-measure lines are the reason the indent is not a gutter, and leaving +// them out is a distinct test below. +func hangingList(x, hang, width, top float64, items, fullMeasureEvery int) []TextRun { + var out []TextRun + y := top + for i := range items { + if fullMeasureEvery > 0 && i%fullMeasureEvery == fullMeasureEvery-1 { + out = append(out, TextRun{ + X: x, Y: y, Width: width, Height: testLineH, + Text: fmt.Sprintf("sub-heading %d, set to the full measure", i), + }) + y += testPitch + continue + } + out = append(out, + TextRun{X: x, Y: y, Width: 12, Height: testLineH, Text: fmt.Sprintf("%d", i+1)}, + TextRun{X: x + hang, Y: y, Width: width - hang, Height: testLineH, + Text: fmt.Sprintf("part name %d", i)}, + ) + y += testPitch + } + return out +} + +// productionSlug is the InDesign filename slug and export timestamp that the +// fixture's placed graphics drag onto 67 of its 68 pages, scaled down with the +// artwork they belong to. Two to six units tall against a body median of 17, +// and this pair is deliberately laid across a gutter. +func productionSlug(x, y float64) []TextRun { + return []TextRun{ + {X: x, Y: y, Width: 59, Height: 4, Text: "29924_Saugerbeschriftungen_DryBoxAmfibia.indd 1"}, + {X: x + 135, Y: y, Width: 18, Height: 2, Text: "16.08.17 13:43"}, + } +} + +// rotatedNote is a marginal note printed on its side. Poppler reports rotated +// text with width 0, at a single x. +func rotatedNote(x, top float64, lines int) []TextRun { + out := make([]TextRun, 0, lines) + for i := range lines { + out = append(out, TextRun{ + X: x, Y: top + float64(i)*testPitch, Width: 0, Height: testLineH, + Text: "text turned on its side", + }) + } + return out +} + +func concat(groups ...[]TextRun) []TextRun { + var out []TextRun + for _, g := range groups { + out = append(out, g...) + } + return out +} + +// wantColumn is an expected column, as x-range. +type wantColumn struct{ min, max float64 } + +func checkColumns(t *testing.T, got ColumnLayout, want []wantColumn) { + t.Helper() + if len(got.Columns) != len(want) { + t.Errorf("got %d columns, want %d\nnote: %s", len(got.Columns), len(want), got.Note) + for i := range got.Columns { + t.Logf(" column %d: x=%.0f-%.0f runs=%d", i+1, + got.Columns[i].Min, got.Columns[i].Max, got.Columns[i].Runs) + } + return + } + for i := range want { + c := &got.Columns[i] + if c.Min != want[i].min || c.Max != want[i].max { + t.Errorf("column %d: got x=%.0f-%.0f, want x=%.0f-%.0f", + i+1, c.Min, c.Max, want[i].min, want[i].max) + } + if c.Note == "" { + t.Errorf("column %d has no note explaining itself", i+1) + } + } +} + +// TestDetectColumnsGroundTruth reproduces the eight pages of the fixture whose +// columns a human verified against the page images. +func TestDetectColumnsGroundTruth(t *testing.T) { + tests := []struct { + name string + runs []TextRun + want []wantColumn + }{ + { + // Contents page: three equal columns, 262 wide, gutters 17 wide. + name: "page 2, three columns with clear gutters", + runs: concat( + textBlock(43, 262, testBodyTop, 34), + textBlock(323, 262, testBodyTop, 34), + textBlock(604, 262, testBodyTop, 34), + productionSlug(300, 780), // laid across the first gutter + ), + want: []wantColumn{{43, 305}, {323, 585}, {604, 866}}, + }, + { + // The facing page: two columns of the same measure, and a blank + // right third that must not be offered as a column. + name: "page 3, two columns and an empty third of the page", + runs: concat( + textBlock(30, 262, testBodyTop, 34), + textBlock(310, 263, testBodyTop, 34), + ), + want: []wantColumn{{30, 292}, {310, 573}}, + }, + { + // Safety notices: two wide columns, 403 units each. Column widths + // vary within one document and this is the widest pair. + name: "page 6, two wide columns", + runs: concat( + textBlock(43, 403, testBodyTop, 33), + textBlock(463, 403, testBodyTop, 19), + ), + want: []wantColumn{{43, 446}, {463, 866}}, + }, + { + // An exploded diagram fills two thirds of the page with numbered + // callouts. None of them is a column; the one real column is the + // parts list on the right, itself hanging-indented. + name: "page 12, one text column beside a diagram of callouts", + runs: concat( + figureCallouts(), + hangingList(604, 30, 262, testBodyTop, 30, 5), + rotatedNote(874, 426, 14), + productionSlug(560, 810), + ), + want: []wantColumn{{604, 866}}, + }, + { + // Parts lists in three languages, each hanging-indented 30 units + // for its numbered markers, with rotated notes down the gutters. + // The gutters here are 18 units, and the indents are 17 — telling + // them apart by width alone is impossible, which is the point. + name: "page 13, three columns each with a hanging indent", + runs: concat( + hangingList(30, 30, 261, testBodyTop, 32, 4), + hangingList(310, 30, 260, testBodyTop, 32, 4), + hangingList(591, 30, 260, testBodyTop, 32, 4), + rotatedNote(300, 549, 6), + rotatedNote(581, 549, 6), + productionSlug(292, 800), + ), + want: []wantColumn{{30, 291}, {310, 570}, {591, 851}}, + }, + { + name: "page 41, three columns", + runs: concat( + textBlock(30, 262, testBodyTop, 26), + textBlock(310, 262, testBodyTop, 26), + textBlock(591, 260, testBodyTop, 26), + ), + want: []wantColumn{{30, 292}, {310, 572}, {591, 851}}, + }, + { + // One language in two columns, with a section heading printed + // across both — the single run that binary coverage lets weld them + // together — and a technical table nested in the left column whose + // value alignment at x=192 is not a column. + name: "page 63, two columns under a spanning heading, with a nested table", + runs: concat( + textBlock(30, 395, testBodyTop, 20), + nestedTable(30, 192, testBodyTop+20*testPitch, 12), + textBlock(451, 400, testBodyTop, 33), + []TextRun{{X: 91, Y: 16, Width: 376, Height: 23, + Text: "Wskazówki dotyczące utylizacji | Obsługa serwisowa | Gwarancja"}}, + ), + want: []wantColumn{{30, 425}, {451, 851}}, + }, + { + // Service addresses. Three columns of unequal width and irregular + // pitch — 271 then 230 — so nothing here can lean on a regular + // grid. The third column holds a second alignment at x=603 that is + // not a fourth column. A banner heading spans all three, and seven + // lines of a superseded address list are parked above the top edge + // of the page where no reader will ever see them. + name: "page 68, three unequal columns with an inner alignment", + runs: concat( + addressBlock(60, 235, testBodyTop+40, 20), + addressBlock(331, 228, testBodyTop+40, 20), + addressBlock(564, 276, testBodyTop+40, 12), + addressBlock(603, 237, testBodyTop+320, 20), + []TextRun{{X: 62, Y: 102, Width: 623, Height: 23, + Text: "Kundendienststellen | Serwis | Служба сервиса"}}, + offPageGhosts(), + rotatedNote(873, 737, 7), + ), + want: []wantColumn{{60, 295}, {331, 559}, {564, 840}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := DetectColumns(tt.runs, testPageW, testPageH) + checkColumns(t, got, tt.want) + }) + } +} + +// figureCallouts scatters numbered labels across an exploded diagram, the way +// the fixture's page 12 does: clusters of one to five short runs, never enough +// of them together to be a column. +func figureCallouts() []TextRun { + at := [][2]float64{ + {144, 195}, {248, 210}, {90, 293}, {144, 318}, {90, 347}, {450, 293}, + {450, 347}, {450, 396}, {450, 436}, {60, 292}, {202, 100}, {467, 135}, + {487, 170}, {300, 209}, {152, 530}, {264, 530}, {202, 605}, {318, 578}, + {469, 599}, {537, 596}, {410, 690}, {487, 669}, {80, 636}, {166, 626}, + {78, 722}, {166, 722}, {247, 745}, {295, 715}, {345, 745}, {392, 745}, + {449, 771}, {541, 731}, {306, 686}, {357, 620}, {434, 650}, + } + out := make([]TextRun, 0, len(at)) + for i, p := range at { + out = append(out, TextRun{ + X: p[0], Y: p[1], Width: 14, Height: testLineH, + Text: fmt.Sprintf("%d", i+1), + }) + } + return out +} + +// nestedTable sets a two-column technical table inside a text column: labels +// at the column's own left edge, values aligned at valueX. The gap between +// them is a real gap in these rows, and it is not a column boundary, because +// the paragraphs above the table cross it. +func nestedTable(labelX, valueX, top float64, rows int) []TextRun { + out := make([]TextRun, 0, rows*2) + for i := range rows { + y := top + float64(i)*testPitch + out = append(out, + TextRun{X: labelX, Y: y, Width: 155, Height: testLineH, + Text: fmt.Sprintf("property %d:", i)}, + TextRun{X: valueX, Y: y, Width: 148, Height: testLineH, + Text: fmt.Sprintf("value %d", i)}, + ) + } + return out +} + +// addressBlock sets short ragged lines, as a postal address is set: only its +// longest line reaches the full measure, so the column has almost no ink at its +// right edge and the gutter beside it is far wider than the nominal gap. This +// is why the fixture's back page has a 13-unit gap between two columns whose +// text is 4 units apart. +func addressBlock(x, width, top float64, lines int) []TextRun { + ratios := []float64{1.0, 0.62, 0.71, 0.55, 0.68, 0.60, 0.74, 0.58} + out := make([]TextRun, 0, lines) + for i := range lines { + out = append(out, TextRun{ + X: x, Y: top + float64(i)*testPitch, + Width: width * ratios[i%len(ratios)], Height: testLineH, + Text: fmt.Sprintf("address line %d", i), + }) + } + return out +} + +// offPageGhosts is a superseded address list left above the top edge of the +// page. It is invisible in print and in the raster, but it is in the text +// layer, and it lies straight across two of the three gutters. +func offPageGhosts() []TextRun { + tops := []float64{-357, -307, -234, -179, -126, -97, -56} + out := make([]TextRun, 0, len(tops)) + for i, y := range tops { + out = append(out, TextRun{ + X: 62, Y: y, Width: 380, Height: 18, + Text: fmt.Sprintf("Kundendienststellen / After Sales Service Addresses %d", i), + }) + } + return out +} + +func TestDetectColumnsEmptyPage(t *testing.T) { + for _, tt := range []struct { + name string + runs []TextRun + }{ + {"no runs at all", nil}, + {"empty slice", []TextRun{}}, + {"only whitespace", []TextRun{ + {X: 30, Y: 65, Width: 200, Height: 17, Text: " "}, + {X: 30, Y: 86, Width: 200, Height: 17, Text: "\n\t"}, + }}, + } { + t.Run(tt.name, func(t *testing.T) { + got := DetectColumns(tt.runs, testPageW, testPageH) + if len(got.Columns) != 0 { + t.Errorf("got %d columns, want none", len(got.Columns)) + } + if got.Note == "" { + t.Error("an empty page must still explain itself") + } + }) + } +} + +func TestDetectColumnsSingleColumn(t *testing.T) { + got := DetectColumns(textBlock(43, 806, testBodyTop, 30), testPageW, testPageH) + checkColumns(t, got, []wantColumn{{43, 849}}) + if len(got.Gutters) != 0 { + t.Errorf("got %d gutters on a one-column page, want none", len(got.Gutters)) + } +} + +// TestDetectColumnsAllCallouts covers a page that is nothing but a diagram. +// Reporting no column is the right answer; naming one would be worse than +// silence, because nothing downstream could tell it was wrong. +func TestDetectColumnsAllCallouts(t *testing.T) { + got := DetectColumns(figureCallouts(), testPageW, testPageH) + if len(got.Columns) != 0 { + t.Errorf("got %d columns from figure callouts alone, want none", len(got.Columns)) + for i := range got.Columns { + t.Logf(" column %d: x=%.0f-%.0f runs=%d", + i+1, got.Columns[i].Min, got.Columns[i].Max, got.Columns[i].Runs) + } + } + if !strings.Contains(got.Note, "text runs a column needs") { + t.Errorf("note should say the density floor was not met, got %q", got.Note) + } +} + +// TestDetectColumnsHangingIndentWithoutFullMeasureLines is the case the +// fixture only narrowly avoids: a list where every single line is indented, so +// nothing crosses the space between the markers and their text. The band is a +// gutter by every test but one, and only its width says otherwise. +func TestDetectColumnsHangingIndentWithoutFullMeasureLines(t *testing.T) { + got := DetectColumns( + hangingList(30, 30, 261, testBodyTop, 32, 0), + testPageW, testPageH) + checkColumns(t, got, []wantColumn{{30, 291}}) +} + +// TestDetectColumnsStripOffBaselineIsNotAbsorbed is the other half of the +// hanging-indent rule. A narrow strip beside a column is folded into it only +// when the two share lines; a vertical run of figure labels that happens to sit +// there does not, and must not stretch the column to reach it. +func TestDetectColumnsStripOffBaselineIsNotAbsorbed(t *testing.T) { + // Labels at half the line pitch, so no two of them land on a body line. + var labels []TextRun + for i := range 12 { + labels = append(labels, TextRun{ + X: 560, Y: testBodyTop + 10 + float64(i)*testPitch*2, Width: 14, Height: testLineH, + Text: fmt.Sprintf("%d", i), + }) + } + got := DetectColumns(concat(textBlock(604, 262, testBodyTop, 30), labels), + testPageW, testPageH) + checkColumns(t, got, []wantColumn{{604, 866}}) +} + +// TestDetectColumnsSubIndentIsNotAColumn isolates the nested-table trap: a +// value alignment 162 units into a column, which a left-alignment peak finder +// cannot tell from a column start. +func TestDetectColumnsSubIndentIsNotAColumn(t *testing.T) { + got := DetectColumns(concat( + textBlock(30, 395, testBodyTop, 20), + nestedTable(30, 192, testBodyTop+20*testPitch, 12), + ), testPageW, testPageH) + checkColumns(t, got, []wantColumn{{30, 425}}) +} + +// TestDetectColumnsSpanningHeadingDoesNotMerge is the failure that binary +// coverage cannot escape: one heading run laid over both columns. +func TestDetectColumnsSpanningHeadingDoesNotMerge(t *testing.T) { + body := concat( + textBlock(30, 395, testBodyTop, 30), + textBlock(451, 400, testBodyTop, 30), + ) + heading := TextRun{X: 91, Y: 16, Width: 376, Height: 23, Text: "one heading over both"} + + before := DetectColumns(body, testPageW, testPageH) + checkColumns(t, before, []wantColumn{{30, 425}, {451, 851}}) + + after := DetectColumns(append(body, heading), testPageW, testPageH) + checkColumns(t, after, []wantColumn{{30, 425}, {451, 851}}) + if after.Spanning != 1 { + t.Errorf("got %d spanning runs, want 1", after.Spanning) + } + if len(after.Gutters) != 1 || after.Gutters[0].Crossings != 1 { + t.Errorf("the gutter should record the one run that crosses it, got %+v", after.Gutters) + } +} + +// TestDetectColumnsProductionArtifactInGutter checks that a slug lying across +// a gutter neither merges the columns nor stretches one into the margin. +func TestDetectColumnsProductionArtifactInGutter(t *testing.T) { + body := concat( + textBlock(43, 262, testBodyTop, 34), + textBlock(323, 262, testBodyTop, 34), + textBlock(604, 262, testBodyTop, 34), + ) + want := []wantColumn{{43, 305}, {323, 585}, {604, 866}} + checkColumns(t, DetectColumns(body, testPageW, testPageH), want) + + // Four placed graphics, each dragging the slug along; one pair sits in + // each gutter and one runs off the right edge of the page. + withSlugs := concat(body, + productionSlug(300, 300), productionSlug(300, 640), + productionSlug(586, 480), productionSlug(805, 854)) + got := DetectColumns(withSlugs, testPageW, testPageH) + checkColumns(t, got, want) + if got.Dropped.Small+got.Dropped.OffPage != 8 { + t.Errorf("got %d artifact runs set aside, want 8 (%+v)", + got.Dropped.Small+got.Dropped.OffPage, got.Dropped) + } +} + +// TestDetectColumnsRepeatedTextIsNotAnArtifact guards the filter that the +// artifacts tempt you into writing. On the fixture's back page "Robert Thomas" +// is printed twelve times, once per service address; a filter keyed on +// repetition within a page would delete 742 of its 769 runs. +func TestDetectColumnsRepeatedTextIsNotAnArtifact(t *testing.T) { + var runs []TextRun + for _, x := range []float64{60, 331, 610} { + for row := range 4 { + top := testBodyTop + float64(row)*140 + for line := range 5 { + runs = append(runs, TextRun{ + X: x, Y: top + float64(line)*testPitch, Width: 230, Height: testLineH, + Text: "Robert Thomas", + }) + } + } + } + got := DetectColumns(runs, testPageW, testPageH) + checkColumns(t, got, []wantColumn{{60, 290}, {331, 561}, {610, 840}}) +} + +// TestDetectColumnsOffPageRunsIgnored covers the ghosts above the page edge. +// Without this the fixture's back page reports two columns instead of three. +func TestDetectColumnsOffPageRunsIgnored(t *testing.T) { + body := concat( + addressBlock(60, 235, testBodyTop+40, 20), + addressBlock(331, 228, testBodyTop+40, 20), + addressBlock(564, 276, testBodyTop+40, 20), + ) + want := []wantColumn{{60, 295}, {331, 559}, {564, 840}} + checkColumns(t, DetectColumns(body, testPageW, testPageH), want) + + got := DetectColumns(concat(body, offPageGhosts()), testPageW, testPageH) + checkColumns(t, got, want) + if got.Dropped.OffPage != 7 { + t.Errorf("got %d off-page runs, want 7", got.Dropped.OffPage) + } +} + +// TestDetectColumnsRotatedRunsDoNotStretchAColumn: a note printed on its side +// in the margin is real text, but it is not a column and must not widen one. +func TestDetectColumnsRotatedRunsDoNotStretchAColumn(t *testing.T) { + got := DetectColumns(concat( + textBlock(604, 262, testBodyTop, 30), + rotatedNote(874, 426, 14), + ), testPageW, testPageH) + checkColumns(t, got, []wantColumn{{604, 866}}) + if got.Dropped.Rotated != 14 { + t.Errorf("got %d rotated runs, want 14", got.Dropped.Rotated) + } +} + +// TestDetectColumnsUnequalWidths states the property the whole exercise exists +// for: nothing may assume a fixed width, count or pitch. +func TestDetectColumnsUnequalWidths(t *testing.T) { + got := DetectColumns(concat( + textBlock(30, 120, testBodyTop, 20), + textBlock(180, 300, testBodyTop, 20), + textBlock(520, 340, testBodyTop, 20), + ), testPageW, testPageH) + checkColumns(t, got, []wantColumn{{30, 150}, {180, 480}, {520, 860}}) +} + +func TestDetectColumnsDegeneratePageWidth(t *testing.T) { + runs := textBlock(30, 200, testBodyTop, 20) + for _, w := range []float64{0, -5, 1, maxProjectionBuckets + 10} { + got := DetectColumns(runs, w, testPageH) + if len(got.Columns) != 0 { + t.Errorf("page width %g: got %d columns, want none", w, len(got.Columns)) + } + if got.Note == "" { + t.Errorf("page width %g: no note explaining the refusal", w) + } + } +} + +// TestDetectColumnsNoteIsCheckable: every number a caller is shown must be one +// they could verify against the page. +func TestDetectColumnsNoteIsCheckable(t *testing.T) { + got := DetectColumns(concat( + textBlock(43, 262, testBodyTop, 34), + textBlock(323, 262, testBodyTop, 34), + productionSlug(300, 300), + ), testPageW, testPageH) + + for _, want := range []string{"2 text columns", "43-305", "323-585", "sub-legible"} { + if !strings.Contains(got.Note, want) { + t.Errorf("note %q does not mention %q", got.Note, want) + } + } + if got.Runs != 68 { + t.Errorf("got %d projected runs, want 68", got.Runs) + } + if total := got.Dropped.Total(); total != 2 { + t.Errorf("got %d dropped runs, want 2", total) + } +} diff --git a/internal/doc/convert.go b/internal/doc/convert.go new file mode 100644 index 0000000..d2cc929 --- /dev/null +++ b/internal/doc/convert.go @@ -0,0 +1,516 @@ +package doc + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/gordon2/manualbox/internal/extern" +) + +// Conversion is everything a reader needs for one household's languages: the +// ordered blocks and the pictures, and nothing belonging to a language nobody +// asked for. +// +// It is the assembly of four pieces that were deliberately built apart — +// [PageRegions] for whose territory a piece of a page is, [RegionBlocks] for the +// reading order inside it, [PageTables] for the cells, [PageFigures] for the +// pictures. Nothing here stores anything: this file is a pure function of the +// document's bytes and the household's languages, the same stance the rest of +// this package takes, and it is what lets the job that calls it run twice. +type Conversion struct { + // Blocks are the readable content in document order, only for the languages in + // scope. Their natural key — page, region left edge, index within the region — + // is assigned by [RegionBlocks] and is unchanged by being collected here. + // + // Page furniture is IN here, flagged rather than removed: a block with + // [Block.Furniture] set is a printed tab, a folio or a running head, and it + // comes last within its region. [Conversion.ContentBlocks] is what a reader and + // an index want; this slice is what a check that must account for every + // character on the page wants. See [Furniture] for why the difference matters. + Blocks []Block + // Figures are the pictures of the pages in scope, in page then reading order. + Figures []ConvertedFigure + // Scope is the intersection of the document's languages with the household's, + // as [Result.ScopeFor] computed it. Carried so that a caller can see what was + // converted, and what else the document holds, without asking twice. + Scope Scope + // Pages are the PDF pages converted, ascending. This is the funnel made + // countable: measured at 26 of the column manual's 68 pages for German, and 22 + // of the sequential manual's 560 for Russian. + Pages []int + // NeutralPages are the pages in Pages that no language owns — the extra scope the + // user opted into, empty when they did not. Carried so that the count a gate + // offered and the count a conversion took can be compared without inferring one + // from the other. + // + // WHAT THESE PAGES CONTRIBUTE IS PICTURES AND NOT TEXT, and that is a decision + // rather than an oversight. Their regions are unnamed, [RegionsBlocks] filters on + // the languages in scope, so they yield no blocks — while `attribute` reaches its + // neutral arm and hands every picture on them to every language in scope. That is + // the whole of the reported problem: the sequential manual's page 5 is a plate of + // 31 drawings that 31 places in its content pages point at, and its own text is + // the labels A-1, C-5 and F-9, which growToLabels has already drawn INSIDE the + // crops a reader is served. + // + // Serving their text as well would mean storing blocks with no language and + // teaching BlocksByLang to union those into every language's answer — a read-path + // change touching every document already converted, for three contents pages that + // need the tab-stop parser conversion.md records as unbuilt anyway. Left undone on + // purpose, and stated here rather than left to be discovered. + NeutralPages []int + // Furniture is what the document repeats in the same place page after page, and + // why each piece was judged so. Carried rather than only applied so that the two + // clauses can be counted apart by a report and by a test — nil when nothing was + // converted. See [Furniture]. + Furniture *Furniture + // Callouts is which runs a figure's labels claimed, and therefore which runs left + // the block flow. Carried for the same reason Furniture is: so a report and a test + // can count what the rule claimed without walking the blocks. See [Callouts]. + Callouts *Callouts + + // Notes say what could not be done, in the caller's terms. A missing pdftocairo + // costs the cells and the pictures and not the text, so it is reported here + // rather than returned as an error — the same stance [ExtractRules] takes one + // level down, and the reason a document is never failed for want of an optional + // tool. + Notes []string +} + +// ConvertedFigure is one picture together with the languages it belongs to. +// +// The languages are plural because of the rule docs/design/conversion.md settles: +// a picture that belongs to no language belongs to every language. A diagram +// spanning the full measure of a parallel-columns page is nobody's column and +// everybody's picture, and a reader of one language must not lose it for having +// no language of its own. +type ConvertedFigure struct { + Figure + + // Langs are the household languages this figure is part of, as base tags, + // sorted. One entry for a figure sitting inside a language's region; every + // language in scope for one sitting in none. + Langs []string + // RegionX0 is the left edge of the region the figure sits in, matching + // [Block.RegionX0] so that a figure can be placed among the blocks of the same + // region. Meaningless when Neutral. + RegionX0 float64 + // Neutral reports that the figure sits inside no region of its page. That is a + // property of the picture and not of the household, so it is recorded rather + // than inferred from len(Langs) — a household reading one language cannot + // otherwise tell a neutral figure from one of its own. + Neutral bool +} + +// ConvertOptions is the scope beyond the household's languages. +// +// A STRUCT WHOSE ZERO VALUE IS TODAY'S BEHAVIOUR, deliberately. Every scope +// decision this pipeline makes has to be one the gate displayed, and the safe +// default for a caller that has not been taught about a new axis is to convert +// exactly what it converted before. A bare bool parameter would have the same +// property; the struct is here so the next axis does not re-break every call site. +type ConvertOptions struct { + // IncludeNeutralPages converts the pages no language owns as well. + // + // Opt-in and never inferred. The set is [Result.NeutralPages] — computed from the + // stored region map, never sent by a caller — so this is a yes or a no to a set + // the server worked out, which is what lets the thing approved stay the thing the + // gate showed. See [ingest.Service.Approve]. + IncludeNeutralPages bool +} + +// figureRegionSlack is how far a figure may reach past a region's edge and still +// count as inside it, in the 1.5-scaled space. The same one unit [RegionBlocks] +// allows a block, and for the same reason: a region's box comes from the extent +// of the runs assigned to it, so an exact comparison is comparing a drawing +// against a measurement of the text beside it. +const figureRegionSlack = 1.0 + +// Convert reads a document for one household's languages. +// +// path is the document, res the probe's result — which already carries the +// regions, so this does not divide any page again — and household the languages +// the user reads. The result is the blocks in document order plus the figures, +// for those languages only. +// +// Three rules govern it, and all three are decisions rather than mechanics: +// +// 1. Only the languages in scope are converted, which is the funnel. [ScopeFor] +// decides which those are; nothing here re-implements the matching, so a +// household reading pt gets a pt-BR section here exactly as the gate promised +// it would. +// 2. A picture inside a language's region belongs to that language. A picture +// inside none belongs to every language in scope. See [ConvertedFigure]. +// 3. Nothing assumes one language's section looks like another's. The pages +// converted are the pages the household's own regions occupy, whatever is on +// them: measured on the sequential manual, Russian is 22 pages carrying 81 +// figures where 32 other languages are 16 pages carrying none, and any +// per-section shape would have hidden that section's illustrated half. +// +// Cost is bounded to the pages in scope. Reading positioned text is one +// pdftohtml over the document, which the probe has already paid for once and +// this pays again because [Result] deliberately does not carry 3.8 MB of +// coordinates. Everything after that is per page and only for a page in scope: +// the cells and the pictures are two pdftocairo spawns each, which +// docs/design/conversion.md records as the accepted price of reading a page +// twice. +// +// A document is not failed for want of an optional tool. Losing pdftocairo loses +// the cells and the pictures, is written into Notes, and leaves the text intact. +func Convert(ctx context.Context, path string, res *Result, household []string, + opts ConvertOptions) (*Conversion, error) { + if res == nil { + return nil, errors.New("doc: Convert needs the probe's result") + } + withNeutral := opts.IncludeNeutralPages + + conv := &Conversion{Scope: res.ScopeFor(household)} + + // Keyed on base language, the key ScopeFor, RegionChars and RegionsBlocks all + // use. Building it from the scope rather than from the household is what makes + // the matching single-sourced: the scope has already resolved pt-BR against pt. + inScope := make(map[string]bool, len(conv.Scope.Languages)) + for _, l := range conv.Scope.Languages { + if base := BaseLanguage(l.Lang); base != "" { + inScope[base] = true + } + } + if len(inScope) == 0 { + conv.note("none of the household's languages appear in this document") + return conv, nil + } + + if len(res.Regions) == 0 { + reason := res.RegionNote + if reason == "" { + reason = "the probe established no language regions" + } + conv.note("nothing could be converted: " + reason) + return conv, nil + } + + // The pages the household's own regions sit on, and no others. + want := make(map[int]bool, len(res.Regions)) + for i := range res.Regions { + if inScope[BaseLanguage(res.Regions[i].Lang)] { + want[res.Regions[i].Page] = true + } + } + + // Plus, when the user asked for them at the gate, the pages no language owns. + // + // This is the ONLY line that decides it, and everything downstream is already + // right: a figure on one of these pages sits inside an unnamed region or none, so + // `attribute` reaches its neutral arm and hands the picture to every language in + // scope — rule 2, unchanged, and the rule that makes the diagram reachable from + // German AND from Russian without either of them owning page 5. + // + // No language is invented for them anywhere. RegionsBlocks below still filters on + // inScope, so an unnamed region contributes no blocks and nothing acquires a + // language it does not have; see [Conversion.NeutralPages] for what that costs and + // why it is the right half to ship. + if withNeutral { + for _, p := range res.NeutralPages() { + want[p] = true + conv.NeutralPages = append(conv.NeutralPages, p) + } + } + + pages, err := ExtractRuns(ctx, path) + if err != nil { + return nil, fmt.Errorf("doc: convert: %w", err) + } + + // Every region of a page in scope, including the ones out of it. A figure in + // the Polish column of a German page belongs to Polish and must be dropped, and + // that can only be seen by holding the Polish region against it — filtering the + // regions first would make every neighbour's picture look language-neutral and + // hand all of them to the reader. + byPage := make(map[int][]int, len(want)) + for i := range res.Regions { + if want[res.Regions[i].Page] { + byPage[res.Regions[i].Page] = append(byPage[res.Regions[i].Page], i) + } + } + + scopeLangs := sortedKeys(inScope) + tables := make(map[int][]RuledTable, len(want)) + // Filled in as the figures are found, and consulted when the blocks are built, so + // that a label a reader draws beside the picture is not printed in the prose too. + callouts := &Callouts{} + var ruleFailures, inkFailures []int + rules, ink := true, true + + for i := range pages { + p := &pages[i] + if !want[p.No] { + continue + } + // A cancelled job stops here rather than in poppler. Without this every + // remaining page still spawns, fails on the dead context, and comes back as a + // note saying its drawings could not be read — which reports a cancellation + // as a defect in the document. + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("doc: convert: %w", err) + } + conv.Pages = append(conv.Pages, p.No) + + if rules { + found, err := PageTables(ctx, path, p) + switch { + case err == nil: + if len(found) > 0 { + tables[p.No] = found + } + case errors.Is(err, extern.ErrNotFound): + // The tool is absent, so every remaining page would fail the same way. + // Stop spawning and say so once. + rules = false + conv.note("no table cells: " + err.Error()) + default: + ruleFailures = append(ruleFailures, p.No) + } + } + + if ink { + found, err := PageFigures(ctx, path, p) + switch { + case err == nil: + for j := range found { + fig, ok := attribute(&found[j], res.Regions, byPage[p.No], inScope, scopeLangs) + if !ok { + // The funnel dropped it: a picture inside a region in a language + // the household does not read. Its labels must NOT leave the + // block flow, because no figure a reader is served carries them. + continue + } + conv.Figures = append(conv.Figures, fig) + callouts.Mark(p.No, &found[j]) + } + case errors.Is(err, extern.ErrNotFound): + ink = false + conv.note("no pictures: " + err.Error()) + default: + inkFailures = append(inkFailures, p.No) + } + } + } + + // The furniture pass, and this is the only place in the pipeline that can run + // it: it needs every page of a language's section at once, which is what this + // function holds and what [RegionBlocks] by construction never does. Free — one + // walk over runs already in memory, no tool spawned. See [FindFurniture]. + fur := FindFurniture(pages, res.Regions, inScope, FoliosOf(res.Pages)) + conv.Furniture = fur + // The callouts were collected as the figures were found above, which is the only + // order that works: a label is claimed from a drawing's geometry, so the figure + // pass has to have run before the blocks are built. See [Callouts]. + conv.Callouts = callouts + conv.Blocks = RegionsBlocks(pages, res.Regions, inScope, tables, fur, + WithCallouts(callouts)) + + // One note per kind rather than one per page: a document whose pdftocairo dies + // on forty pages should say so in a line a user can read. + if len(ruleFailures) > 0 { + conv.note(fmt.Sprintf("the ruled lines of %d page(s) could not be read, so their "+ + "tables read as lines of text: %s", len(ruleFailures), pageList(ruleFailures))) + } + if len(inkFailures) > 0 { + conv.note(fmt.Sprintf("the drawings of %d page(s) could not be read, so their "+ + "pictures are missing: %s", len(inkFailures), pageList(inkFailures))) + } + return conv, nil +} + +// attribute decides which languages a figure belongs to. +// +// A figure is inside a region when its horizontal extent lies within the +// region's box; a region spans the page's full height, so there is no vertical +// question to ask. A figure that lies inside none of its page's regions — which +// includes one straddling two of them — belongs to every language in scope, +// which is rule 2 of [Convert]. +// +// The extent asked about is [Figure.DrawnExtent], never the rendered crop. Those differ once [doc.growToLabels] has taken a label in, and using the +// crop would let a drawing grown sideways onto its label reach out of its own +// column and be served to every household — the one failure the funnel may not +// have. A picture's language is a property of the picture, not of how much of the +// page around it was rendered. +// +// The false return is the third case, and it is the funnel: a figure inside a +// region in a language the household does not read is that language's picture, +// and is dropped exactly as its text is. +func attribute(f *Figure, regions []Region, onPage []int, inScope map[string]bool, + scopeLangs []string) (ConvertedFigure, bool) { + for _, i := range onPage { + r := ®ions[i] + drawn := f.DrawnExtent() + if drawn.X0 < r.X0-figureRegionSlack || drawn.X1 > r.X1+figureRegionSlack { + continue + } + base := BaseLanguage(r.Lang) + if base == "" { + // A region no signal could name is not a language, so a figure inside it + // has none either and falls through to the neutral rule. That is the + // stance everywhere else here: an unnamed region is a reportable state, + // never a sixth language. + continue + } + if !inScope[base] { + return ConvertedFigure{}, false + } + return ConvertedFigure{Figure: *f, Langs: []string{base}, RegionX0: r.X0}, true + } + // A copy per figure rather than the one slice shared by all of them. Langs is an + // exported field on a value type, and handing every neutral figure the same + // backing array makes one caller's append visible in the others. + langs := make([]string, len(scopeLangs)) + copy(langs, scopeLangs) + return ConvertedFigure{Figure: *f, Langs: langs, Neutral: true}, true +} + +// FiguresFor returns the figures belonging to one language, which is every +// figure of its own regions plus every language-neutral one. +func (c *Conversion) FiguresFor(lang string) []ConvertedFigure { + base := BaseLanguage(lang) + var out []ConvertedFigure + for i := range c.Figures { + for _, l := range c.Figures[i].Langs { + if l == base { + out = append(out, c.Figures[i]) + break + } + } + } + return out +} + +// BlocksFor returns the blocks of one language, in document order, furniture +// included. A caller wanting what a person reads intersects it with +// [Conversion.ContentBlocks] — or, more simply, skips the blocks whose +// [Block.Furniture] is set, which is what that method does. +func (c *Conversion) BlocksFor(lang string) []Block { + base := BaseLanguage(lang) + var out []Block + for i := range c.Blocks { + if BaseLanguage(c.Blocks[i].Lang) == base { + out = append(out, c.Blocks[i]) + } + } + return out +} + +// ContentBlocks returns the blocks a person reads in the prose: everything except +// the page furniture and the figure callout labels. +// +// This is the slice a reader renders and an index indexes, and it exists as a +// method rather than as a filter each caller writes so that "what is content" has +// one answer. [Conversion.Blocks] keeps both, because a check comparing a +// conversion against a second extraction of the same page must be able to account +// for every character the page prints. +// +// A callout label is excluded for a different reason from furniture, and the +// difference matters to anyone reading this list to see what a reader gets. Furniture +// is not shown at all. A label IS shown — [Figure.Labels] carries it and the reader +// draws it against the picture — so it is not missing from the reader, only from the +// prose. See [Callouts]. +func (c *Conversion) ContentBlocks() []Block { + out := make([]Block, 0, len(c.Blocks)) + for i := range c.Blocks { + if !c.Blocks[i].Furniture && !c.Blocks[i].Callout { + out = append(out, c.Blocks[i]) + } + } + return out +} + +// CalloutBlocks returns only the figure callout labels, in document order. The +// counterpart of [Conversion.FurnitureBlocks], and what a report counts. +func (c *Conversion) CalloutBlocks() []Block { + var out []Block + for i := range c.Blocks { + if c.Blocks[i].Callout { + out = append(out, c.Blocks[i]) + } + } + return out +} + +// FurnitureBlocks returns only the page furniture, in document order. The +// complement of [Conversion.ContentBlocks], and what a report counts. +func (c *Conversion) FurnitureBlocks() []Block { + var out []Block + for i := range c.Blocks { + if c.Blocks[i].Furniture { + out = append(out, c.Blocks[i]) + } + } + return out +} + +// Summary describes a conversion in one line, for logs and for a test that wants +// the shape rather than every row. It carries no filename and no text, only +// counts, so it is safe in a log line — the same stance [Result.String] takes. +func (c *Conversion) Summary() string { + langs := make([]string, 0, len(c.Scope.Languages)) + for _, l := range c.Scope.Languages { + langs = append(langs, l.Lang) + } + neutral := 0 + for i := range c.Figures { + if c.Figures[i].Neutral { + neutral++ + } + } + // Content first and the two excluded classes named separately, because the same + // document converted before and after either pass reports the same total and a + // summary that only totalled would hide the whole change. + fur, callouts := 0, 0 + for i := range c.Blocks { + switch { + case c.Blocks[i].Furniture: + fur++ + case c.Blocks[i].Callout: + callouts++ + } + } + labels := 0 + for i := range c.Figures { + labels += len(c.Figures[i].Labels) + } + s := fmt.Sprintf("%d blocks (%d furniture, %d callout), %d figures (%d language-neutral, "+ + "%d labels) over %d of %d pages, %s", + len(c.Blocks)-fur-callouts, fur, callouts, len(c.Figures), neutral, labels, + len(c.Pages), c.Scope.TotalPages, strings.Join(langs, "+")) + if len(c.Notes) > 0 { + s += fmt.Sprintf(", %d note(s)", len(c.Notes)) + } + return s +} + +func (c *Conversion) note(s string) { c.Notes = append(c.Notes, s) } + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// pageList renders a handful of page numbers, because a note naming forty pages +// is not a note anybody reads. +func pageList(pages []int) string { + const show = 8 + parts := make([]string, 0, show+1) + for i, p := range pages { + if i == show { + parts = append(parts, fmt.Sprintf("and %d more", len(pages)-show)) + break + } + parts = append(parts, fmt.Sprint(p)) + } + return strings.Join(parts, ", ") +} diff --git a/internal/doc/convert_fixture_test.go b/internal/doc/convert_fixture_test.go new file mode 100644 index 0000000..27309fd --- /dev/null +++ b/internal/doc/convert_fixture_test.go @@ -0,0 +1,387 @@ +package doc_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/gordon2/manualbox/internal/doc" +) + +// The acceptance docs/design/conversion.md asks for, applied to the whole +// assembly rather than to any one of its four pieces: the column manual's German +// as readable content in reading order with no Polish, Russian, Ukrainian or +// Kazakh text in it, and the sequential manual's Russian with the illustrated +// maintenance pages the other thirty-two languages do not have. +// +// Everything asserted here was also read off a 108 dpi render while it was +// written. The pages compared are 14 and 57 of the column manual and 533 of the +// sequential one, and what each shows is recorded at its assertion. + +// convertFixture probes a fixture and converts it for one household. +func convertFixture(t *testing.T, name string, langs ...string) *doc.Conversion { + t.Helper() + var path string + if name == "thomas-drybox-amfibia" { + _, path = columnFixture(t) + } else { + _, path = loadFixture(t) + } + + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if res.RegionNote != "" { + t.Skipf("no regions were produced: %s", res.RegionNote) + } + + start := time.Now() + conv, err := doc.Convert(context.Background(), path, res, langs, doc.ConvertOptions{}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + t.Logf("%v converting %v: %s", time.Since(start).Round(time.Millisecond), langs, conv.Summary()) + for _, n := range conv.Notes { + t.Logf(" note: %s", n) + } + return conv +} + +// TestConvertTheColumnManualForGerman is the acceptance criterion, and the +// negative it insists on. +// +// Compared against a 108 dpi render of page 14, which is a three-column spread: +// a column of two photographs of the machine on the left at x=43-288, German +// text in the middle at x=323-584, Polish on the right at x=604-862. And against +// page 57, whose printed troubleshooting tables have "Allgemein (alle +// Funktionen)" as a full-width banner cell and question/answer pairs under it. +func TestConvertTheColumnManualForGerman(t *testing.T) { + conv := convertFixture(t, "thomas-drybox-amfibia", "de") + + // The funnel, counted. 26 of 68 pages is what the gate charged this household + // for, and conversion must not read a page more. + if len(conv.Pages) != 26 { + t.Errorf("converted %d pages, the gate reports German on 26 of this manual's 68: %v", + len(conv.Pages), conv.Pages) + } + if len(conv.Blocks) == 0 { + t.Fatal("the German regions produced no blocks at all") + } + + // The negative, exactly as the contract words it: no page may contribute text + // from a language that was not asked for. One Cyrillic letter means the + // Russian, Ukrainian or Kazakh column was read. + leaked := 0 + for i := range conv.Blocks { + b := &conv.Blocks[i] + if n := scriptsIn(b.Text)["cyrillic"]; n > 0 { + leaked++ + if leaked <= 5 { + t.Errorf("page %d block %d holds %d Cyrillic letters: %q", + b.Page, b.Index, n, truncate(b.Text, 120)) + } + } + if strings.ContainsAny(b.Text, polishOnlyLetters) { + leaked++ + if leaked <= 5 { + t.Errorf("page %d block %d holds Polish-only letters: %q", + b.Page, b.Index, truncate(b.Text, 120)) + } + } + if doc.BaseLanguage(b.Lang) != "de" { + t.Errorf("page %d block %d is labelled %q", b.Page, b.Index, b.Lang) + } + } + if leaked > 0 { + t.Errorf("%d of %d German blocks carry another language's letters", leaked, len(conv.Blocks)) + } + + // German has to be there, or forbidding the other four passes on an empty + // result. Umlauts and eszett are what German writes and none of the other four + // does. + umlauts := 0 + for i := range conv.Blocks { + umlauts += strings.Count(conv.Blocks[i].Text, "ä") + strings.Count(conv.Blocks[i].Text, "ö") + + strings.Count(conv.Blocks[i].Text, "ü") + strings.Count(conv.Blocks[i].Text, "ß") + } + if umlauts < 200 { + t.Errorf("%d umlauts and eszetts over %d blocks; this is 26 pages of German", + umlauts, len(conv.Blocks)) + } + + // Headings as headings. Page 62 was rendered and read: four of them. + headings := map[string]bool{} + for i := range conv.Blocks { + if conv.Blocks[i].Kind == doc.BlockHeading { + headings[conv.Blocks[i].Text] = true + } + } + for _, want := range []string{"Hinweis zur Entsorgung", "Kundendienst", "Technische Daten", "Garantie"} { + if !headings[want] { + t.Errorf("%q is a heading on the render of page 62 and is not one here", want) + } + } + + // The troubleshooting tables as tables with the right cells. Page 57's render + // shows two of them; the banner cell spans both columns and each question has + // its answer beside it. + cells := map[string]*doc.Block{} + for i := range conv.Blocks { + if conv.Blocks[i].Kind == doc.BlockTable && conv.Blocks[i].Page == 57 { + cells[conv.Blocks[i].Text] = &conv.Blocks[i] + } + } + if len(cells) < 20 { + t.Errorf("page 57 came back with %d distinct table cells; the render prints 29 and "+ + "conversion.md records 25 recovered", len(cells)) + } + banner := cells["Allgemein (alle Funktionen)"] + if banner == nil { + t.Error("page 57's banner cell is missing; on the render it is a heading printed across the whole table") + } else if !strings.Contains(banner.Note, "spanning 2") { + t.Errorf("the banner cell's note is %q; on the render it spans both columns", banner.Note) + } + answer := cells["Das Gerät lässt sich nicht in Betrieb nehmen"] + if answer == nil { + t.Error("page 57's first question cell is missing") + } + + // The pictures. 53 over this scope, and 51 of them from the shared picture + // column — which the render of page 14 shows is a column of photographs + // belonging to neither text column. + // + // It was 40 before the clip was read. The 13 extra are drawings that had been + // merged into the one above them: page 42 now returns its four printed panels + // and page 22 its three, both checked against renders. Page 14 still returns + // exactly its two photographs, which is the assertion below and what says the + // rise is a split rather than furniture getting through. + if len(conv.Figures) != 53 { + t.Errorf("%d figures, measured at 53 for this scope", len(conv.Figures)) + } + p14 := 0 + for i := range conv.Figures { + f := &conv.Figures[i] + if f.Page == 14 { + p14++ + if !f.Neutral { + t.Errorf("page 14 figure %d at x=%.0f-%.0f is not language-neutral; the render "+ + "shows the picture column belongs to neither text column", f.Index, f.Rect.X0, f.Rect.X1) + } + if f.Rect.X1 > 323 { + t.Errorf("page 14 figure %d reaches x=%.0f, into the German column at 323", + f.Index, f.Rect.X1) + } + } + if len(f.PNG) == 0 { + t.Errorf("page %d figure %d carries no bytes", f.Page, f.Index) + } + } + if p14 != 2 { + t.Errorf("page 14 produced %d figures; the render shows two photographs of the machine", p14) + } +} + +// TestConvertTheSequentialManualForRussian is the case the user asked to be sure +// of: a language whose section is unlike the others must come back whole. +// +// Russian occupies 22 pages of this manual where 32 other languages get 16, and +// the extra is an illustrated maintenance section. Page 533 was rendered and +// read: prose about the charging contacts, the waste tank and the vents, and nine +// line drawings — the count here said eight for a while, from a box overlay rather +// than from the print. It also carries "Плановое обслуживание" at the top, which +// this test called page 533's heading and which the render says is the running +// head of the section that starts on page 528. +// +// The same document is then converted for German, which is the comparison that +// makes the point: 16 pages and not one picture, from the same code, on the same +// bytes. Anything assuming the sections are alike passes one of these two and +// fails the other. +func TestConvertTheSequentialManualForRussian(t *testing.T) { + conv := convertFixture(t, "dreame-l40-ultra", "ru") + + if len(conv.Pages) != 22 || conv.Pages[0] != 517 || conv.Pages[21] != 538 { + t.Errorf("converted %d pages %v; the Russian section is 517-538, 22 pages where 32 "+ + "other languages get 16", len(conv.Pages), conv.Pages) + } + + // The illustrated maintenance pages, present with their figures. A conversion + // returning 16 uniform pages of prose has silently lost them. + byPage := map[int]int{} + for i := range conv.Figures { + byPage[conv.Figures[i].Page]++ + } + // Two of these four numbers were wrong, and they were wrong in the way a count + // taken off a box overlay is wrong: they counted boxes and called them drawings. + // Both pages were re-rendered and re-read once candidate boxes that overlap were + // merged. + // + // Page 525 prints FOUR drawings — the base station with its compartment open, + // the bottle being poured, the station again, and the water tank on the right — + // and returned eight, because each station had clustered in three pieces. Page + // 533 prints NINE and returned eight, of which one was a scrap: a 48x36 patch of + // the station's ribbed panel, wholly inside the station's own box, cropped and + // served as a picture of its own. It now returns seven. The two that are still + // missing are the small tank drawings at the top right, which no merge can + // recover — they are under the ink guard, and that is the honest state. + // + // Pages 522 and 524 are here because 524 is the page the fault was reported on. + // It prints four drawings — the robot from above with its side-brush inset, the + // robot's underside with the mop pads and the hand holding the pin, the robot on + // its base station, and the robot with the QR code beside the phone — and + // returned six, one of which was that hand, cropped out of the drawing behind it + // and served as a picture. Page 522 prints nine and returned thirteen. + // Page 529 is 7 where it was 8: one of its crops lay wholly inside another once the + // band widened them -- figure 7 was figure 4's band with the top cut off, both + // having claimed the numbered step underneath -- and doc.ServedFigures drops it, + // moving its labels onto the crop that swallowed it. The page still prints what it + // printed; a reader is served one picture instead of a picture and its own lower + // half. + for _, c := range []struct{ page, figures int }{ + {522, 9}, {524, 4}, {525, 4}, {529, 7}, {531, 7}, {533, 7}, + } { + if byPage[c.page] != c.figures { + t.Errorf("page %d came back with %d figures, %d were counted on the render", + c.page, byPage[c.page], c.figures) + } + } + // 63, where it was 81 before the clip was read, 84 after, and 65 until the crop + // became the band the page prints. The two that went are crops that lay wholly + // inside another once the band widened them -- page 529's figure 7 was figure 4's + // band with the top cut off -- and [doc.ServedFigures] drops them, moving their + // labels onto the crop that swallowed them. Read the sequence 81 -> 84 -> 65 -> 63. + if len(conv.Figures) != 63 { + t.Errorf("%d figures over the Russian section, measured at 63", len(conv.Figures)) + } + + // Page 533's prose, from the render, and the title where the section it belongs + // to actually begins. + // + // `Плановое обслуживание` was asserted here as page 533's heading. Re-rendered at + // 108 dpi: it heads pages 528 to 533 identically, and each of those pages carries + // its own grey-pill sub-heading under it — "Компоненты" on 528, "Основание + // промывочной панели" on 529, "Зарядные контакты и область сигнала" on 533. So it + // is the section's running head, printed once as a title on 528 and repeated + // five times; the furniture pass's clause 3 keeps the first and takes the rest. + // What page 533 heads is not a section of its own. + var head, note, title bool + for i := range conv.Blocks { + b := &conv.Blocks[i] + if !strings.Contains(b.Text, "Плановое обслуживание") { + if b.Page == 533 && strings.Contains(b.Text, "Поплавковый уровнемер") { + note = true + } + continue + } + switch { + case b.Page == 528 && b.Kind == doc.BlockHeading: + title = true + case b.Page > 528 && b.Page <= 533 && b.Furniture: + head = head || b.Page == 533 + default: + t.Errorf("page %d serves %q as a %s (furniture=%v); the title belongs to page 528 "+ + "and every later printing of it is a running head", b.Page, b.Text, b.Kind, b.Furniture) + } + } + if !title { + t.Error("page 528's heading Плановое обслуживание did not come back as a heading; " + + "that is where the section starts and the title must survive there") + } + if !head { + t.Error("page 533 still serves Плановое обслуживание as content; it is a running head there") + } + if !note { + t.Error("page 533's note about the float gauge is missing; the render prints it under the tank drawings") + } + + // Every figure of this manual sits inside a whole-page region, so none of them + // is language-neutral. Stated because it is the opposite of the column manual + // and both must hold from the same rule. + for i := range conv.Figures { + if conv.Figures[i].Neutral { + t.Errorf("page %d figure %d is language-neutral; every page of this manual is one "+ + "language edge to edge", conv.Figures[i].Page, conv.Figures[i].Index) + } + } + + // The comparison. German is 16 pages of the same document and no pictures at + // all, and that is a fact about the document rather than a failure. + german := convertFixture(t, "dreame-l40-ultra", "de") + if len(german.Pages) != 16 { + t.Errorf("German came back as %d pages; the manifest records 16", len(german.Pages)) + } + if len(german.Figures) != 0 { + t.Errorf("German came back with %d figures; conversion.md measures none outside the "+ + "Russian and Japanese sections", len(german.Figures)) + } + if len(german.Blocks) == 0 { + t.Error("German came back with no blocks") + } +} + +// TestALanguageNeutralFigureIsInEveryLanguagesConversion is rule 2 on the real +// document, checked the way the brief asks: the same document converted for two +// different languages, and a figure belonging to no region present in both. +// +// Page 14's two photographs are that figure. They are the same bytes both times, +// asserted on the digest rather than on a count, because two conversions each +// finding "a figure on page 14" would pass a count while returning different +// pictures. +func TestALanguageNeutralFigureIsInEveryLanguagesConversion(t *testing.T) { + german := convertFixture(t, "thomas-drybox-amfibia", "de") + polish := convertFixture(t, "thomas-drybox-amfibia", "pl") + + digests := func(c *doc.Conversion, page int) []string { + var out []string + for i := range c.Figures { + f := &c.Figures[i] + if f.Page == page && f.Neutral { + out = append(out, f.Digest) + } + } + return out + } + + de, pl := digests(german, 14), digests(polish, 14) + if len(de) != 2 || len(pl) != 2 { + t.Fatalf("page 14 gave German %d language-neutral figures and Polish %d; the render "+ + "shows two photographs in a column belonging to neither", len(de), len(pl)) + } + for i := range de { + if de[i] != pl[i] { + t.Errorf("figure %d of page 14 is %s for German and %s for Polish; a picture "+ + "belonging to no language must be the same picture in every language", + i, de[i][:12], pl[i][:12]) + } + } + + // And the count of them across the whole scope, which is the same for both + // because it is a property of the document and not of the household. + neutral := func(c *doc.Conversion) int { + n := 0 + for i := range c.Figures { + if c.Figures[i].Neutral { + n++ + } + } + return n + } + if neutral(german) != neutral(polish) { + t.Errorf("German sees %d language-neutral figures and Polish %d", neutral(german), neutral(polish)) + } + + // FiguresFor is the accessor a reader screen will use, and it must answer for + // both languages at once when a household reads both. + both := convertFixture(t, "thomas-drybox-amfibia", "de", "pl") + if len(both.FiguresFor("de")) < len(de) || len(both.FiguresFor("pl")) < len(pl) { + t.Errorf("a household reading both languages gets %d German and %d Polish figures", + len(both.FiguresFor("de")), len(both.FiguresFor("pl"))) + } + for i := range both.Figures { + if f := &both.Figures[i]; f.Neutral && len(f.Langs) != 2 { + t.Errorf("page %d figure %d is language-neutral but belongs to %v; it belongs to "+ + "every language in scope", f.Page, f.Index, f.Langs) + } + } +} diff --git a/internal/doc/convert_internal_test.go b/internal/doc/convert_internal_test.go new file mode 100644 index 0000000..e8f4a33 --- /dev/null +++ b/internal/doc/convert_internal_test.go @@ -0,0 +1,237 @@ +package doc + +import ( + "reflect" + "strings" + "testing" +) + +// The three rules of Convert, stated against geometry alone. They are the whole +// decision content of this file — everything else is spawning poppler — so they +// are tested where no poppler is needed and no fixture has to be downloaded. +// +// The coordinates are the real ones. Page 14 of the column manual is a +// three-column spread whose leftmost column is pictures: German at x=323-584, +// Polish at x=604-862, and two photographs of the machine at x=43-288 belonging +// to neither. Verified against a 108 dpi render of that page. + +// page14 is that page's regions. +func page14() []Region { + return []Region{ + {Page: 14, X0: 323, X1: 584, Code: "DE", Lang: "de"}, + {Page: 14, X0: 604, X1: 862, Code: "PL", Lang: "pl"}, + } +} + +func figureAt(x0, x1 float64) *Figure { + return &Figure{Page: 14, Rect: CellRect{X0: x0, Y0: 241, X1: x1, Y1: 431}} +} + +func attributeOn(f *Figure, regions []Region, scope ...string) (ConvertedFigure, bool) { + inScope := map[string]bool{} + for _, l := range scope { + inScope[l] = true + } + onPage := make([]int, len(regions)) + for i := range regions { + onPage[i] = i + } + return attribute(f, regions, onPage, inScope, scope) +} + +// TestAGrownCropDoesNotChangeAFiguresLanguage is the funnel's one unforgivable +// failure, asserted where it can actually be refuted. +// +// [growToLabels] grows a figure's box sideways onto the labels its leaders point at, +// so a drawing in the German column can end up with a CROP that reaches into the +// Polish one. If [attribute] asked that crop which region it lies inside, the answer +// would be "none" — a figure straddling two regions is language-neutral — and the +// picture would be handed to every household in scope. A German drawing served to a +// Russian reader is the exact promise the funnel makes and may not break. +// +// It is asserted here, on geometry, because **neither fixture can refute it.** The +// only document with side-by-side language columns is the columns manual, and it +// grows nothing at all — both its claims are prose and both are blocked; the only +// document that grows has whole-page regions, where there is no neighbouring column +// to reach into. So the fixture-level check of this is a vacuous pass by +// construction, and a hand-built figure is the only thing that can fail when the +// rule is wrong. Verified by making [attribute] read Rect: this test fails and the +// two fixture ones do not. +// +// The geometry is page 14's real regions and a figure 100 units wider than its own +// ink, which is the order of growth measured — page 521's lidar drawing grew 134. +func TestAGrownCropDoesNotChangeAFiguresLanguage(t *testing.T) { + grown := figureAt(340, 660) // the crop reaches 76 units into the Polish column + grown.InkRect = CellRect{X0: 340, Y0: 241, X1: 560, Y1: 431} + + got, ok := attributeOn(grown, page14(), "de", "pl") + if !ok { + t.Fatal("a figure whose drawing is inside the German column was dropped") + } + if !reflect.DeepEqual(got.Langs, []string{"de"}) { + t.Errorf("langs = %v, want just de: the crop grew into the Polish column but "+ + "the DRAWING is German, and a picture's language is the picture's", got.Langs) + } + if got.Neutral { + t.Error("a figure whose drawing sits inside one region reported itself as " + + "language-neutral, so every household in scope would be served it") + } + if got.RegionX0 != 323 { + t.Errorf("RegionX0 = %v, want the German region's 323", got.RegionX0) + } + + // And the other direction, so this cannot be satisfied by ignoring the crop + // entirely: a drawing that genuinely straddles two columns is still neutral, + // grown or not. + straddling := figureAt(340, 660) + straddling.InkRect = CellRect{X0: 340, Y0: 241, X1: 660, Y1: 431} + got, ok = attributeOn(straddling, page14(), "de", "pl") + if !ok { + t.Fatal("a figure straddling two regions was dropped") + } + if !got.Neutral { + t.Error("a DRAWING straddling the German and Polish columns is nobody's " + + "column and must be neutral") + } +} + +// TestAFigureInsideARegionBelongsToItsLanguage is rule 2's first half. +func TestAFigureInsideARegionBelongsToItsLanguage(t *testing.T) { + got, ok := attributeOn(figureAt(340, 560), page14(), "de", "pl") + if !ok { + t.Fatal("a figure inside the German column was dropped") + } + if !reflect.DeepEqual(got.Langs, []string{"de"}) { + t.Errorf("langs = %v, want just de: a figure inside one language's column is that language's", got.Langs) + } + if got.Neutral { + t.Error("a figure inside a region reported itself as language-neutral") + } + if got.RegionX0 != 323 { + t.Errorf("RegionX0 = %v, want the German region's 323 so the figure can be placed among its blocks", got.RegionX0) + } +} + +// TestALanguageNeutralFigureBelongsToEveryLanguage is rule 2's second half, and +// the decision docs/design/conversion.md records as the user's: a reader must not +// lose a diagram because the diagram has no language of its own. +func TestALanguageNeutralFigureBelongsToEveryLanguage(t *testing.T) { + got, ok := attributeOn(figureAt(43, 288), page14(), "de", "pl") + if !ok { + t.Fatal("the shared picture column was dropped") + } + if !got.Neutral { + t.Error("a figure inside no region did not report itself as language-neutral") + } + if !reflect.DeepEqual(got.Langs, []string{"de", "pl"}) { + t.Errorf("langs = %v, want both languages in scope", got.Langs) + } + + // And for a household reading one language it is still that language's, which + // is what a reader of German alone must see on this page. + one, ok := attributeOn(figureAt(43, 288), page14(), "de") + if !ok || !reflect.DeepEqual(one.Langs, []string{"de"}) || !one.Neutral { + t.Errorf("for a German-only household the shared picture came back ok=%t neutral=%t langs=%v", + ok, one.Neutral, one.Langs) + } +} + +// TestAFigureInAnotherLanguagesColumnIsDropped is rule 1, the funnel, applied to +// pictures rather than to text. A German household must not be handed the Polish +// column's screenshot. +func TestAFigureInAnotherLanguagesColumnIsDropped(t *testing.T) { + if got, ok := attributeOn(figureAt(620, 850), page14(), "de"); ok { + t.Errorf("a figure inside the Polish column was given to a German household: %v", got.Langs) + } +} + +// TestAFigureStraddlingTwoRegionsIsNeutral is the case the containment test has +// to get right for the reason regions.md gives: a diagram set across the full +// measure of a parallel-columns page is inside no column and is everybody's. +func TestAFigureStraddlingTwoRegionsIsNeutral(t *testing.T) { + got, ok := attributeOn(figureAt(400, 700), page14(), "de", "pl") + if !ok || !got.Neutral { + t.Errorf("a figure spanning the German and Polish columns came back ok=%t neutral=%t", ok, got.Neutral) + } +} + +// TestAFigureInAnUnnamedRegionIsNeutral keeps an unnamed region from becoming a +// language. A region no signal could name is a reportable state everywhere else +// in this package, and a picture inside one has no language either. +func TestAFigureInAnUnnamedRegionIsNeutral(t *testing.T) { + regions := []Region{ + {Page: 14, X0: 40, X1: 300}, // no Code, no Lang: nothing was established + {Page: 14, X0: 323, X1: 584, Code: "DE", Lang: "de"}, + } + got, ok := attributeOn(figureAt(43, 288), regions, "de") + if !ok || !got.Neutral { + t.Errorf("a figure inside an unnamed region came back ok=%t neutral=%t; an unnamed "+ + "region is not a language", ok, got.Neutral) + } +} + +// TestAWholePageRegionOwnsEveryFigureOnIt is the sequential manual's shape, where +// a page is one language from edge to edge. The Russian maintenance pages depend +// on it: their eight line drawings each are inside the page's Russian region. +func TestAWholePageRegionOwnsEveryFigureOnIt(t *testing.T) { + regions := []Region{{Page: 533, X0: 0, X1: 918, Code: "RU", Lang: "ru"}} + f := &Figure{Page: 533, Rect: CellRect{X0: 88, Y0: 200, X1: 400, Y1: 420}} + got, ok := attribute(f, regions, []int{0}, map[string]bool{"ru": true}, []string{"ru"}) + if !ok || got.Neutral || !reflect.DeepEqual(got.Langs, []string{"ru"}) { + t.Errorf("a figure on a whole-page Russian region came back ok=%t neutral=%t langs=%v", + ok, got.Neutral, got.Langs) + } +} + +// TestARegionsEdgeHasTheSameSlackABlockGets pins the one tolerance here against +// the one RegionBlocks allows, because two different slacks would put a figure +// outside a region whose text is inside it. +func TestARegionsEdgeHasTheSameSlackABlockGets(t *testing.T) { + // Half a unit over the German region's right edge: inside. + if got, _ := attributeOn(figureAt(323, 584.5), page14(), "de"); got.Neutral { + t.Error("half a unit past the region's edge read as outside it") + } + // Two units over: outside, and so neutral. + if got, _ := attributeOn(figureAt(323, 586), page14(), "de"); !got.Neutral { + t.Error("two units past the region's edge still read as inside it") + } +} + +func TestConvertNeedsAResult(t *testing.T) { + if _, err := Convert(t.Context(), "irrelevant.pdf", nil, []string{"de"}, ConvertOptions{}); err == nil { + t.Error("a nil probe result was accepted") + } +} + +// TestConvertSaysWhyItConvertedNothing is the stance the rest of this package +// takes: a document is reported on, never failed, when something is missing. +func TestConvertSaysWhyItConvertedNothing(t *testing.T) { + res := &Result{ + Info: Info{Pages: 4}, + Runs: []Run{{Lang: "de", Start: 1, End: 4, Source: SourceReconciled}}, + RegionNote: "pdftohtml is not installed", + } + + // A household reading a language the document does not hold. + got, err := Convert(t.Context(), "irrelevant.pdf", res, []string{"ja"}, ConvertOptions{}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if len(got.Notes) != 1 || len(got.Blocks) != 0 { + t.Errorf("notes = %v over %d blocks; expected one note and nothing converted", + got.Notes, len(got.Blocks)) + } + + // A document whose regions could not be read at all. The note has to carry the + // probe's own reason, or the user is told "nothing" without being told why. + got, err = Convert(t.Context(), "irrelevant.pdf", res, []string{"de"}, ConvertOptions{}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if len(got.Notes) != 1 || !strings.Contains(got.Notes[0], "pdftohtml") { + t.Errorf("notes = %v; expected the probe's own RegionNote to be carried through", got.Notes) + } + if len(got.Scope.Languages) != 1 { + t.Errorf("the scope was not reported: %v", got.Scope.Languages) + } +} diff --git a/internal/doc/convert_pdf_test.go b/internal/doc/convert_pdf_test.go new file mode 100644 index 0000000..8f3878a --- /dev/null +++ b/internal/doc/convert_pdf_test.go @@ -0,0 +1,141 @@ +package doc_test + +import ( + "context" + "errors" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// The whole of Convert against a PDF this package writes, so the default suite +// covers the assembly offline with nothing committed. What it cannot cover is +// the judgement — a generated document has no shared picture column, no cell +// dividers shaped like language boundaries and no section that is illustrated +// where the other thirty-three are not. Those are what convert_fixture_test.go +// is for. + +// TestConvertReadsOnlyTheHouseholdsPages is the funnel end to end. Two languages, +// two pages each, a drawing on one page of each: a German household gets the +// German pages and the German drawing, and nothing of the Polish ones. +func TestConvertReadsOnlyTheHouseholdsPages(t *testing.T) { + d := testpdf.TaggedSections([]string{"DE", "PL"}, 2, true) + // The contents page is page 1, so DE is pages 2-3 and PL is 4-5. A drawing on + // the first page of each section, well over the ink guard. + d.Pages[1].Drawings = []testpdf.Drawing{{X: 100, Y: 400, W: 200, H: 150, Strokes: 40}} + d.Pages[3].Drawings = []testpdf.Drawing{{X: 100, Y: 400, W: 200, H: 150, Strokes: 40}} + path := figurePDF(t, d) + + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + conv, err := doc.Convert(context.Background(), path, res, []string{"de"}, doc.ConvertOptions{}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + t.Logf("%s", conv.Summary()) + if len(conv.Notes) > 0 { + t.Logf("notes: %v", conv.Notes) + } + + // The pages. Not "fewer than five": exactly the two the German section owns, + // because a page too many is a page the household is charged for. + if len(conv.Pages) != 2 || conv.Pages[0] != 2 || conv.Pages[1] != 3 { + t.Errorf("converted pages %v, want the German section's 2 and 3", conv.Pages) + } + + // Every block German, and every block from a German page. + if len(conv.Blocks) == 0 { + t.Fatal("the German section produced no blocks") + } + for i := range conv.Blocks { + b := &conv.Blocks[i] + if doc.BaseLanguage(b.Lang) != "de" { + t.Errorf("page %d block %d is %q, not German", b.Page, b.Index, b.Lang) + } + if b.Page < 2 || b.Page > 3 { + t.Errorf("block from page %d, outside the German section", b.Page) + } + } + + // One drawing, the German one, and it belongs to German because it sits inside + // the page's whole-page German region — not because it was the only one left. + if len(conv.Figures) != 1 { + t.Fatalf("got %d figures, want the one drawing on the German section's first page", len(conv.Figures)) + } + f := &conv.Figures[0] + if f.Page != 2 { + t.Errorf("the figure is on page %d, want page 2; page 4's drawing is Polish", f.Page) + } + if f.Neutral { + t.Error("the figure reported itself as language-neutral; it is inside a whole-page German region") + } + if len(f.Langs) != 1 || f.Langs[0] != "de" { + t.Errorf("the figure's languages are %v, want just de", f.Langs) + } + if len(f.PNG) == 0 || len(f.Digest) != 64 { + t.Errorf("the figure carries %d bytes and the digest %q; a converted figure has to carry its picture", + len(f.PNG), f.Digest) + } + if len(conv.FiguresFor("de")) != 1 || len(conv.FiguresFor("pl")) != 0 { + t.Errorf("FiguresFor: %d for German, %d for Polish", len(conv.FiguresFor("de")), len(conv.FiguresFor("pl"))) + } + // The regional form a household may have configured: a reader of de-AT reads + // the de section, which is the matching ScopeFor already did and which the + // accessors must not undo. + if len(conv.BlocksFor("de-AT")) != len(conv.Blocks) || len(conv.BlocksFor("pl")) != 0 { + t.Errorf("BlocksFor: %d blocks for de-AT and %d for Polish, out of %d", + len(conv.BlocksFor("de-AT")), len(conv.BlocksFor("pl")), len(conv.Blocks)) + } +} + +// TestConvertStopsWhenItsJobIsCancelled separates the two kinds of bad news. A +// missing tool is a note; a cancelled job is an error, and it must not come back +// as a page-by-page complaint that the document could not be read. +func TestConvertStopsWhenItsJobIsCancelled(t *testing.T) { + d := testpdf.TaggedSections([]string{"DE"}, 3, true) + d.Pages[1].Drawings = []testpdf.Drawing{{X: 100, Y: 400, W: 200, H: 150, Strokes: 40}} + path := figurePDF(t, d) + + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + conv, err := doc.Convert(ctx, path, res, []string{"de"}, doc.ConvertOptions{}) + if !errors.Is(err, context.Canceled) { + t.Errorf("Convert returned %v with %v; a cancelled context is an error, not a note", + err, conv) + } +} + +// TestConvertReportsALanguageTheDocumentDoesNotHold is the state a household +// reaches by configuring a language this manual was never printed in. It is +// reported, not failed. +func TestConvertReportsALanguageTheDocumentDoesNotHold(t *testing.T) { + path := figurePDF(t, testpdf.TaggedSections([]string{"DE", "PL"}, 2, true)) + + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + conv, err := doc.Convert(context.Background(), path, res, []string{"ja"}, doc.ConvertOptions{}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if len(conv.Blocks) != 0 || len(conv.Figures) != 0 || len(conv.Pages) != 0 { + t.Errorf("%d blocks, %d figures over %d pages for a language the document does not hold", + len(conv.Blocks), len(conv.Figures), len(conv.Pages)) + } + if len(conv.Notes) != 1 { + t.Errorf("notes = %v; the user has to be told why nothing came back", conv.Notes) + } + if len(conv.Scope.OtherLanguages) != 2 { + t.Errorf("%d other languages reported; the two the document does hold must still be offered", + len(conv.Scope.OtherLanguages)) + } +} diff --git a/internal/doc/doc.go b/internal/doc/doc.go new file mode 100644 index 0000000..5c32ee2 --- /dev/null +++ b/internal/doc/doc.go @@ -0,0 +1,726 @@ +// Package doc turns an uploaded document into facts about itself, cheaply, +// before anything expensive happens to it. +// +// The pipeline this package implements is the funnel described in +// docs/design/ingest.md. Its purpose is not to convert anything: it is to find +// out what is being held, for free, so that the expensive work can be aimed at +// the small part of the document that matters. On a measured 560-page, +// 34-language manual, 98% of a naive conversion spend buys nothing. +// +// Three stages run here, and all three are free: +// +// Stage 0 pdfinfo page count, encryption, structure tags ~0.06 s +// Stage 1 pdftotext per-page text, is there a text layer? ~1.8 s +// Stage 2 (local) the language map, from several signals ~0 s +// +// Nothing in this package calls a model, sends anything over a network, or +// costs money. Stage 3 — asking the user what to process — and stage 4 — actually +// processing it — happen elsewhere, after this package has reported what it found. +package doc + +import ( + "context" + "fmt" + "slices" + "sort" +) + +// Source identifies which signal produced a language run. Every run records its +// own source so that a disagreement between signals stays inspectable instead of +// being averaged into a single unattributable answer. +type Source string + +// The language signals, cheapest first. See docs/design/language-detection.md. +const ( + // SourcePageTag is the language code a manual prints on each page. + SourcePageTag Source = "page-tag" + // SourceIndex is the manual's own printed contents table. + SourceIndex Source = "index" + // SourceScript is Unicode script analysis. + SourceScript Source = "script" + // SourceRepertoire is which alphabet the text uses — the letters only some + // languages sharing a script can write. Free, and it settles cases Unicode + // script analysis cannot: Russian, Ukrainian and Kazakh in one document. + SourceRepertoire Source = "repertoire" + // SourceDetector is statistical language detection. No implementation is + // wired up yet; the constant exists so stored rows and the reconciliation + // order do not change when one is added. + SourceDetector Source = "detector" + // SourceReconciled is the resolved view built from the others. + SourceReconciled Source = "reconciled" +) + +// Run is a contiguous span of pages in one language, according to one signal. +type Run struct { + Source Source `json:"source"` + // Code is the language as the document expresses it, which may not be a valid + // tag: real manuals print UA, CZ and ZH-HK. + Code string `json:"code"` + // Lang is Code normalised to BCP-47, empty when it could not be normalised. + Lang string `json:"lang"` + // Start and End are inclusive 1-based PDF page numbers. + Start int `json:"start"` + End int `json:"end"` + // Title is the section title as printed in the manual's contents table, in + // that language. Only the index signal can supply it. + Title string `json:"title,omitempty"` + // PrintedPage is the start page the printed index claims. Only the index + // signal sets it, and it is frequently 1-2 off from reality. + PrintedPage *int `json:"printedPage,omitempty"` + // Confidence is this signal's confidence in this run, 0 to 1. + Confidence float64 `json:"confidence"` + // Conflict marks a reconciled run the signals disagreed about. + Conflict bool `json:"conflict"` + // Note explains a conflict, or records how the run was established. + Note string `json:"note,omitempty"` +} + +// Pages is how many pages the run covers. +// +// A run that named a language but could not place it covers none. Start 0 means +// "unplaceable", not page zero — the arithmetic span reported the printed index's +// unplaceable HE, AR and CZ entries as one-page sections at 0-0. +func (r Run) Pages() int { + if r.Start == 0 { + return 0 + } + return r.End - r.Start + 1 +} + +// Contains reports whether a page falls inside the run. +func (r Run) Contains(page int) bool { return page >= r.Start && page <= r.End } + +// Result is everything the free stages discovered about a document. +type Result struct { + Info Info `json:"info"` + + // Pages holds the per-page facts, one entry per page of the original. + Pages []Page `json:"-"` + + // BySource holds each signal's own view of the language map, unreconciled. + BySource map[Source][]Run `json:"bySource"` + // Runs is the reconciled language map: what manualbox actually believes. + Runs []Run `json:"runs"` + + // Regions is the language map at the resolution a page can hold several: one + // entry per language territory, whole-page where the page holds one language + // and boxed where it holds more. See docs/design/regions.md. + // + // Empty when positioned text could not be read, which is a stated state rather + // than a silent one — see RegionNote. Regions need coordinates and there is no + // honest way to invent them, so without pdftohtml the pipeline reports exactly + // what it reported before regions existed. + Regions []Region `json:"-"` + // RegionNote says why Regions is empty, when it is empty for a reason worth + // telling the user rather than because the document has no text. + RegionNote string `json:"regionNote,omitempty"` + // NeutralNote says why the pictures on the pages no language claims were not + // counted, when they were not. Separate from RegionNote because the two are + // different things being unavailable: regions can exist while the ink pass is + // skipped, which is what happens when the set is too large to census. + NeutralNote string `json:"neutralNote,omitempty"` + + // MedianChars is the median rune count across all pages. A scan yields ~0, + // which is the number that selects between the free extraction path and one + // costing a vision call per page. + MedianChars int `json:"medianChars"` + // HasTextLayer reports whether text extraction is viable at all. + HasTextLayer bool `json:"hasTextLayer"` + // PagesWithText counts pages that yielded meaningful text. + PagesWithText int `json:"pagesWithText"` + + // ContentStart and ContentEnd bound the pages holding actual content, + // excluding front matter and back cover. + ContentStart int `json:"contentStart"` + ContentEnd int `json:"contentEnd"` + + // Unlabelled counts pages that carry text, sit in no language run, and are + // not a contents table. It is the honest measure of how much a statistical + // detector would add for this document. + // + // A small non-zero value is normal and not a fault: a cover and a colophon + // carry text and belong to no section. On the measured 560-page fixture it is + // 4. What matters is the magnitude — 4 says the free signals covered the + // document, 100 says they did not. + Unlabelled int `json:"unlabelled"` +} + +// MinTextChars is how many runes a page needs before it counts as carrying text. +// Page furniture alone — a folio, a language tab, a header — is a few dozen runes +// on an otherwise scanned page, so the floor has to sit above that. +// +// Exported because the pre-flight gate counts unnamed pages from stored rows +// rather than from a Result, and it has to apply the same floor as +// [CountUnlabelled] does here. Two floors would disagree about the same document. +const MinTextChars = 50 + +// textLayerPageFraction is the share of pages that must carry text before +// extraction is considered viable. A median alone misjudges a document that is +// half scanned, which is common when someone photographs the pages they need. +const textLayerPageFraction = 0.5 + +// Analyze runs stages 0 through 2 against a document on disk. +// +// It never mutates the file and never calls anything remote, so it is safe to run +// on upload and safe to re-run: it is a pure function of the bytes, which is what +// lets the probe job be idempotent. +func Analyze(ctx context.Context, path string) (*Result, error) { + info, err := ProbeInfo(ctx, path) + if err != nil { + return nil, err + } + + res := &Result{Info: info, BySource: make(map[Source][]Run, 4)} + + // An encrypted PDF cannot be extracted from. Report what stage 0 found and + // stop rather than failing: the original is still stored, and the user can be + // told precisely why nothing else happened. + if info.Encrypted { + res.Runs = []Run{} + return res, nil + } + + pages, err := ExtractText(ctx, path, info.Pages) + if err != nil { + return nil, err + } + res.Pages = pages + + res.MedianChars = medianChars(pages) + res.PagesWithText = countWithText(pages) + if len(pages) > 0 { + res.HasTextLayer = float64(res.PagesWithText)/float64(len(pages)) >= textLayerPageFraction + } + + // With no text there is nothing for the free language signals to read. The + // OCR path handles this, and it is not part of this package's job. + if !res.HasTextLayer { + res.Runs = []Run{} + res.ContentStart, res.ContentEnd = firstLastWithText(pages) + return res, nil + } + + // The index is parsed first even though the page tag outranks it, because the + // index supplies the vocabulary of codes that makes a loose tag reading safe. + // The ordering here is about evidence availability; the ordering that decides + // disagreements lives in reconcile.go. + indexRuns := IndexRuns(pages) + res.BySource[SourceIndex] = indexRuns + + tags := EffectiveTags(pages, IndexCodes(indexRuns)) + for i := range res.Pages { + res.Pages[i].Tag = tags[i] + } + res.BySource[SourcePageTag] = TagRuns(res.Pages, tags) + res.BySource[SourceScript] = ScriptRuns(res.Pages) + + res.Runs = Reconcile(res.Pages, res.BySource) + + res.ContentStart, res.ContentEnd = ContentRange(pages, res.Runs) + res.Unlabelled = CountUnlabelled(pages, res.Runs) + + // Regions run last because a whole-page region records the reconciled language, + // so they need the map above to already exist. + var runs []PageRuns + res.Regions, runs, res.RegionNote = analyzeRegions(ctx, path, res, IndexCodes(indexRuns)) + + // And the picture census over the pages no region named runs after the regions, + // because which pages those are is the regions' answer. It is bounded and lazy — + // see countNeutralInk — and its note joins RegionNote rather than replacing it, + // since the two report different things being unavailable. + if len(res.Regions) > 0 { + if note := countNeutralInk(ctx, path, res, runs); note != "" { + res.NeutralNote = note + } + } + + return res, nil +} + +// analyzeRegions reads the document's positioned text and divides each page into +// language regions. +// +// This is the second poppler pass, and the whole probe was timed rather than +// extrapolated from it — best of three, page cache warm: +// +// 560-page, 15 MB manual Analyze 3.71 s, of which this pass 1.86 s (50%) +// 68-page, 9 MB manual Analyze 4.09 s, of which this pass 3.03 s (74%) +// +// So the probe roughly doubles on the first document and is dominated by this pass +// on the second — which is the SMALLER document, 68 pages costing more than 560. +// Cost here is evidently per page of content rather than per page: the 68-page +// manual carries 139 KB and 110 runs per page against the other's 27 KB and 61. Why +// that ratio produces this one has not been measured, so it is recorded as a fact +// and not explained. What matters for the funnel is the bound, and both are a few +// seconds, free, and local. +// +// It buys the only reading of a parallel-columns manual that is not wrong: a page +// there holds three languages, so no per-page answer about it can be right. +// +// A missing or failing pdftohtml is reported, not fatal. The document has already +// been probed by this point and its per-page language map is complete; losing +// regions costs the column resolution and nothing else, so the honest outcome is +// the previous behaviour plus a note saying what is unavailable and why. +// The positioned text is returned alongside the regions rather than dropped, +// because the neutral-page picture census needs the same runs and re-extracting +// them would pay for the document's whole 3.8 MB of coordinates a second time to +// learn nothing new. +func analyzeRegions(ctx context.Context, path string, res *Result, + knownCodes map[string]bool) (regions []Region, runs []PageRuns, note string) { + pages, err := ExtractRuns(ctx, path) + if err != nil { + return nil, nil, fmt.Sprintf("per-column languages are unavailable: %s", err) + } + + byNo := make(map[int]*Page, len(res.Pages)) + for i := range res.Pages { + byNo[res.Pages[i].No] = &res.Pages[i] + } + + regions = make([]Region, 0, len(pages)) + for i := range pages { + p := &pages[i] + code, lang, source := res.pageLanguage(p.No) + resolved := PageResolution{Code: code, Lang: lang, Source: source} + if page, ok := byNo[p.No]; ok { + resolved.Contents = IsContentsPage(page) + } + regions = append(regions, pageRegionsWithTables(ctx, path, p, knownCodes, resolved)...) + } + return regions, pages, "" +} + +// pageRegionsWithTables derives one page's regions, reading its ruled lines only +// where a table could have decided the answer. +// +// The lazy read is the whole point and it is a cost decision taken from +// measurement rather than a shortcut. Reading the ruled lines is a pdftocairo +// spawn per page — 5.9 s over the column manual's 68 pages and 42.3 s over the +// sequential manual's 560, against a probe of about 4 s for either — and this pass +// runs inside the free pre-flight, where docs/design/conversion.md's whole cost +// argument is that ruled lines are read only for the pages a user has paid for. +// Reading them for every page would make the free gate ten times slower on the +// larger document. +// +// What narrows it: [mergeCellColumns] can only change a stored answer by stopping +// a page from dividing, so the ruled lines are worth reading exactly for a page +// that just divided. Measured, that is 44 of the column manual's 68 pages and 0 +// of the sequential manual's 560 — which is a document with no parallel columns +// paying nothing at all, and the column manual's probe going from 4.1 s to 7.9 s. +// +// A page whose columns already agree keeps today's reading untouched, and that is +// deliberate rather than a gap. The column manual's pages 58 to 61 are tables +// covering almost the whole measure, and it is their cell columns — all of one +// language — that name them at all; subtracting the table there would leave a +// running head and lose the page's language entirely. +// +// A failing or missing pdftocairo leaves the pre-table reading in place, which is +// exactly the behaviour that shipped. It is silent because there is nothing +// actionable to say: the answer is the same one the previous release gave. +func pageRegionsWithTables(ctx context.Context, path string, p *PageRuns, + knownCodes map[string]bool, resolved PageResolution) []Region { + regions := PageRegions(p, knownCodes, resolved, nil) + if len(regions) < 2 { + return regions + } + tables, err := PageTables(ctx, path, p) + if err != nil || len(tables) == 0 { + return regions + } + return PageRegions(p, knownCodes, resolved, tables) +} + +// Languages returns the language map collapsed to one entry per language, in +// document order. This is what the pre-flight gate shows. +// +// It reads the regions where there are regions, and the per-page runs otherwise. +// That is not a preference between two equivalent sources: on the parallel-columns +// manual the per-page map names nothing on any of its verified pages, so summarised +// from runs alone that document reports no languages at all while plainly +// containing five. Regions are the finer-grained record of the same reconciliation, +// so on a sectioned manual the two agree exactly — asserted against that manual's +// 34 sections, page counts and spans included. +func (r *Result) Languages() []LanguageSummary { + if len(r.Regions) > 0 { + return r.regionLanguages() + } + return r.runLanguages() +} + +// regionLanguages summarises the regions, one entry per language. +func (r *Result) regionLanguages() []LanguageSummary { + titles := indexTitles(r.BySource[SourceIndex]) + + type acc struct { + code, lang string + pages map[int]bool + disputed bool + } + order := make([]string, 0, 8) + seen := make(map[string]*acc, 8) + + for i := range r.Regions { + region := &r.Regions[i] + if region.Lang == "" { + continue + } + // Keyed by language rather than by printed label, so a section the document + // calls UA and a signal calls uk are one language and not two. + key := BaseLanguage(region.Lang) + if key == "" { + key = region.Code + } + a, ok := seen[key] + if !ok { + a = &acc{code: region.Code, lang: region.Lang, pages: make(map[int]bool, 16)} + seen[key] = a + order = append(order, key) + } + a.pages[region.Page] = true + if region.Conflict { + a.disputed = true + } + // Keep the most specific tag seen for this language: zh-HK beats zh. + if len(region.Lang) > len(a.lang) { + a.lang, a.code = region.Lang, region.Code + } + } + + out := make([]LanguageSummary, 0, len(order)) + for _, key := range order { + a := seen[key] + first, last, spans := pageSpans(a.pages) + out = append(out, LanguageSummary{ + Code: a.code, Lang: a.lang, Title: titles[a.code], + Name: DisplayName(a.lang), Pages: len(a.pages), + FirstPage: first, LastPage: last, Runs: spans, + Disputed: a.disputed, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].FirstPage < out[j].FirstPage }) + return out +} + +// pageSpans reports the first and last page a language occupies and how many +// contiguous stretches it occupies them in. +// +// The count of stretches is what [LanguageSummary.Runs] means, and it is worth +// computing rather than approximating: a section wrongly split in two still totals +// the right number of pages, which is how one such split went unnoticed. +func pageSpans(pages map[int]bool) (first, last, spans int) { + if len(pages) == 0 { + return 0, 0, 0 + } + sorted := make([]int, 0, len(pages)) + for page := range pages { + sorted = append(sorted, page) + } + slices.Sort(sorted) + + spans = 1 + for i := 1; i < len(sorted); i++ { + if sorted[i] != sorted[i-1]+1 { + spans++ + } + } + return sorted[0], sorted[len(sorted)-1], spans +} + +// runLanguages summarises the per-page reconciled runs, for a document whose +// positioned text could not be read. +func (r *Result) runLanguages() []LanguageSummary { + type acc struct { + code, lang, title string + pages, first, last int + runs int + disputed bool + } + order := make([]string, 0, 8) + seen := make(map[string]*acc, 8) + + for _, run := range r.Runs { + // Keyed by language, not by printed label, so a section the document calls + // UA and a signal calls uk are one language rather than two. + key := BaseLanguage(run.Lang) + if key == "" { + key = run.Code + } + a, ok := seen[key] + if !ok { + a = &acc{code: run.Code, lang: run.Lang, title: run.Title, first: run.Start, last: run.End} + seen[key] = a + order = append(order, key) + } + a.pages += run.Pages() + a.runs++ + if run.End > a.last { + a.last = run.End + } + if run.Conflict { + a.disputed = true + } + if a.title == "" { + a.title = run.Title + } + // Keep the most specific tag seen for this language: zh-HK beats zh. + if len(run.Lang) > len(a.lang) { + a.lang, a.code = run.Lang, run.Code + } + if run.Start < a.first { + a.first = run.Start + } + } + + out := make([]LanguageSummary, 0, len(order)) + for _, key := range order { + a := seen[key] + out = append(out, LanguageSummary{ + Code: a.code, Lang: a.lang, Title: a.title, + Name: DisplayName(a.lang), Pages: a.pages, + FirstPage: a.first, LastPage: a.last, Runs: a.runs, + Disputed: a.disputed, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].FirstPage < out[j].FirstPage }) + return out +} + +// LanguageSummary is one language's total presence in a document. +type LanguageSummary struct { + // Code is the label as the document printed it, e.g. "UA". + Code string `json:"code"` + // Lang is the BCP-47 tag, e.g. "uk". + Lang string `json:"lang"` + // Name is the English display name, e.g. "Ukrainian". + Name string `json:"name"` + // Title is the section title in that language, when the index supplied one. + Title string `json:"title,omitempty"` + // Pages is how many pages of the document are in this language. + Pages int `json:"pages"` + // FirstPage is where it starts. + FirstPage int `json:"firstPage"` + // LastPage is where it ends. + LastPage int `json:"lastPage"` + // Runs is how many separate spans this language occupies. More than one is + // legitimate — a manual may return to a language — but it is also how a + // wrongly split section shows itself, since page totals alone cannot. + Runs int `json:"runs"` + // Disputed reports that the signals disagreed somewhere in this language. + Disputed bool `json:"disputed"` +} + +// Scope is what would actually be processed for a given set of household +// languages: the answer the pre-flight gate needs. +type Scope struct { + // Languages are the household languages present in this document. + Languages []LanguageSummary `json:"languages"` + // Pages is how many pages those languages occupy. + Pages int `json:"pages"` + // TotalPages is the document's page count, for the comparison that makes the + // saving visible. + TotalPages int `json:"totalPages"` + // Chars is the extracted character count of those pages, which is the honest + // free proxy for size. A token count needs a provider and is not invented + // here; see docs/design/providers.md. + Chars int `json:"chars"` + // OtherLanguages are the languages present that the household does not read. + // They are never discarded — the original is kept whole, so importing them + // later is a button rather than a re-upload. + OtherLanguages []LanguageSummary `json:"otherLanguages"` +} + +// Fraction is the share of the document the scope covers, 0 to 1. +func (s Scope) Fraction() float64 { + if s.TotalPages == 0 { + return 0 + } + return float64(s.Pages) / float64(s.TotalPages) +} + +// ScopeFor intersects the document's languages with the household's. +func (r *Result) ScopeFor(household []string) Scope { + scope := Scope{TotalPages: r.Info.Pages} + + // Keyed by base language, not by printed code. A summary carries one code per + // language — the most specific tag seen wins a contest between them — while the + // runs each carry their own. Keying on the code counted the pages of every run + // but the characters of only those whose label happened to win: a document + // printing CN, JA and ZH-HK reported 4 pages and 2000 characters where the same + // pages hold 4000. + inScope := make(map[string]bool, len(household)) + for _, summary := range r.Languages() { + if _, ok := MatchesAny(summary.Lang, household); ok { + scope.Languages = append(scope.Languages, summary) + scope.Pages += summary.Pages + inScope[BaseLanguage(summary.Lang)] = true + } else { + scope.OtherLanguages = append(scope.OtherLanguages, summary) + } + } + + scope.Chars = r.scopeChars(inScope) + return scope +} + +// scopeChars measures the text the household's languages actually occupy. +// +// From the regions where there are regions, because that is the only correct +// answer on a manual whose languages share a page: three languages in three +// columns, and charging a household for all three because one of them is theirs +// overstates the work by the number of languages on the page. On the measured +// column manual that is a factor of about three. +// +// Falling back to whole pages where there are none is not a lesser answer for a +// sectioned manual — there, one page is one language and the two agree — but it is +// a different measurement, taken with pdftotext rather than pdftohtml. Measured, +// the two tools disagree by 3.3% and 2.5% on the two fixtures' totals, 1 to 2% on a +// median page, and by up to 51% on individual pages where a text layer parks runs +// off the page. So the number moves slightly with which tool produced it, which is +// acceptable for a free proxy that docs/design/providers.md already refuses to turn +// into a token count, and is worth writing down rather than discovering later. +func (r *Result) scopeChars(inScope map[string]bool) int { + if len(r.Regions) > 0 { + return RegionChars(r.Regions, inScope) + } + + byPage := make(map[int]bool, 64) + for i := range r.Runs { + if inScope[BaseLanguage(r.Runs[i].Lang)] { + for p := r.Runs[i].Start; p <= r.Runs[i].End; p++ { + byPage[p] = true + } + } + } + chars := 0 + for i := range r.Pages { + if byPage[r.Pages[i].No] { + chars += r.Pages[i].Chars + } + } + return chars +} + +// PageLang returns the reconciled language for a page, or "" if none was +// established. +func (r *Result) PageLang(page int) (string, Source) { + _, lang, source := r.pageLanguage(page) + return lang, source +} + +// pageLanguage returns the reconciled label as well as the tag, which a region +// stores: the code is how the document expressed it — D, RUS, UA — and dropping it +// would leave an unnormalisable code unreportable. +func (r *Result) pageLanguage(page int) (code, lang string, source Source) { + for i := range r.Runs { + if r.Runs[i].Contains(page) { + return r.Runs[i].Code, r.Runs[i].Lang, r.Runs[i].Source + } + } + return "", "", "" +} + +// String renders a one-line summary for logs. It deliberately carries no +// filename or path, only shape, so it is safe in a log line. +func (r *Result) String() string { + return fmt.Sprintf("%d pages, text=%t, median %d chars, %d languages, %d unlabelled", + r.Info.Pages, r.HasTextLayer, r.MedianChars, len(r.Languages()), r.Unlabelled) +} + +func medianChars(pages []Page) int { + if len(pages) == 0 { + return 0 + } + counts := make([]int, len(pages)) + for i := range pages { + counts[i] = pages[i].Chars + } + slices.Sort(counts) + mid := len(counts) / 2 + if len(counts)%2 == 1 { + return counts[mid] + } + return (counts[mid-1] + counts[mid]) / 2 +} + +func countWithText(pages []Page) int { + n := 0 + for i := range pages { + if pages[i].Chars >= MinTextChars { + n++ + } + } + return n +} + +func firstLastWithText(pages []Page) (first, last int) { + for i := range pages { + if pages[i].Chars >= MinTextChars { + if first == 0 { + first = pages[i].No + } + last = pages[i].No + } + } + return first, last +} + +// ContentRange is the span the language map covers: the first and last page +// belonging to an identified language section. +// +// Deliberately not "where the body of the document is". Those two readings pull +// in opposite directions and no evidence here separates them — six unnameable +// pages at the front are furniture to be excluded, fifty are a body the signals +// failed on, and the only difference is how many. An earlier attempt to serve +// both needed a page-count threshold tuned to one document, which is the kind of +// constant that silently misbehaves on the next one. +// +// So this answers the narrow question, which the runs answer exactly: on the +// measured fixture, 7-559 of 560, correctly excluding six front-matter pages and +// an English colophon. +// +// It must NOT be used to decide which pages count as unlabelled. That was the +// original defect — the range comes from the runs, so a page no signal could name +// fell outside it by construction and could never be counted. [CountUnlabelled] +// no longer consults it. +func ContentRange(pages []Page, runs []Run) (start, end int) { + for i := range runs { + // A run that fixed no boundary says nothing about where content lies. + if runs[i].Start == 0 { + continue + } + if start == 0 || runs[i].Start < start { + start = runs[i].Start + } + end = max(end, runs[i].End) + } + if start == 0 { + // Nothing was labelled, so the text itself is the only evidence. + return firstLastWithText(pages) + } + return start, end +} + +func CountUnlabelled(pages []Page, runs []Run) int { + labelled := make(map[int]bool, len(pages)) + for i := range runs { + for p := runs[i].Start; p <= runs[i].End; p++ { + labelled[p] = true + } + } + + n := 0 + for i := range pages { + p := &pages[i] + switch { + case p.Chars < MinTextChars: + // Nothing to name. + case labelled[p.No]: + case IsContentsPage(p): + // A contents table is furniture, and it is the one kind this code can + // identify structurally rather than by guessing. + default: + n++ + } + } + return n +} diff --git a/internal/doc/doc_test.go b/internal/doc/doc_test.go new file mode 100644 index 0000000..f0c88eb --- /dev/null +++ b/internal/doc/doc_test.go @@ -0,0 +1,990 @@ +package doc_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for the language signals and their reconciliation, built from +// synthetic pages. No PDF and no poppler: these state the rules, and +// fixture_test.go checks them against a real 34-language manual. + +// page builds a synthetic page. body is repeated so the page clears the +// text-layer floor, since a page with almost no text is deliberately ignored. +func page(no int, tag, script string, folio int, body string) doc.Page { + text := strings.Repeat(body+" ", 30) + p := doc.Page{ + No: no, Text: text, Chars: len([]rune(text)), + Script: script, Tag: tag, + } + if tag != "" { + p.TagCandidates = []string{tag} + } + if folio > 0 { + p.Folio = &folio + } + return p +} + +// thinPage is a page with too little text to classify: a full-page illustration. +func thinPage(no int) doc.Page { + return doc.Page{No: no, Text: "12", Chars: 2} +} + +func TestTagRunsNeedsTwoConsecutivePages(t *testing.T) { + // A contents page lists every language code in the same position the per-page + // tab occupies, producing one-page runs. Requiring two consecutive pages is + // what keeps a contents page from becoming a section — measured on a real + // manual, where it produced three bogus sections. + pages := []doc.Page{ + page(1, "EN", doc.ScriptLatin, 0, "contents"), // a contents page + page(2, "DE", doc.ScriptLatin, 1, "guten tag"), + page(3, "DE", doc.ScriptLatin, 2, "guten tag"), + } + + runs := doc.TagRuns(pages, nil) + if len(runs) != 1 { + t.Fatalf("expected 1 run, got %d: %+v", len(runs), runs) + } + if runs[0].Code != "DE" || runs[0].Start != 2 || runs[0].End != 3 { + t.Errorf("run = %s %d-%d, want DE 2-3", runs[0].Code, runs[0].Start, runs[0].End) + } +} + +func TestContentsPageAdjacentToItsFirstSectionIsExcluded(t *testing.T) { + // The run-length guard is not enough on its own. A contents page listing EN + // first, sitting immediately before the EN section, is contiguous with it and + // gets absorbed — inflating that section by one page. A contents table is not a + // page of any language, so it is excluded outright. + contents := doc.Page{No: 1} + contents.Text = "Contents\nEN\nUser Manual\n1\nDE\nBenutzerhandbuch\n4\nFR\nManuel\n7\n" + contents.Chars = len([]rune(contents.Text)) + contents.Tag = "EN" // what the naive reading of the first lines produces + contents.TagCandidates = []string{"EN", "DE", "FR"} + + pages := []doc.Page{ + contents, + page(2, "EN", doc.ScriptLatin, 1, "english"), + page(3, "EN", doc.ScriptLatin, 2, "english"), + page(4, "DE", doc.ScriptLatin, 3, "german"), + page(5, "DE", doc.ScriptLatin, 4, "german"), + } + + if !doc.IsContentsPage(&contents) { + t.Fatal("the contents page was not recognised as one") + } + + tags := doc.EffectiveTags(pages, doc.IndexCodes(doc.IndexRuns(pages))) + if tags[0] != "" { + t.Errorf("the contents page kept the tag %q", tags[0]) + } + + for _, r := range doc.TagRuns(pages, tags) { + if r.Contains(1) { + t.Errorf("run %s %d-%d absorbed the contents page", r.Code, r.Start, r.End) + } + if r.Code == "EN" && r.Pages() != 2 { + t.Errorf("EN covers %d pages, want 2", r.Pages()) + } + } +} + +func TestTagRunsCorroboratedByScriptRankHigher(t *testing.T) { + greek := []doc.Page{ + page(1, "EL", doc.ScriptGreek, 1, "οδηγίες"), + page(2, "EL", doc.ScriptGreek, 2, "οδηγίες"), + } + runs := doc.TagRuns(greek, nil) + if len(runs) != 1 || runs[0].Confidence != 1.0 { + t.Fatalf("script-corroborated tag should have confidence 1.0, got %+v", runs) + } + + // A tag the script contradicts is disbelieved rather than trusted: a run + // tagged EL whose pages are Cyrillic is not Greek. + mismatched := []doc.Page{ + page(1, "EL", doc.ScriptCyrillic, 1, "инструкция"), + page(2, "EL", doc.ScriptCyrillic, 2, "инструкция"), + } + runs = doc.TagRuns(mismatched, nil) + if len(runs) != 1 { + t.Fatalf("expected 1 run, got %d", len(runs)) + } + if runs[0].Confidence > 0.3 { + t.Errorf("tag contradicted by script should be low confidence, got %.1f: %s", + runs[0].Confidence, runs[0].Note) + } +} + +func TestEffectiveTagsNarrowsByIndexVocabulary(t *testing.T) { + // Searching a whole page for a code is necessary for right-to-left layouts but + // unsafe on its own: NO, IT, IS and BE are all valid language codes and + // ordinary English words. A candidate is adopted only if the document's own + // contents table lists that code. + pages := []doc.Page{ + {No: 1, Chars: 500, TagCandidates: []string{"NO"}}, // stray word in a table + {No: 2, Chars: 500, TagCandidates: []string{"AR"}}, + } + + tags := doc.EffectiveTags(pages, map[string]bool{"AR": true}) + if tags[0] != "" { + t.Errorf("page 1 adopted %q, but NO is not in the index vocabulary", tags[0]) + } + if tags[1] != "AR" { + t.Errorf("page 2 tag = %q, want AR", tags[1]) + } +} + +func TestEffectiveTagsPrefersTheConservativeReading(t *testing.T) { + // A code found at the top of the page is trusted without needing the index's + // blessing, so a manual with no parseable contents table still works. + pages := []doc.Page{{No: 1, Chars: 500, Tag: "SV", TagCandidates: []string{"SV"}}} + if got := doc.EffectiveTags(pages, nil); got[0] != "SV" { + t.Errorf("tag = %q, want SV even with an empty vocabulary", got[0]) + } +} + +func TestIndexRunsResolveClaimsThroughFolios(t *testing.T) { + // The index claims printed page numbers; folios printed on the pages + // themselves are what convert a claim into a PDF page, with no global offset + // assumed. Here the front matter is 2 pages, so folio n is PDF page n+2. + contents := page(1, "", doc.ScriptLatin, 0, + "") // replaced below + contents.Text = "Contents\nEN\nUser Manual\n1\nDE\nBenutzerhandbuch\n3\nFR\nManuel\n5\n" + contents.Chars = len([]rune(contents.Text)) + + pages := []doc.Page{ + contents, + page(2, "", doc.ScriptLatin, 0, "cover"), + page(3, "EN", doc.ScriptLatin, 1, "english"), + page(4, "EN", doc.ScriptLatin, 2, "english"), + page(5, "DE", doc.ScriptLatin, 3, "german"), + page(6, "DE", doc.ScriptLatin, 4, "german"), + page(7, "FR", doc.ScriptLatin, 5, "french"), + page(8, "FR", doc.ScriptLatin, 6, "french"), + } + + runs := doc.IndexRuns(pages) + if len(runs) != 3 { + t.Fatalf("expected 3 index entries, got %d: %+v", len(runs), runs) + } + want := map[string]int{"EN": 3, "DE": 5, "FR": 7} + for _, r := range runs { + if got := want[r.Code]; r.Start != got { + t.Errorf("%s resolved to page %d, want %d", r.Code, r.Start, got) + } + if r.Title == "" { + t.Errorf("%s carries no title; titles are the index's unique contribution", r.Code) + } + } +} + +func TestContentsPageFoliosDoNotResolveIndexClaims(t *testing.T) { + // A contents page's trailing number is an index entry's page reference, not a + // folio — the page is listing "FR ... 5", not declaring itself page 5. Treating + // it as one maps a claimed page onto the contents page itself: on a real + // document, page 2 ends with "194", so the Arabic section's claimed start of + // 194 resolved to page 2 and produced a one-page Arabic section at the front. + // + // The contents page therefore carries a folio here, exactly as the extractor + // would derive one, because that is the condition that triggers the bug. + contentsFolio := 5 + contents := doc.Page{No: 1, Folio: &contentsFolio} + contents.Text = "Contents\nEN\nUser Manual\n1\nDE\nBenutzerhandbuch\n3\nFR\nManuel\n5\n" + contents.Chars = len([]rune(contents.Text)) + + pages := []doc.Page{ + contents, + page(2, "", doc.ScriptLatin, 0, "cover"), + page(3, "EN", doc.ScriptLatin, 1, "english"), + page(4, "EN", doc.ScriptLatin, 2, "english"), + page(5, "DE", doc.ScriptLatin, 3, "german"), + page(6, "DE", doc.ScriptLatin, 4, "german"), + page(7, "FR", doc.ScriptLatin, 5, "french"), + page(8, "FR", doc.ScriptLatin, 6, "french"), + } + + for _, r := range doc.IndexRuns(pages) { + if r.Code != "FR" { + continue + } + if r.Start == 1 { + t.Fatal("the French claim resolved onto the contents page, whose trailing " + + "number is an index reference rather than a folio") + } + if r.Start != 7 { + t.Errorf("French resolved to page %d, want 7", r.Start) + } + return + } + t.Fatal("no French index entry was parsed") +} + +func TestIndexRunsRejectClaimOnTheWrongScript(t *testing.T) { + // A real manual's contents table lists Czech at a page that is actually + // Arabic — a typo the manufacturer ships. Resolving it faithfully produces a + // Czech claim over Arabic pages, so script has to veto it. + contents := doc.Page{No: 1} + contents.Text = "Contents\nCZ\nUzivatelska prirucka\n5\nAR\nArabic manual\n7\nEN\nUser Manual\n9\n" + contents.Chars = len([]rune(contents.Text)) + + pages := []doc.Page{ + contents, + page(2, "", doc.ScriptLatin, 0, "cover"), + page(3, "AR", doc.ScriptArabic, 5, "عربي"), + page(4, "AR", doc.ScriptArabic, 6, "عربي"), + page(5, "AR", doc.ScriptArabic, 7, "عربي"), + page(6, "EN", doc.ScriptLatin, 9, "english"), + } + + for _, r := range doc.IndexRuns(pages) { + if r.Code != "CZ" { + continue + } + if r.Start != 0 { + t.Errorf("Czech claim resolved to page %d, which is Arabic script; it should contribute no boundary", r.Start) + } + // The note must state what was observed, so a user can judge it themselves: + // which language was claimed, what is actually on that page, and the + // consequence. + for _, want := range []string{"CZ", "Arabic", "no boundary"} { + if !strings.Contains(r.Note, want) { + t.Errorf("the note should mention %q to explain the rejection, got %q", want, r.Note) + } + } + return + } + t.Fatal("the Czech entry was dropped entirely; it should be kept for its label and title") +} + +func TestReconcilePrefersTheTagOverTheIndex(t *testing.T) { + pages := []doc.Page{ + page(1, "DA", doc.ScriptLatin, 1, "dansk"), + page(2, "DA", doc.ScriptLatin, 2, "dansk"), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "DA", Lang: "da", Start: 1, End: 2, Confidence: 1}}, + doc.SourceIndex: {{Source: doc.SourceIndex, Code: "FI", Lang: "fi", Start: 1, End: 2, Confidence: 0.6}}, + } + + runs := doc.Reconcile(pages, bySource) + if len(runs) != 1 { + t.Fatalf("expected 1 run, got %d", len(runs)) + } + if runs[0].Lang != "da" { + t.Errorf("lang = %q, want da: the printed tag outranks the index", runs[0].Lang) + } +} + +func TestReconcileRejectsALanguageItsScriptForbids(t *testing.T) { + // A printed index's final entry claims every remaining page, which on a real + // manual swallowed an English back cover into the Japanese section. Japanese + // cannot be written in the Latin alphabet, so the claim must not win. + pages := []doc.Page{ + page(1, "JA", doc.ScriptKana, 1, "説明書です"), + page(2, "JA", doc.ScriptKana, 2, "説明書です"), + page(3, "", doc.ScriptLatin, 0, "Made in China. For support contact us."), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "JA", Lang: "ja", Start: 1, End: 2, Confidence: 1}}, + doc.SourceIndex: {{Source: doc.SourceIndex, Code: "JA", Lang: "ja", Start: 1, End: 3, Confidence: 0.6}}, + } + + runs := doc.Reconcile(pages, bySource) + + // The negative assertion alone is vacuous: it passes if Reconcile returns + // nothing at all, which a total failure would. Verified by gutting Reconcile — + // this test went green. So assert what must still be true as well. + labelled := 0 + for _, r := range runs { + if r.Contains(3) { + t.Errorf("Latin-script page 3 was labelled %s (%s)", r.Code, r.Lang) + } + for p := r.Start; p <= r.End; p++ { + if p == 1 || p == 2 { + labelled++ + } + } + if r.Lang != "ja" { + t.Errorf("run %s has lang %q, want ja", r.Code, r.Lang) + } + } + if labelled != 2 { + t.Errorf("%d of pages 1-2 were labelled Japanese, want 2 — rejecting page 3 must not cost the real section", labelled) + } +} + +func TestReconcileBridgesALowTextPage(t *testing.T) { + // A full-page illustration between two pages of the same language belongs to + // that language. Splitting the section there produced two spans whose page + // totals still summed correctly, which is how it escaped notice. + pages := []doc.Page{ + page(1, "JA", doc.ScriptKana, 1, "説明書"), + page(2, "JA", doc.ScriptKana, 2, "説明書"), + thinPage(3), + page(4, "JA", doc.ScriptKana, 4, "説明書"), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: { + {Source: doc.SourcePageTag, Code: "JA", Lang: "ja", Start: 1, End: 2, Confidence: 1}, + {Source: doc.SourcePageTag, Code: "JA", Lang: "ja", Start: 4, End: 4, Confidence: 1}, + }, + } + + runs := doc.Reconcile(pages, bySource) + if len(runs) != 1 { + t.Fatalf("expected the illustration page to be bridged into 1 run, got %d: %+v", len(runs), runs) + } + if runs[0].Start != 1 || runs[0].End != 4 { + t.Errorf("run = %d-%d, want 1-4", runs[0].Start, runs[0].End) + } +} + +func TestReconcileFlagsInteriorDisagreementsOnly(t *testing.T) { + // An index start that is one page off disagrees about exactly one page: the + // last of the previous section. That is boundary noise, reported once per + // section elsewhere. A disagreement in the middle of a run is a real conflict. + pages := []doc.Page{ + page(1, "IT", doc.ScriptLatin, 1, "italiano"), + page(2, "IT", doc.ScriptLatin, 2, "italiano"), + page(3, "IT", doc.ScriptLatin, 3, "italiano"), + } + + boundary := map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "IT", Lang: "it", Start: 1, End: 3, Confidence: 1}}, + doc.SourceIndex: {{Source: doc.SourceIndex, Code: "ES", Lang: "es", Start: 3, End: 3, Confidence: 0.6}}, + } + for _, r := range doc.Reconcile(pages, boundary) { + if r.Conflict { + t.Errorf("a one-page disagreement at the run's edge should not flag a conflict: %s", r.Note) + } + } + + interior := map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "IT", Lang: "it", Start: 1, End: 3, Confidence: 1}}, + doc.SourceIndex: {{Source: doc.SourceIndex, Code: "ES", Lang: "es", Start: 2, End: 2, Confidence: 0.6}}, + } + runs := doc.Reconcile(pages, interior) + if len(runs) != 1 || !runs[0].Conflict { + t.Errorf("a disagreement inside the run must be flagged: %+v", runs) + } + if !strings.Contains(runs[0].Note, "index") { + t.Errorf("the note should name the disagreeing signal, got %q", runs[0].Note) + } +} + +func TestReconcileLeavesUnknowableLanguagesUnlabelled(t *testing.T) { + // A page nobody can name stays unnamed. Guessing would be worse than + // reporting that a statistical detector is needed. + pages := []doc.Page{page(1, "", doc.ScriptLatin, 1, "some latin prose")} + if runs := doc.Reconcile(pages, map[doc.Source][]doc.Run{}); len(runs) != 0 { + t.Errorf("expected no runs, got %+v", runs) + } + + // The control matters: an empty result also occurs when reconciliation is + // broken outright, so prove the same page IS labelled once a signal names it. + // Without this the assertion above passes against a Reconcile that returns + // nothing for every input. + withSignal := doc.Reconcile(pages, map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "EN", Lang: "en", Start: 1, End: 1, Confidence: 1}}, + }) + if len(withSignal) != 1 || withSignal[0].Lang != "en" { + t.Fatalf("the same page with a signal should be labelled en, got %+v", withSignal) + } +} + +func TestNormalizeCodeHandlesLabelsManualsActuallyPrint(t *testing.T) { + tests := []struct{ in, want string }{ + {"EN", "en"}, + {"UA", "uk"}, // country code used for Ukrainian + {"CZ", "cs"}, // country code used for Czech + {"ZH-HK", "zh-HK"}, + {"DK", "da"}, + {"JP", "ja"}, + {"de", "de"}, + } + for _, tc := range tests { + got, ok := doc.NormalizeCode(tc.in) + if !ok || got != tc.want { + t.Errorf("NormalizeCode(%q) = %q, %t; want %q, true", tc.in, got, ok, tc.want) + } + } + if _, ok := doc.NormalizeCode("QQ"); ok { + t.Error("NormalizeCode accepted QQ, which is not a language") + } +} + +func TestDominantScriptSeparatesJapaneseFromChinese(t *testing.T) { + // Japanese mixes kanji with kana and the kanji usually outnumber the kana, so + // a plain maximum reports Han and loses the distinction. + if got := doc.DominantScript("取扱説明書をお読みください"); got != doc.ScriptKana { + t.Errorf("Japanese text = %q, want %q", got, doc.ScriptKana) + } + if got := doc.DominantScript("用戶手冊請仔細閱讀本手冊"); got != doc.ScriptHan { + t.Errorf("Chinese text = %q, want %q", got, doc.ScriptHan) + } + if got := doc.DominantScript("123 456 !!!"); got != "" { + t.Errorf("text with no letters = %q, want empty", got) + } +} + +func TestScriptCompatibleChecksBothDirections(t *testing.T) { + tests := []struct { + script, lang string + want bool + }{ + {doc.ScriptGreek, "el", true}, + {doc.ScriptGreek, "de", false}, // script rules the language out + {doc.ScriptLatin, "ja", false}, // language rules the script out + {doc.ScriptLatin, "de", true}, + {doc.ScriptLatin, "sr", true}, // Serbian is written in both alphabets + {doc.ScriptCyrillic, "sr", true}, + {doc.ScriptKana, "ja", true}, + {"", "ja", true}, // no script evidence rules nothing out + } + for _, tc := range tests { + if got := doc.ScriptCompatible(tc.script, tc.lang); got != tc.want { + t.Errorf("ScriptCompatible(%q, %q) = %t, want %t", tc.script, tc.lang, got, tc.want) + } + } +} + +func TestScopeIntersectsWithTheHousehold(t *testing.T) { + res := &doc.Result{ + Info: doc.Info{Pages: 100}, + Pages: []doc.Page{ + {No: 1, Chars: 1000}, {No: 2, Chars: 1000}, + {No: 3, Chars: 1000}, {No: 4, Chars: 1000}, + }, + Runs: []doc.Run{ + {Source: doc.SourceReconciled, Code: "EN", Lang: "en", Start: 1, End: 2}, + {Source: doc.SourceReconciled, Code: "ZH-HK", Lang: "zh-HK", Start: 3, End: 4}, + }, + } + + scope := res.ScopeFor([]string{"en"}) + if len(scope.Languages) != 1 || scope.Languages[0].Lang != "en" { + t.Fatalf("in scope = %+v, want just en", scope.Languages) + } + if scope.Pages != 2 { + t.Errorf("scope pages = %d, want 2", scope.Pages) + } + if scope.Chars != 2000 { + t.Errorf("scope chars = %d, want 2000", scope.Chars) + } + if len(scope.OtherLanguages) != 1 { + t.Errorf("other languages = %+v, want zh-HK reported so it can be imported later", scope.OtherLanguages) + } + + // A regional variant satisfies a household that reads the base language. + if scope := res.ScopeFor([]string{"zh"}); scope.Pages != 2 { + t.Errorf("zh should match zh-HK, got %d pages", scope.Pages) + } +} + +// --- edge cases in run construction, pinned deliberately --- + +func TestReconcileBridgesAChainOfLowTextPages(t *testing.T) { + // Three runs of the same language separated by thin pages must collapse to + // one, not two. Merging keeps a pointer into the output slice and remaps a + // dispute map keyed by run index, so a chain is where that bookkeeping would + // go wrong. + pages := []doc.Page{ + page(1, "JA", doc.ScriptKana, 1, "説明書"), + thinPage(2), + page(3, "JA", doc.ScriptKana, 3, "説明書"), + thinPage(4), + page(5, "JA", doc.ScriptKana, 5, "説明書"), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: { + {Source: doc.SourcePageTag, Code: "JA", Lang: "ja", Start: 1, End: 1, Confidence: 1}, + {Source: doc.SourcePageTag, Code: "JA", Lang: "ja", Start: 3, End: 3, Confidence: 1}, + {Source: doc.SourcePageTag, Code: "JA", Lang: "ja", Start: 5, End: 5, Confidence: 1}, + }, + } + + runs := doc.Reconcile(pages, bySource) + if len(runs) != 1 { + t.Fatalf("expected 1 run after bridging a chain, got %d: %+v", len(runs), runs) + } + if runs[0].Start != 1 || runs[0].End != 5 { + t.Errorf("run = %d-%d, want 1-5", runs[0].Start, runs[0].End) + } +} + +func TestTwoPageTagVariantsAreTwoSectionsInEitherOrder(t *testing.T) { + // This test previously asserted the opposite, and was wrong. It gave both + // pages a *page tag* — one ZH, one ZH-HK — and required them to merge into a + // single section, which encoded the very defect review later found: the + // document naming two variants is the document distinguishing two sections, + // and merging them loses a real boundary. + // + // The case the merge rule genuinely exists for is a vaguer *script* signal + // filling a gap, which cannot express a region at all. That lives in + // reconcile_test.go as TestAVaguerScriptSignalStillContinuesASection. + for _, name := range []string{"specific first", "base first"} { + t.Run(name, func(t *testing.T) { + pages := []doc.Page{ + page(1, "", doc.ScriptHan, 1, "用戶手冊"), + page(2, "", doc.ScriptHan, 2, "用戶手冊"), + } + specific := doc.Run{Source: doc.SourcePageTag, Code: "ZH-HK", Lang: "zh-HK", Start: 1, End: 1, Confidence: 1} + base := doc.Run{Source: doc.SourcePageTag, Code: "CN", Lang: "zh", Start: 2, End: 2, Confidence: 1} + if name == "base first" { + specific.Start, specific.End = 2, 2 + base.Start, base.End = 1, 1 + } + + runs := doc.Reconcile(pages, map[doc.Source][]doc.Run{ + doc.SourcePageTag: {base, specific}, + }) + if len(runs) != 2 { + t.Fatalf("expected 2 runs — two printed tags name two variants, got %d: %+v", len(runs), runs) + } + for _, r := range runs { + if r.Pages() != 1 { + t.Errorf("run %s covers %d pages, want 1", r.Code, r.Pages()) + } + } + }) + } +} + +func TestIndexRunsNeverProduceAnInvertedRange(t *testing.T) { + // The schema requires pdf_end >= pdf_start, and a violation fails the whole + // probe — that already happened once. An index whose entries resolve out of + // order relative to their claimed pages is the way to provoke it. + contents := doc.Page{No: 1} + contents.Text = "Contents\nEN\nEnglish\n9\nDE\nGerman\n1\nFR\nFrench\n5\n" + contents.Chars = len([]rune(contents.Text)) + + pages := []doc.Page{contents} + for i, folio := range []int{1, 3, 5, 7, 9} { + pages = append(pages, page(i+2, "", doc.ScriptLatin, folio, "body text here")) + } + + for _, r := range doc.IndexRuns(pages) { + if r.Start == 0 { + continue // a claim that fixed no boundary is allowed + } + if r.End < r.Start { + t.Errorf("%s produced an inverted range %d-%d", r.Code, r.Start, r.End) + } + } +} + +func TestDegenerateInputsDoNotPanic(t *testing.T) { + // Empty, single-page, and all-thin documents must return an empty map rather + // than panicking or inventing a run. + cases := map[string][]doc.Page{ + "no pages": {}, + "one thin page": {thinPage(1)}, + "all thin": {thinPage(1), thinPage(2), thinPage(3)}, + "one good page": {page(1, "EN", doc.ScriptLatin, 1, "english text")}, + "non-contiguous": {page(1, "EN", doc.ScriptLatin, 1, "english"), page(9, "EN", doc.ScriptLatin, 9, "english")}, + } + for name, pages := range cases { + t.Run(name, func(t *testing.T) { + runs := doc.Reconcile(pages, map[doc.Source][]doc.Run{}) + for _, r := range runs { + if r.End < r.Start || r.Start < 1 { + t.Errorf("invalid run %+v", r) + } + } + if got := doc.IndexRuns(pages); got != nil { + for _, r := range got { + if r.Start != 0 && r.End < r.Start { + t.Errorf("invalid index run %+v", r) + } + } + } + _ = doc.TagRuns(pages, doc.EffectiveTags(pages, nil)) + _ = doc.ScriptRuns(pages) + }) + } +} + +func TestNonContiguousPagesAreTwoRuns(t *testing.T) { + // A document whose page 1 and page 9 share a tag has two runs, not one + // nine-page run — bridging only applies across pages that exist and are thin. + pages := []doc.Page{ + page(1, "EN", doc.ScriptLatin, 1, "english"), + page(9, "EN", doc.ScriptLatin, 9, "english"), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: { + {Source: doc.SourcePageTag, Code: "EN", Lang: "en", Start: 1, End: 1, Confidence: 1}, + {Source: doc.SourcePageTag, Code: "EN", Lang: "en", Start: 9, End: 9, Confidence: 1}, + }, + } + runs := doc.Reconcile(pages, bySource) + if len(runs) != 2 { + t.Errorf("expected 2 runs across a real gap, got %d: %+v", len(runs), runs) + } +} + +func TestARejectedIndexClaimEndsNothing(t *testing.T) { + // A claim the parser refused is not a boundary. This index lists AR at printed + // page 5, CZ at 7 and EN at 9; the Czech claim resolves onto a page that is + // Arabic script and is vetoed, exactly as a real manual's Czech-inside-Arabic + // typo is. Ending the Arabic section one page before that rejected claim cut the + // section in half: 3-4 instead of 3-6. + contents := doc.Page{No: 1} + contents.Text = "Contents\nAR\nArabic manual\n5\nCZ\nUzivatelska prirucka\n7\nEN\nUser Manual\n9\n" + contents.Chars = len([]rune(contents.Text)) + + pages := []doc.Page{ + contents, + page(2, "", doc.ScriptLatin, 0, "cover"), + page(3, "AR", doc.ScriptArabic, 5, "عربي"), + page(4, "AR", doc.ScriptArabic, 6, "عربي"), + page(5, "AR", doc.ScriptArabic, 7, "عربي"), + page(6, "AR", doc.ScriptArabic, 8, "عربي"), + page(7, "EN", doc.ScriptLatin, 9, "english"), + page(8, "EN", doc.ScriptLatin, 10, "english"), + } + + byCode := make(map[string]doc.Run, 3) + for _, r := range doc.IndexRuns(pages) { + byCode[r.Code] = r + } + ar, ok := byCode["AR"] + if !ok { + t.Fatal("no Arabic index entry was parsed") + } + if ar.Start != 3 || ar.End != 6 { + t.Errorf("Arabic = %d-%d, want 3-6: the section ends where the next *accepted* claim begins", + ar.Start, ar.End) + } + if cz := byCode["CZ"]; cz.Start != 0 { + t.Errorf("the Czech claim resolved to page %d; that page is Arabic, so it fixes no boundary", cz.Start) + } +} + +func TestThreeLetterLabelsParseAsTheirOwnEntry(t *testing.T) { + // Manufacturers print three-letter labels — POR, SPA, CHI, RUS, KAZ. They were + // once unrecognised, so such a line was read as title text and the walk + // continued into the FOLLOWING entry's page number: EN claimed the Portuguese + // section's start page and carried its title along too. + // + // This test previously asserted only that the damage did not happen, because + // three-letter codes could not be parsed. They can now — a real manual marks + // two of its five languages RUS and KAZ — so the entry belongs to POR, with its + // own title and page, and EN keeps its own. + contents := doc.Page{No: 1} + contents.Text = "Contents\n" + + "EN\nUser Manual\n1\nPOR\nManual do utilizador\n17\n" + + "FR\nManuel d'utilisation\n33\n" + + "DE\nBenutzerhandbuch\n49\n" + + "ES\nManual de usuario\n65\n" + contents.Chars = len([]rune(contents.Text)) + + pages := []doc.Page{ + contents, + page(2, "", doc.ScriptLatin, 0, "body"), + page(3, "", doc.ScriptLatin, 0, "body"), + } + + byCode := make(map[string]doc.Run, 5) + for _, r := range doc.IndexRuns(pages) { + byCode[r.Code] = r + } + + por, ok := byCode["POR"] + if !ok { + t.Fatal("POR was not parsed as an entry") + } + if por.Lang != "pt" { + t.Errorf("POR resolved to %q, want pt", por.Lang) + } + if !strings.Contains(por.Title, "utilizador") { + t.Errorf("POR title = %q, want the Portuguese one", por.Title) + } + if por.PrintedPage == nil || *por.PrintedPage != 17 { + t.Errorf("POR claims %v, want printed page 17", por.PrintedPage) + } + + // EN must keep its own page and title rather than the next entry's. + en, ok := byCode["EN"] + if !ok { + t.Fatal("EN was not parsed") + } + if en.PrintedPage == nil || *en.PrintedPage != 1 { + t.Errorf("EN claims %v, want printed page 1", en.PrintedPage) + } + if strings.Contains(en.Title, "utilizador") { + t.Errorf("EN absorbed the Portuguese title: %q", en.Title) + } + + // And the rest of the table still parses. + for code, want := range map[string]int{"FR": 33, "DE": 49, "ES": 65} { + r, ok := byCode[code] + if !ok { + t.Errorf("%s was not parsed at all", code) + continue + } + if r.PrintedPage == nil || *r.PrintedPage != want { + t.Errorf("%s claims %v, want printed page %d", code, r.PrintedPage, want) + } + } +} + +func TestATagOnALatinPageIsNotCorroboratedByIt(t *testing.T) { + // Two pages tagged JA whose CJK glyphs failed to extract leave nothing but Latin + // furniture behind. A Latin page permits any language — that is what the script + // signal means by "no information" — so a one-directional check read it as + // corroboration and stored confidence 1.0, while reconciliation, which checks + // both directions, discarded the run and left the section unlabelled. Maximum + // confidence for a section nothing believes in is the worst of both answers. + pages := []doc.Page{ + page(1, "JA", doc.ScriptLatin, 1, "L40 Ultra"), + page(2, "JA", doc.ScriptLatin, 2, "L40 Ultra"), + } + + runs := doc.TagRuns(pages, nil) + if len(runs) != 1 { + t.Fatalf("expected 1 run, got %d: %+v", len(runs), runs) + } + if runs[0].Confidence > 0.3 { + t.Errorf("confidence = %.1f (%s); Japanese is not written in the Latin alphabet", + runs[0].Confidence, runs[0].Note) + } +} + +func TestIndexEntriesClaimingOnePageKeepPrintedOrder(t *testing.T) { + // Two entries claiming the same printed page must stay in the order the contents + // table printed them. Sorting them unstably left the order to the sort's + // internals, and reconciliation keeps whichever claim it sees last — so which + // language those pages were said to be in depended on nothing in the document. + // + // Thirteen entries, because Go's sort is stable in effect on very short slices; + // the entry with a wildly out-of-order claim is the typo shape a real contents + // table has. + type listed struct { + code, title string + printed int + } + table := []listed{ + {"EN", "User Manual", 1}, + {"DE", "Benutzerhandbuch", 17}, + {"FR", "Manuel d'utilisation", 33}, + {"IT", "Manuale utente", 49}, + {"ES", "Manual de usuario", 65}, + {"PL", "Instrukcja obslugi", 65}, // the same page as ES + {"NL", "Handleiding", 97}, + {"NO", "Brukerhandbok", 113}, + {"SV", "Bruksanvisning", 129}, + {"EL", "Odigies chrisis", 7}, // a typo: page 7 is inside the English section + {"PT", "Manual do utilizador", 161}, + {"HE", "Hebrew manual", 177}, + {"AR", "Arabic manual", 193}, + } + + var b strings.Builder + b.WriteString("Contents\n") + for _, e := range table { + fmt.Fprintf(&b, "%s\n%s\n%d\n", e.code, e.title, e.printed) + } + contents := doc.Page{No: 1, Text: b.String()} + contents.Chars = len([]rune(contents.Text)) + + pages := []doc.Page{ + contents, + page(2, "", doc.ScriptLatin, 0, "body"), + page(3, "", doc.ScriptLatin, 0, "body"), + } + + order := make(map[string]int, len(table)) + runs := doc.IndexRuns(pages) + for i, r := range runs { + order[r.Code] = i + } + es, haveES := order["ES"] + pl, havePL := order["PL"] + if !haveES || !havePL { + t.Fatalf("expected both ES and PL entries, got %d runs", len(runs)) + } + if es > pl { + t.Errorf("ES and PL both claim page 65 and the table lists ES first, "+ + "but they came out in the order PL (%d) then ES (%d)", pl, es) + } +} + +func TestUnlabelledSeesPagesOutsideTheLabelledSpan(t *testing.T) { + // A 120-page manual printing no tags and carrying no parseable index: script + // names the Greek section and nothing else, because 25 languages share the Latin + // alphabet. Deriving the content range from the runs made the count circular — + // the range became 51-70 and the count ran only inside it, so a document of which + // 100 pages are unnameable reported none. Unlabelled is the number that says + // whether a statistical detector would earn its 118 MB, so a structural zero is + // the one answer it must never give. + pages := make([]doc.Page, 0, 120) + for no := 1; no <= 120; no++ { + if no >= 51 && no <= 70 { + pages = append(pages, page(no, "", doc.ScriptGreek, no, "οδηγίες χρήσης")) + continue + } + pages = append(pages, page(no, "", doc.ScriptLatin, no, "latin prose nobody can name")) + } + + runs := doc.Reconcile(pages, map[doc.Source][]doc.Run{doc.SourceScript: doc.ScriptRuns(pages)}) + + // The count is the point, and it must no longer be bounded by the range. + if got := doc.CountUnlabelled(pages, runs); got != 100 { + t.Errorf("unlabelled = %d, want 100", got) + } + // The range reports what the language map covers, which here really is only + // the Greek section — the other 100 pages are content nobody could name, and + // CountUnlabelled above is what says so. + if start, end := doc.ContentRange(pages, runs); start != 51 || end != 70 { + t.Errorf("content range = %d-%d, want 51-70", start, end) + } +} + +func TestContentRangeStillExcludesFrontMatterAndABackCover(t *testing.T) { + // The runs stay the better evidence at both ends when what they exclude is + // plausibly furniture: on the measured fixture six pages of front matter and an + // English colophon carry text without being content. This is what counting + // unlabelled pages honestly must not trade away. + pages := make([]doc.Page, 0, 30) + for no := 1; no <= 6; no++ { + pages = append(pages, page(no, "", doc.ScriptLatin, 0, "front matter")) + } + for no := 7; no <= 29; no++ { + pages = append(pages, page(no, "EL", doc.ScriptGreek, no-6, "οδηγίες")) + } + pages = append(pages, page(30, "", doc.ScriptLatin, 0, "Made in China")) + + runs := doc.Reconcile(pages, map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "EL", Lang: "el", Start: 7, End: 29, Confidence: 1}}, + }) + start, end := doc.ContentRange(pages, runs) + if start != 7 || end != 29 { + t.Errorf("content range = %d-%d, want 7-29", start, end) + } + // Front matter and a colophon carry text and belong to no section, so they + // are counted — deliberately. A small non-zero count is the honest answer; + // suppressing it is what made the number structurally unable to report a + // problem. Six front pages plus one back cover. + if got := doc.CountUnlabelled(pages, runs); got != 7 { + t.Errorf("unlabelled = %d, want 7 (six front-matter pages and a back cover)", got) + } +} + +func TestScopeCountsCharsOfEveryRunOfAHouseholdLanguage(t *testing.T) { + // A language's summary carries one printed code — the most specific tag wins a + // contest between them — while each run carries its own. Counting characters by + // the summary's code therefore counted only the runs whose label happened to win: + // the CN and ZH-HK sections both put their pages in scope, but only one of them + // contributed any characters, and the character count is what the pre-flight gate + // turns into a price. + res := &doc.Result{ + Info: doc.Info{Pages: 6}, + Pages: []doc.Page{ + {No: 1, Chars: 1000}, {No: 2, Chars: 1000}, {No: 3, Chars: 1000}, + {No: 4, Chars: 1000}, {No: 5, Chars: 1000}, {No: 6, Chars: 1000}, + }, + Runs: []doc.Run{ + {Source: doc.SourceReconciled, Code: "CN", Lang: "zh", Start: 1, End: 2}, + {Source: doc.SourceReconciled, Code: "JA", Lang: "ja", Start: 3, End: 4}, + {Source: doc.SourceReconciled, Code: "HK", Lang: "zh-HK", Start: 5, End: 6}, + }, + } + + scope := res.ScopeFor([]string{"zh"}) + if scope.Pages != 4 { + t.Errorf("scope pages = %d, want 4", scope.Pages) + } + if scope.Chars != 4000 { + t.Errorf("scope chars = %d, want 4000: every Chinese run's pages are in scope", scope.Chars) + } +} + +func TestAnUnplaceableRunCoversNoPages(t *testing.T) { + // The printed index names languages it cannot place — the measured fixture's HE, + // AR and CZ entries all resolve to nothing. A start of 0 means unplaceable, not + // page zero, and the arithmetic span turned each of them into a one-page section + // spanning 0-0. + unplaceable := doc.Run{Source: doc.SourceIndex, Code: "AR", Lang: "ar"} + if got := unplaceable.Pages(); got != 0 { + t.Errorf("an unplaceable run covers %d pages, want 0", got) + } + placed := doc.Run{Source: doc.SourceIndex, Code: "EN", Lang: "en", Start: 7, End: 22} + if got := placed.Pages(); got != 16 { + t.Errorf("run 7-22 covers %d pages, want 16", got) + } +} + +// TestAnAddressPageIsNotAContentsTable is the measured failure this guards. +// +// The column manual's back page prints service addresses for six countries. The +// index parser read it as the document's contents table — the ONLY page of that +// manual it read at all — and produced VIA from "Via Monte Rosa" claiming pages +// 28-45, FAX claiming 46-48, and UA from a Ukrainian postal address claiming 49-68 +// with the title "Telefax". Those reached the user as "68 pages in 2 languages, +// none of them yours. It has fax and Ukrainian." +// +// A code-shaped token is not enough. It must name a language something recognises. +func TestAnAddressPageIsNotAContentsTable(t *testing.T) { + // The shape of that page: a country or label token alone on a line, then an + // address line, then a number that looks like a page reference. + addresses := doc.Page{No: 68} + addresses.Text = strings.Join([]string{ + "Robert Thomas GmbH", "Service", + "VIA", "Monte Rosa", "28", + "FAX", "", "46", + "UA", "Telefax", "49", + "Z", "o.o. Telefon", "90", + "NDE", "Tel. 555 0100", "4931", + }, "\n") + addresses.Chars = len([]rune(addresses.Text)) + + if doc.IsContentsPage(&addresses) { + t.Error("a page of postal addresses is treated as the document's contents table") + } + if runs := doc.IndexRuns([]doc.Page{addresses}); len(runs) != 0 { + t.Errorf("it produced %d index entries:", len(runs)) + for _, r := range runs { + t.Errorf(" code=%q lang=%q title=%q", r.Code, r.Lang, r.Title) + } + } +} + +// TestARealContentsTableStillParses is the other side of that guard. The sectioned +// manual's contents pages are the reason the index signal exists at all: they supply +// localised section titles no other signal can. +func TestARealContentsTableStillParses(t *testing.T) { + contents := doc.Page{No: 2} + contents.Text = strings.Join([]string{ + "Contents", + "EN", "User Manual", "1", + "DE", "Bedienungsanleitung", "17", + "UA", "Посібник користувача", "33", + "CZ", "Návod k použití", "49", + }, "\n") + contents.Chars = len([]rune(contents.Text)) + + if !doc.IsContentsPage(&contents) { + t.Fatal("a real contents table was not recognised") + } + runs := doc.IndexRuns([]doc.Page{contents}) + if len(runs) != 4 { + t.Fatalf("got %d index entries, want 4", len(runs)) + } + // Including the codes real manuals print that are not valid tags on their own. + byCode := make(map[string]string, 4) + for _, r := range runs { + byCode[r.Code] = r.Lang + } + for code, want := range map[string]string{"EN": "en", "DE": "de", "UA": "uk", "CZ": "cs"} { + if byCode[code] != want { + t.Errorf("%s normalised to %q, want %q", code, byCode[code], want) + } + } + if runs[0].Title == "" { + t.Error("no title survived; titles are what only the index can supply") + } +} diff --git a/internal/doc/figures.go b/internal/doc/figures.go new file mode 100644 index 0000000..b2727cc --- /dev/null +++ b/internal/doc/figures.go @@ -0,0 +1,2224 @@ +package doc + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "math" + "os/exec" + "slices" + "sort" + "strconv" + "strings" + + "github.com/gordon2/manualbox/internal/extern" +) + +// A picture is found from the shapes the page draws, and on both fixtures it is +// not an embedded image at all. +// +// That is the measurement this file exists on top of, and it inverts the obvious +// plan. `pdfimages` extracts embedded rasters, and over 628 pages of two real +// manuals it yields **not one illustration**: +// +// embedded images on pages what they are +// 68-page columns manual 1,358 31 of 68 see below +// 560-page sequential manual 54 4 of 560 certification badges +// +// Of the columns manual's 1,358, exactly 1,301 are on pages 11 and 12 — 650 and +// 651 tiny 12x4-to-13x3 slivers in separation and indexed colour spaces, the mesh +// a gradient decomposes into. Of the remaining 57, one object (a 97x73 grey JPEG, +// the corner logo) accounts for 30, appearing once on each of pages 2-6, 15-51 odd +// and 52, 57, 62; the rest are the cover's five 88x52 icons, an RGB award badge and +// three CCITT wordmark stencils. The sequential manual's 54 sit only on pages 530, +// 531, 551 and 552 and run from 3x3 to 212x72 pixels: CE marks and recycling +// symbols. Neither set contains a diagram. +// +// Meanwhile the pictures a reader would name are all vector. Page 42 of the +// columns manual prints four framed line drawings of the appliance and reports +// **zero** embedded images; so do pages 18, 20, 22, 28, 34 and 44. `pdfimages -list` +// on page 57 reports one 97x73 grey JPEG, which is the corner logo and not that +// page's artwork — and `pdftohtml -xml` without -i agrees, emitting exactly one +// there, at top=5 left=20, while pages 22 and 42 get none at all. +// +// So the geometry comes from the same place the ruled lines do — `pdftocairo -svg`, +// read with the walker rules.go already has — and what is collected is every drawn +// shape's bounding box rather than only the axis-aligned thin ones. The bytes then +// come from `pdftoppm`, rendering the rectangle that geometry found. +// +// Two consequences of taking the vector route are real and are not worked around: +// +// **An embedded raster is invisible here.** parseSVG drops structurally, +// so a manual whose illustrations are photographs finds nothing. That is accepted +// rather than overlooked, on the measurement above: on the two documents this +// project has, every embedded raster is furniture and every illustration is drawn. +// A photographic manual needs the raster path added, and `pdfimages -list` plus +// pdftohtml's elements are where its geometry would come from — noting +// that pdftohtml writes the raster to a file beside the blob store unless it is +// run with -i, which is why [ExtractRuns] passes -i. +// +// **A shape's box is its visible extent, because the clip is read.** This was the +// one real cost of the vector route and it is now paid: clip.go resolves each +// shape's effective clip and [inkWalker.add] intersects the path's extent with it, +// so a drawing whose artwork runs past its frame is recorded at the frame. +// +// What that was worth, measured end to end with `manualbox verify` on both +// manuals: +// +// columns manual sequential manual +// figures 46 -> 59 163 -> 168 +// figures cut off by their crop 22 -> 15 74 -> 71 +// figures with a blank band 4 -> 0 6 -> 2 +// +// The figure count rises because the unclipped extent *merged neighbouring +// pictures*, and the pages that were counted by eye now agree with the print: page +// 42 of the columns manual returns its four framed drawings where it returned +// three, page 22 three for three, and page 16 four for four — which the render +// settles and conversion.md had wrong twice over, since that page prints four +// panels rather than the three it records. Both documents keep the same number of +// pages carrying figures, which is what says these are splits rather than newly +// admitted furniture — 27 and 23, counted over what [FindFigures] returns for every +// page. +// +// **A candidate whose box overlaps another's is a piece of it.** Reading the clip +// split drawings that had been merged; it also revealed that some of them had been +// in pieces all along, because a group's box is the union of its shapes and can +// cover a neighbouring group entirely without any shape of the two touching. See +// [mergeOverlapping], which is the second pass that fixes it, and what it was worth +// end to end: +// +// columns manual sequential manual +// figures 59 -> 59 168 -> 134 +// figures cut off by their crop 3 -> 3 70 -> 25 +// figures with a blank band 0 -> 0 2 -> 2 +// +// The columns manual does not move at all, at any merge threshold: it has no page +// where two candidates overlap. On the sequential manual the pages carrying figures +// again do not move, and the clipped count falls by two thirds, because a piece of a +// drawing is crossed by the shapes of the piece beside it. +// +// Note the level: the tables above are what `manualbox verify` converts, and +// conversion keeps 134 of the sequential manual's 195 figures, landing on 20 of +// those 23 pages. Both counts are pinned, in TestGuardSweep and in verify's +// fixture tests respectively. +// +// The residual counts were not the clip either, and most of the columns manual's +// have since gone: 15 of them were [trimToPicture] cutting into a drawing that had +// a label at its edge, and teaching the trim to leave a label the artwork encloses +// alone took that document to 3. What remains is three classes, none of them the +// clip. Pages 11 and 12 of the columns manual report one shape crossing out of +// 2,741, which is a page-sized path the geometric matching in `internal/verify` +// cannot attribute; page 1 is the cover, whose artwork genuinely runs behind the +// title block the trim excludes; and the sequential manual's 25 are leader lines on +// its crowded diagram pages, where a line more than half inside one figure's box +// belongs to the drawing beside it. See the note on [trimToPicture]. +// +// Nothing here emits a block and nothing here writes to the blob store. This file +// answers only "where are the pictures, and what are their bytes"; the digest is +// carried because the store is content-addressed and that is where they go next. + +// Bounds on what counts as a picture, all measured against the two fixtures in +// the 1.5-scaled space [ExtractRuns] documents. +// +// The two guards below are the same pair rules.go needs and for the same reason: +// a shape guard alone keeps page furniture, and a text guard alone keeps a ruled +// table. What is different is which shape signal works. Area does not: the +// smallest real illustration measured is 0.15% of its page (34x22 units, one of +// the thirty small diagrams on page 5 of the sequential manual) while the printed +// D and PL language badges the columns manual repeats on 110 pages are 0.5% and its +// cover ornament is 26.4%, so no area threshold separates them — a threshold that +// admits the diagram admits both. Ink volume does, by an order of magnitude: see +// [minFigureInk]. +const ( + // figureDPI is what a figure is rendered at. Twice the 108 dpi the run + // coordinates are in, chosen for that ratio and not for a picture quality + // target: the crop rectangle poppler wants is in output pixels, so an exact + // factor of two turns a rect in this package's space into a crop with no + // rounding to argue about. Measured output: the columns manual's largest + // figure, page 11's parts diagram, is 1077x1510 pixels and 353 KB of PNG. + figureDPI = 216 + + // figureScale is figureDPI over the 108 dpi of [PageRuns]. Not a tunable. + figureScale = figureDPI / 108 + + // minFigureWidth and minFigureHeight are how small a picture may be, and this + // is a soft cut with no gap to put it in — the same shape of problem + // docs/design/conversion.md records for a heading's share of the measure, and + // it is recorded here rather than presented as a threshold. + // + // The sweep is in TestGuardSweep and says two things. On the columns manual the + // guard discriminates *nothing whatever*: every value from 10 to 120 returns the + // same 59 figures, because that document draws no picture smaller than 128 units + // on its short side. On the sequential manual it is a smooth continuum with no + // step anywhere — 293 figures at 10, 238 at 20, 201 at 30, 161 at 40, 127 at 50, + // 93 at 60, 52 at 80, 15 at 120. + // + // So the value is chosen by looking at what falls out, and 40 was wrong. Page 5 + // of the sequential manual is a grid of nine panels holding about thirty small + // diagrams, and at 40 sixteen of them are lost — three were rendered and looked + // at: a 32x26 cutaway of the base station, a 32x32 wheel, a 34x22 robot under a + // hand. All three are pictures a reader would want. 20 units is about one line of + // body text on either document (17 units at the columns manual's 14pt, 17 at the + // sequential's 11pt), which is the smallest thing that can be a picture rather + // than a mark, and it is what the guard is actually for: a dashed rule is a + // hundred small shapes and would otherwise pass [minFigureInk]. + // + // The cost of going lower is not measured on these documents and is stated + // rather than dismissed: at 10 the sequential manual gains 52 more clusters, + // which were not inspected. + minFigureWidth = 20.0 + minFigureHeight = 20.0 + + // minFigureInk is how many drawn shapes an area must contain to be a picture + // rather than a piece of page furniture, and it is the guard that does the work. + // + // What it separates is not size but complexity, and the furniture measured here + // is uniformly two or three shapes: the columns manual's printed D and PL + // language badges are 3 each and appear on 110 pages, its cover wordmark is 1, + // its three award badges 2, 3 and 5. The case that settles it is the sequential + // manual's cover, where a single grey decorative swash covers 26.4% of the page — + // the largest cluster there by a wide margin, which every area threshold accepts + // as that page's picture — and is exactly 1 shape. + // + // The value is chosen off the sweep in TestGuardSweep rather than off a gap, + // because there is no gap: the counts fall smoothly, 291 / 286 / 244 / 238 / 236 + // on the sequential manual at 10 / 15 / 20 / 25 / 30. 25 is still the one value + // in that range that is not on a step — five either side moves both documents by + // under 3%, where 20 gains 20% on a step down to 15 — and that stability is + // asserted rather than described. Reading the clip moved every number in that + // sweep and moved neither the shape of it nor the value chosen. + minFigureInk = 25 + + // maxFigureTextFraction is how much of a candidate's area may be covered by + // text before it is a table rather than a picture, as a fraction. + // + // The same discrimination rules.go's [tableHasText] makes, from the other side. + // The separation is wide — no accepted figure of either document is over 9% + // text, while page 57 of the columns manual draws two tables that are 37.4% and + // 38.7% — and 0.15 sits in the middle of it. + // + // It is also, measured, nearly dead, and that is stated rather than left to be + // discovered. [trimToPicture] took most of its work: a candidate that has reached + // over a column of prose now has the prose trimmed off instead of being rejected + // whole, which is the better outcome. What is left is one decision in 297 figures + // across both documents — page 53 of the sequential manual, the French recycling + // label, which is a picture with a paragraph inside it and therefore a loss + // rather than a save. TestWhatTheTextGuardIsStillWorth holds both numbers. + // + // It is kept on the reasoning [ruleWalker.filled] sets out for a branch measured + // to be worth almost nothing: a ruled table with more parts than [minFigureInk] + // and cells full of text is an ordinary thing for a document to contain, the + // guard is three lines, and the next manual gets no say in which cases this code + // understands. What it cannot catch either way is an *empty* table, and that is + // measured too — see the note at the end of this file. + maxFigureTextFraction = 0.15 + + // washFraction is how wide a drawn shape may be, as a fraction of the page, + // before it is read as a background rather than as part of a picture. It is + // structural rather than fitted: the columns manual paints its alternating + // section bands as filled rects exactly the page's width — 891.8 on an 892-unit + // page — and a band touches every picture it runs behind, so leaving them in + // merges the whole page into one cluster. 0.98 admits the band and nothing real: + // the widest figure measured is 807 units on a 918-unit page, 0.88. + washFraction = 0.98 + + // figureMergeOverlap is how much of the smaller of two candidate boxes may lie + // inside the other before the two are read as one picture, as a fraction. + // + // Zero: any positive overlap merges, and a box that merely touches another does + // not. The measurement behind that — 53 overlapping pairs over both documents, + // every one of them a single drawing that clustered in pieces, and no pair + // anywhere that needs the opposite answer — is at [mergeOverlapping]. + figureMergeOverlap = 0.0 + + // maxFigureClusterInk caps how many drawn shapes one page's clustering will + // consider, because the clustering is quadratic in them and a real page reaches + // six figures. Page 42 of the columns manual returns 165,759 shapes, 83,014 of + // them on-page, because its water-spray gradients are meshes of tens of + // thousands of hairlines; page 5 of the sequential manual returns 16,014. + // 200,000 is above the largest measured and bounds the pass at a few seconds + // rather than minutes. + maxFigureClusterInk = 200_000 + + // trimReachSlack is how far a line of text may poke out of a candidate's box + // and still count as printed inside it, in units. Zero: the comparison is exact. + // + // A tolerance is the obvious thing to want here, because the two rectangles come + // from two tools — the text box from pdftohtml, the ink box from pdftocairo — and + // a unit is what this project allows elsewhere when it compares one measurement + // of a drawing against another. It is not free, and that is why it is not taken. + // Swept at 0, 1 and 2 over both documents, the columns manual does not move at + // all; the sequential one loses a trim at 1, on page 523, where a two-line Russian + // caption clears the box's top edge by 0.5 units and so stops being seen as + // reaching over it. Its own right-hand exclusion is blocked by [maxTrimFraction], + // so the tolerance is the difference between excluding that caption and keeping + // it. Nothing is gained anywhere in exchange, so the exact test stands. + trimReachSlack = 0.0 + + // labelTerminator is how large a leader's end mark may be, in units, and it is + // the signal the whole of [growToLabels] turns on. + // + // Both documents draw one: a small open circle where a leader line stops, just + // short of the label it points at. Measured, they are 3.3 to 3.4 units square on + // page 521 of the sequential manual and 3.4 on its plate pages. 8 is chosen to + // admit a mark of twice that with a stroke's width on top, and the sweep in + // TestGrowSweep says the value is not on a cliff: at 4, 6, 8 and 12 the + // sequential manual grows 48, 53, 55 and 66 figures. 12 is where it starts + // admitting a drawing's own small details as terminators, and 4 misses marks that + // carry a stroke. + labelTerminator = 8.0 + + // labelAlign is how far off a run's midline a terminator may sit, in units. + // A leader points AT its label, so the mark and the label's middle line up; 4 is + // about a third of a line of body text on either document (12 to 14 units). + // Swept: 2, 4, 6 and 8 grow 48, 55, 59 and 62 figures of the sequential manual, + // and the overlapping crops it creates go 11, 11, 14, 14 over that range. + labelAlign = 4.0 + + // labelCorridor is how far outside a figure's edge a label may sit and still be + // that figure's, in units. + // + // Measured on page 521 of the sequential manual, where the labels of three + // drawings sit 0.1 to 35.3 units out: the box's edge is the leader's terminator, + // so the near ones are 3 units away, and the far ones are labels whose leader + // ends short of the drawing. 40 covers all of them. It cannot be much tighter: + // at 20 the underside diagram's six left labels are out of reach. It must not be + // much wider either, and there are two measurements for that rather than a + // preference — the parts list in [growToLabels], and the overlapping crops, which + // stay on the two front-matter plates at 40 and 60 and reach page 524, a page a + // reader is served, at 80. + labelCorridor = 40.0 + + // maxLabelGrowth is how far one edge may move to take in labels, as a fraction + // of the side it is on. One: an edge may not move further than the drawing's own + // width or height. + // + // IT IS NOT THE SHIPPED SETTING. [defaultGuards] sets growth to zero, because + // [figureLabels] carries every label as text and a crop that also contained them + // would print each one twice. This constant is what TestGrowSweep and the test-only + // `grownGuards` restore in order to keep measuring the pass this replaced — the same + // reason TestAPlateMergeOnSharedLabelsIsRefused is kept. Nothing in the shipped path + // reads it. + // + // Swept over the sequential manual as figures grown / labels taken: 18/51 at + // 0.25, 32/107 at 0.5, 55/229 at 1, 63/255 at 2 and 64/262 with no cap at all, + // where the largest single growth reaches 3.56 of a side. 1 is where the + // document's own labelled diagrams are all served — page 521's three drawings + // need 0.26, 0.68 and 0.65 — and it is a bound with a meaning rather than a + // fitted number: past it the labels are larger than the picture and the crop is + // no longer a picture with its labels. + maxLabelGrowth = 1.0 + + // maxFigurePNGBytes caps one rendered figure held in memory. Measured over + // every figure of both fixtures the largest is 353 KB, page 11's parts diagram + // at 1077x1510; 32 MB is two orders above that and still bounds a page-sized + // crop of a hostile document at 216 dpi. + maxFigurePNGBytes = 32 << 20 +) + +// errFigureTooLarge is returned when a rendered figure exceeds the cap. +var errFigureTooLarge = errors.New("doc: rendered figure exceeds the size limit") + +// Ink is one drawn shape's bounding box, in the same 1.5-scaled coordinate space +// as [PageRuns] and a `pdftoppm -r 108` raster. +// +// It is deliberately only a box. A picture is recognised by where its shapes are +// and how many there are, never by what they draw, and carrying the path would +// invite a caller to ask a question this file cannot answer — the flattening +// [subpaths] does means the box is exact only for straight edges. +type Ink struct { + // Rect is the shape's extent. + Rect CellRect `json:"rect"` + // Stroked reports that this came from a stroked path rather than a fill, the + // same distinction [Rule.Filled] records and for the same reason: a wrong + // answer can be traced back to the half of the walker that produced it. + Stroked bool `json:"stroked,omitempty"` +} + +// Figure is one illustration found on a page: where it is, and its bytes. +type Figure struct { + // Page is the 1-based page number in the original PDF. + Page int `json:"page"` + // Index is the figure's position in the page's reading order, from 0, sorted + // down then across. It is not a document-wide figure number: nothing here has + // the whole document in view, and numbering across pages is the caller's. + Index int `json:"index"` + // Rect is what was rendered, in the 1.5-scaled space: the band [labelBand] cuts, + // which is the drawing together with everything the claim rule reaches for it. + // Carried beside the bytes because it is half the answer — a picture has to land + // in the right place in a column's reading order — and it is the rectangle + // PixelWidth and PixelHeight describe, so a caller scaling the pixels back onto + // the page is scaling them onto this. + Rect CellRect `json:"rect"` + // InkRect is the drawn extent alone, before any label was taken in, and it is + // what the two guards judged. + // + // It is carried separately because one caller must not use Rect: [attribute] + // decides which language a picture belongs to by which region its box lies + // inside, and a box grown sideways onto a label could reach out of its own + // column and be served to every household — which is the one failure the funnel + // may not have. The language question is asked of the drawing, the crop is what + // a reader is shown. + // + // Not stored. Nothing reading a figure back out of the database asks the + // language question again; it was answered when the conversion was made. + InkRect CellRect `json:"inkRect,omitzero"` + // Ink is how many drawn shapes the figure holds — the shape guard's evidence, + // kept rather than reduced to the verdict, so a rejected page can be shown to + // have been rejected for the right reason. + Ink int `json:"ink"` + // TextFraction is how much of the figure's area is covered by text, the text + // guard's evidence. + TextFraction float64 `json:"textFraction"` + // DPI, PixelWidth and PixelHeight describe the render. The pixel size is read + // back out of the PNG rather than computed, so a caller comparing it against + // Rect is comparing what poppler did with what was asked for. + DPI int `json:"dpi"` + PixelWidth int `json:"pixelWidth"` + PixelHeight int `json:"pixelHeight"` + // Labels are the callout labels a leader points at, as printed, in the page's own + // order: down, then across. Empty for a picture nothing points at, which is most + // of them. + // + // ONE ENTRY IS ONE PRINTED LINE, so a wrapped label is several: page 521's + // `Вентиляционное отверстие системы автоопорожнения` is three. That is + // [claimLabels]' own unit — a continuation is its own claim — and it did not show + // while the reader placed each line against the drawing, because the paper's + // arrangement put them back together. It shows now, in an alt text that reads three + // labels where the page prints one. Joining a chain back up is a real improvement + // and is not done, because the chain is not recorded: [continuesLabel] answers the + // question one pair at a time and nothing keeps which line continued which. + // + // These are the runs [claimLabels] claims, filtered by the one test that makes the + // claim safe — see [figureLabels]. + // + // THEY ARE INSIDE Rect, and that is the whole of the current design. The crop is + // the band the page laid the drawing and its labels out in, so a caller that draws + // the picture is already showing every one of these; what the strings are for is + // everything that cannot read pixels — a screen reader, and later search and + // translation. TestALabelReachesAReaderWhole asserts the containment, which is the + // invariant that replaced a position. + // + // No position, and no side. Carrying one is what the reader used to re-lay the page + // out from, and re-laying out an arrangement the paper had already solved is what + // produced every defect in this area: labels colliding on page 522, wrapped tails + // orphaned, and each figure placed with no idea where its neighbour's labels were. + Labels []string `json:"labels,omitempty"` + + // LabelBoxes is where each of Labels was printed, in page coordinates, in the + // same order. + // + // NEVER SERIALISED AND NEVER STORED, which the `json:"-"` says and this says + // again because it is the point: the crop already contains these labels, so a + // stored position would be a second representation of something the picture + // carries, and two representations of one fact are how they come to disagree. + // This exists only to hand one pass's answer to the next inside this package — + // [findFigures] fills it and [Callouts.Mark] reads it, both in the same process + // on the same value — and it is exported only because the test that drives Mark + // has to be able to build one. + LabelBoxes []CellRect `json:"-"` + // Digest is the lowercase hex SHA-256 of PNG. The blob store's filename is the + // SHA-256, so this is the name these bytes will have if a later stage stores + // them — this file deliberately stores nothing. + Digest string `json:"digest"` + // PNG is the rendered figure. + PNG []byte `json:"-"` +} + +// DrawnExtent is the figure's drawn box: [Figure.InkRect] when it is known, and +// [Figure.Rect] when it is not. +// +// The fallback is there for the two callers that legitimately have no ink box. A +// figure read back out of the database has only the rectangle that was stored, +// because the drawn extent is not stored — nothing reading a conversion asks the +// language question again. And a figure built by hand in a test states the box it +// is about. Both mean the same thing when nothing has been grown, which is why +// this is a fallback rather than an error: before [growToLabels] the two rects +// were one rect. +func (f *Figure) DrawnExtent() CellRect { + if f.InkRect == (CellRect{}) { + return f.Rect + } + return f.InkRect +} + +// ExtractInk reads every shape one page draws, as bounding boxes. +// +// Like [ExtractRules] it never mutates the file and calls nothing remote, so it +// is a pure function of the bytes and safe to re-run, which is what lets the job +// that calls it be idempotent. One page per invocation, forced by the same +// property of `pdftocairo -svg` [ExtractRules] records. +// +// A caller that wants both the rules and the ink of a page pays for pdftocairo +// twice. That is left as it is rather than fused: the two halves are built +// separately and joined later, which is the split that let the table work and the +// block work proceed without agreeing on anything, and the fusion is one function +// away once there is a caller that needs both. +func ExtractInk(ctx context.Context, path string, page int) ([]Ink, error) { + if page < 1 { + return nil, fmt.Errorf("doc: page %d is not a page number", page) + } + bin, err := extern.Require(extern.PDFToCairo) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, extractTimeout) + defer cancel() + + // The flags are exactly [ExtractRules]'s, for the reasons it gives: "-" keeps + // the SVG in memory rather than writing a derived file beside the immutable + // blob store, and -f/-l bound the range to one page. + // #nosec G204 -- see ProbeInfo: bin comes from extern's own tool table, path + // is a blob-store path derived from a validated SHA-256 digest, and page is + // an int. + cmd := exec.CommandContext(ctx, bin, "-svg", + "-f", strconv.Itoa(page), "-l", strconv.Itoa(page), path, "-") + out := &limitedBuffer{limit: maxRuleSVGBytes} + var errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = out, &errOut + if err := cmd.Run(); err != nil { + if errors.Is(err, errOutputTooLarge) { + return nil, fmt.Errorf("%w (limit %d bytes)", errOutputTooLarge, maxRuleSVGBytes) + } + return nil, fmt.Errorf("doc: pdftocairo failed on page %d: %w: %s", + page, err, redact(strings.TrimSpace(errOut.String()), path)) + } + + ink, err := parseInk(out.buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("doc: reading pdftocairo output for %s page %d: %w", + redact(path, path), page, err) + } + return ink, nil +} + +// PageFigures finds one page's pictures and renders each of them. +// +// The page's text is taken as a parameter rather than extracted again, for the +// reason [PageTables] gives: the text guard needs it and [ExtractRuns] has +// already paid for it. A nil page is an error here rather than a skipped guard, +// because without the page box there is no way to tell a background wash from a +// picture and the answer would be one figure covering the page. +func PageFigures(ctx context.Context, path string, page *PageRuns) ([]Figure, error) { + if page == nil { + return nil, errors.New("doc: PageFigures needs a page to read") + } + ink, err := ExtractInk(ctx, path, page.No) + if err != nil { + return nil, err + } + found := ServedFigures(ink, page) + for i := range found { + if err := renderFigure(ctx, path, &found[i]); err != nil { + return nil, err + } + } + return found, nil +} + +// ServedFigures is what a reader is given: [FindFigures] with the crops that lie +// wholly inside another removed, by [absorbNested]. +// +// THE TWO ARE SEPARATE ON PURPOSE, and putting the absorption inside FindFigures was a +// mistake worth recording. FindFigures answers "what pictures are on this page", which +// is the question every threshold sweep in this package measures, and reshaping its +// answer with a serving decision made those sweeps measure a hybrid: TestGrowSweep's +// label count fell from 229 to 227 and TestAPlateMergeOnSharedLabelsIsRefused's claim +// count from 327 to 319, both of them measurements of a pass this one replaced, moved +// by a rule that has nothing to do with either. Ten pinned counts moved and none of +// them should have. +// +// So the geometry is one question and what is served is another. Only the second moves +// here: the sequential manual's Russian goes 65 pictures to 63, its neutral pages 61 to +// 60, and a conversion of all 34 languages 134 to 128, with the labels carried +// unchanged at 166. +func ServedFigures(ink []Ink, page *PageRuns) []Figure { + return absorbNested(FindFigures(ink, page)) +} + +// FindFigures groups a page's ink into pictures and returns those that pass both +// guards, in reading order. +// +// PURE GEOMETRY, and it may return a crop that lies wholly inside another: the band +// takes in the neighbourhood, so two drawings that reach the same run can produce +// nested crops. [ServedFigures] is the one that removes them, and it is the one a +// conversion calls. +// +// Pure geometry: it spawns nothing and reads no file, so the guards are testable +// without poppler. The returned figures carry no bytes — see [PageFigures] for +// those. +func FindFigures(ink []Ink, page *PageRuns) []Figure { + return findFigures(ink, page, defaultGuards) +} + +// figureGuards are the two guards' thresholds, taken as a value rather than read +// from the constants directly so that a test can sweep them over both whole +// documents. That is how every threshold in this package was set, and it is what +// makes the sensitivity ranges quoted above checkable rather than remembered. +type figureGuards struct { + minWidth, minHeight float64 + minInk int + maxText float64 + // mergeOverlap is how much of the smaller of two candidate boxes may lie + // inside the other before they are read as one picture. See + // [mergeOverlapping] for why it is zero. + mergeOverlap float64 + // The label-growth rule's four numbers, here for the same reason: TestGrowSweep + // moves them over both whole documents. growth of zero turns the pass off, which + // is how the sweep measures what it is worth. + terminator, align, corridor, growth float64 +} + +var defaultGuards = figureGuards{ + minWidth: minFigureWidth, minHeight: minFigureHeight, + minInk: minFigureInk, maxText: maxFigureTextFraction, + mergeOverlap: figureMergeOverlap, + terminator: labelTerminator, align: labelAlign, + corridor: labelCorridor, + // THE CROP NO LONGER GROWS. Zero turns [growToLabels] off, and it is off because + // [Figure.Labels] carries every label as text instead — so a crop that also + // contained them would print each one twice, once in the picture and once beside + // it. The function is kept, and TestGrowSweep still sweeps it over both whole + // documents, because the growth pass is now the measured-and-replaced alternative + // and that record is what stops it being proposed again. See [figureLabels]. + growth: 0, +} + +func findFigures(ink []Ink, page *PageRuns, g figureGuards) []Figure { + if page == nil || page.Width <= 0 || page.Height <= 0 { + return nil + } + drawn := onPageInk(ink, page.Width, page.Height) + if len(drawn) > maxFigureClusterInk { + return nil + } + + var dropped DroppedRuns + text := usableRuns(page.Runs, page.Width, page.Height, &dropped) + + var out []Figure + for _, area := range clusterInk(drawn, g.mergeOverlap) { + if area.Width() < g.minWidth || area.Height() < g.minHeight { + continue + } + count := 0 + for i := range drawn { + if contains(area, drawn[i].Rect) { + count++ + } + } + if count < g.minInk { + continue + } + area = trimToPicture(area, text) + if area.Width() < g.minWidth || area.Height() < g.minHeight { + continue + } + fraction := textFraction(area, text) + if fraction > g.maxText { + continue + } + // The band comes last, after both guards have judged the drawing, for the + // reason growth used to: a diagram's own labels are text, so a box that took + // them in would be legitimately over [maxFigureTextFraction] — page 521's lidar + // diagram reaches 0.162 with its eleven labels — and judging the drawing plus + // its labels would reject the very pictures the labels complete. + marks := terminatorMarks(drawn, g) + out = append(out, Figure{ + Page: page.No, Index: len(out), + Rect: labelBand(area, text, marks, g), + InkRect: area, + Ink: count, TextFraction: fraction, + }) + f := &out[len(out)-1] + f.Labels, f.LabelBoxes = figureLabels(area, text, marks, g) + } + return out +} + +// absorbNested drops a figure whose crop lies wholly inside another's, and gives its +// labels to the one that swallowed it. +// +// A band takes in the neighbourhood, so two drawings that reach the same run can end +// up with one crop inside the other: page 529's figures 4 and 7 both claim the numbered +// step under them, and 7's band is 4's band with the top cut off. Serving both shows a +// reader the picture and then its own lower half, which is the defect +// TestNoFigureOverlapsAnotherOnEitherManual calls "a scrap of that drawing served as a +// picture of its own". Measured over both documents: 6 nested crops on the sequential +// manual, 0 on the columns one. +// +// # This is not the plate merge, and the difference is that nothing new is made +// +// The merge TestAPlateMergeOnSharedLabelsIsRefused refuses UNIONS two want boxes, so it +// invents a rectangle larger than either and the transitive closure of that reaches +// 0.629 of a page. This invents nothing. The surviving crop is one of the crops that +// already existed, unchanged to the unit; a chain of containments collapses onto the +// outermost, which is the only one that was going to be drawn anyway. There is no +// cascade to bound because there is no growth. +// +// The labels move rather than being dropped, or a callout block would be left with no +// figure carrying its text — the invariant TestOptingOutIsTodaysConversionExactly holds +// — and they are re-sorted into the survivor's own order, down then across, so the +// stored index still reads as the page reads. +// +// What it does not preserve is the absorbed figure's Ink count, and the survivor's is +// left alone: Ink is the shape guard's evidence about the drawing it judged, and adding +// a neighbour's shapes to it would make a passed guard unattributable. The crop is +// wider than the picture its counts describe, which is true of every band. +func absorbNested(figs []Figure) []Figure { + if len(figs) < 2 { + return figs + } + gone := make([]bool, len(figs)) + area := func(i int) float64 { return figs[i].Rect.Width() * figs[i].Rect.Height() } + for again := true; again; { + again = false + for i := range figs { + if gone[i] { + continue + } + for j := range figs { + // Strictly larger, or the same size and earlier: identical crops contain + // each other, so index order is what stops both being absorbed. + if gone[j] || j == i || !contains(figs[j].Rect, figs[i].Rect) || + (area(j) < area(i) || (area(j) == area(i) && j > i)) { + continue + } + figs[j].Labels = append(figs[j].Labels, figs[i].Labels...) + figs[j].LabelBoxes = append(figs[j].LabelBoxes, figs[i].LabelBoxes...) + gone[i], again = true, true + break + } + } + } + + kept := make([]Figure, 0, len(figs)) + for i := range figs { + if gone[i] { + continue + } + f := figs[i] + f.Index = len(kept) + sortLabels(&f) + kept = append(kept, f) + } + return kept +} + +// sortLabels puts a figure's labels back into the page's own order — down, then across +// — after an absorbed figure's have been appended. Two runs cannot share a top-left +// corner, so the two keys are a total order and a re-conversion writes the same rows. +func sortLabels(f *Figure) { + order := make([]int, len(f.Labels)) + for i := range order { + order[i] = i + } + sort.SliceStable(order, func(a, b int) bool { + x, y := f.LabelBoxes[order[a]], f.LabelBoxes[order[b]] + if x.Y0 != y.Y0 { + return x.Y0 < y.Y0 + } + return x.X0 < y.X0 + }) + labels := make([]string, len(f.Labels)) + boxes := make([]CellRect, len(f.LabelBoxes)) + for i, o := range order { + labels[i], boxes[i] = f.Labels[o], f.LabelBoxes[o] + } + f.Labels, f.LabelBoxes = labels, boxes +} + +// onPageInk drops the shapes that cannot be part of a picture: those outside the +// page box, and those as wide as the page. +// +// Both are structural rather than fitted, and both were found by reading a wrong +// answer. Cairo's compositing machinery paints rects larger than the page — on +// page 57 of the columns manual, 1286x1225 on an 892x850 page, starting at +// (-196.4,-187.1) — and a shape covering the page touches everything on it, so +// unfiltered every page clusters into one figure. See [washFraction] for the +// second. +func onPageInk(ink []Ink, width, height float64) []Ink { + out := make([]Ink, 0, len(ink)) + for i := range ink { + r := ink[i].Rect + switch { + case r.Width() <= 0 || r.Height() <= 0: + case r.X0 < -1 || r.Y0 < -1 || r.X1 > width+1 || r.Y1 > height+1: + case r.Width() >= washFraction*width: + default: + out = append(out, ink[i]) + } + } + return out +} + +// clusterInk groups shapes that overlap or touch into candidate pictures, and +// returns each group's bounding box in reading order. +// +// Touching rather than within a gap, deliberately. A tolerance was measured and +// costs more than it buys: at 2 units page 42 of the columns manual returns 3 +// figures where 0 returns the same 3, and at that page's other extreme a gap +// large enough to join a drawing to its own caption also joins two drawings 27 +// units apart. What holds a real picture together is that its strokes meet, and +// they do. +func clusterInk(ink []Ink, overlap float64) []CellRect { + n := len(ink) + parent := make([]int, n) + for i := range parent { + parent[i] = i + } + // Iterative rather than recursive, halving the path as it goes: a page of + // 83,014 shapes can chain deeply enough for a recursive find to be a real + // stack. + find := func(i int) int { + for parent[i] != i { + parent[i] = parent[parent[i]] + i = parent[i] + } + return i + } + + // Sweeping down the page rather than comparing every pair: a page of 83,014 + // shapes is 3.4 billion pairs, and the quadratic version of this took minutes + // on page 42 of the columns manual. Sorted by top edge, a shape can only touch + // one already seen whose bottom edge has not yet passed it, so the active set + // stays small. + order := make([]int, n) + for i := range order { + order[i] = i + } + sort.Slice(order, func(a, b int) bool { + return ink[order[a]].Rect.Y0 < ink[order[b]].Rect.Y0 + }) + active := make([]int, 0, 64) + for _, i := range order { + r := ink[i].Rect + // Compacted in place: everything whose bottom edge is above this shape's + // top can never touch anything later, so it leaves the active set. + live := 0 + for _, j := range active { + s := ink[j].Rect + if s.Y1 < r.Y0 { + continue + } + active[live] = j + live++ + if s.X0 <= r.X1 && r.X0 <= s.X1 { + if a, b := find(i), find(j); a != b { + parent[a] = b + } + } + } + active = append(active[:live], i) + } + + boxes := make(map[int]CellRect, n) + for i := range ink { + root := find(i) + r := ink[i].Rect + if cur, ok := boxes[root]; ok { + boxes[root] = CellRect{ + math.Min(cur.X0, r.X0), math.Min(cur.Y0, r.Y0), + math.Max(cur.X1, r.X1), math.Max(cur.Y1, r.Y1), + } + continue + } + boxes[root] = r + } + + out := make([]CellRect, 0, len(boxes)) + for _, r := range boxes { + out = append(out, r) + } + // Down then across, which is the reading order [DetectColumns] establishes for + // text and the order a figure has to take its place in. + byReadingOrder := func(a, b CellRect) bool { + if a.Y0 != b.Y0 { + return a.Y0 < b.Y0 + } + return a.X0 < b.X0 + } + // Sorted BEFORE the merge as well as after, and that is correctness rather than + // tidiness: the groups come out of a map, so their order is random. + // + // At the shipped threshold of zero the order cannot change the answer, and the + // reason is worth stating because it is what makes that value safe: merging only + // ever grows a box, so it can never destroy an intersection, and the result is + // the connected components of "these two boxes intersect" however they are + // visited. Above zero that stops holding — a merged box is wider, so the smaller + // box's SHARE of it falls, and a merge can put a pair below the threshold that + // was above it. Measured: with the boxes left in map order, TestMergeThresholdSweep + // returned 195, 197 and 196 figures at 0.01, 0.05 and 0.1 in one pass and 194 to + // 200 in the next, in the same run. These are the pictures a reader is served out + // of a content-addressed store, so the sweep has to be reproducible too. + // + // Reading order is used because it is already the order this function ends in; + // nothing depends on which order it is, only that it is always the same one. + sort.Slice(out, func(i, j int) bool { return byReadingOrder(out[i], out[j]) }) + out = mergeOverlapping(out, overlap) + sort.Slice(out, func(i, j int) bool { return byReadingOrder(out[i], out[j]) }) + return out +} + +// mergeOverlapping joins candidate boxes that overlap into one, until none of +// them does. +// +// This is [clusterInk]'s own rule applied to its own output, and the second pass +// is needed because the first cannot see it. A group's box is the union of its +// shapes and is far larger than any of them, so two groups can share most of a +// rectangle while no shape of one touches a shape of the other — which is exactly +// how the fault the user reported arises. Page 524 of the sequential manual draws +// a hand holding a pin over the robot's underside; the hand's strokes reach none +// of the robot's, so it clusters alone, and 90.8% of its box lies inside the +// robot's. It was served as a separate picture: a duplicate scrap of the drawing +// above it. +// +// # Why any overlap at all, and not a fraction of one +// +// A threshold is the obvious thing to want, and the measurement says there is +// nothing for it to separate. Over both whole documents the parallel-columns +// manual has NO overlapping pair of candidates at all — every change here is the +// sequential manual's — and that document has 53, whose overlap as a fraction of +// the smaller box runs 1.00, 0.96, 0.91, 0.91, 0.88 … 0.20, 0.19, 0.14, 0.11, +// 0.10, 0.01 with no gap anywhere. Each was rendered as a crop of the two boxes' +// union and looked at. Every one of the 53 is a single printed drawing that +// clustered in pieces: at 0.91 the hand above, at 0.57 the base station of page +// 522 split at its own waist, at 0.39 the station of page 5 and the wall socket +// it is being plugged into, at 0.01 the water tank of page 522 and the magnified +// detail circle its leader lines run to. Not one is two drawings that merely sit +// close together, so no threshold in the range has a case to decide and every +// value from 0 to 0.01 gives the same answer as containment plus 46 more merges +// that a reader wants. +// +// Two drawings printed close together do exist on these pages and are not +// affected, because their boxes do not overlap: the two mop pads of page 522 are +// 46 units apart, and the two halves of page 524's top illustration 23. That is +// the same fact [clusterInk] records about a gap tolerance, from the other side — +// what does NOT hold a picture together is proximity. +// +// So the threshold is kept as a parameter and set to zero: any positive overlap +// merges. It is a parameter because that sweep is the evidence, and +// TestMergeThresholdSweep re-runs it. +// +// Repeated to a fixpoint, because a merged box is larger and can reach a third +// group that neither part reached. That cannot run away: every round but the last +// removes at least one box, so it terminates. +// +// The result depends on the order the boxes are considered in — absorbing B into A +// can make a box that reaches C where absorbing C into B first need not reach A — +// so the order is fixed by the caller and this pass preserves it. See [clusterInk]. +func mergeOverlapping(boxes []CellRect, overlap float64) []CellRect { + gone := make([]bool, len(boxes)) + for { + merged := false + for i := range boxes { + if gone[i] { + continue + } + for j := i + 1; j < len(boxes); j++ { + if gone[j] || boxOverlap(boxes[i], boxes[j]) <= overlap { + continue + } + boxes[i] = CellRect{ + math.Min(boxes[i].X0, boxes[j].X0), math.Min(boxes[i].Y0, boxes[j].Y0), + math.Max(boxes[i].X1, boxes[j].X1), math.Max(boxes[i].Y1, boxes[j].Y1), + } + gone[j] = true + merged = true + } + } + if !merged { + break + } + } + // Compacted in place, keeping the order the boxes arrived in. The order is what + // makes the answer reproducible; see the note in [clusterInk]. + out := boxes[:0] + for i := range boxes { + if !gone[i] { + out = append(out, boxes[i]) + } + } + return out +} + +// boxOverlap is how much of the smaller box lies inside the larger, 1 when one +// contains the other. +// +// Measured per axis and multiplied, rather than as an area ratio, for the reason +// [verify.overlapFraction] gives: a candidate can be degenerate. A single hairline +// clusters alone and its box has zero height, so an area ratio divides by zero; +// per axis the question becomes containment on that axis, which is the same +// question asked of a shape with no thickness. +func boxOverlap(a, b CellRect) float64 { + return axisOverlap(a.X0, a.X1, b.X0, b.X1) * axisOverlap(a.Y0, a.Y1, b.Y0, b.Y1) +} + +// axisOverlap is the share of the shorter of two intervals that lies inside the +// other. +func axisOverlap(a0, a1, b0, b1 float64) float64 { + // A zero-length interval has no share to take, so the question becomes whether + // its one point lies inside the other — the same reading [verify.overlap1D] + // gives a shape with no thickness. Asked before the width test below, because + // an interval of zero length overlaps nothing by measure. + if a1 <= a0 { + return inside(a0, b0, b1) + } + if b1 <= b0 { + return inside(b0, a0, a1) + } + in := math.Min(a1, b1) - math.Max(a0, b0) + if in <= 0 { + // Touching exactly is not overlapping. The shape-level pass has already + // joined everything whose boxes meet, so a second pass that merged on + // contact would only undo its own answer. + return 0 + } + return in / math.Min(a1-a0, b1-b0) +} + +// inside reports 1 when a point lies within an interval and 0 when it does not. +func inside(p, lo, hi float64) float64 { + if p >= lo && p <= hi { + return 1 + } + return 0 +} + +// contains reports whether inner sits wholly inside outer. +func contains(outer, inner CellRect) bool { + const slack = 0.01 // arithmetic slack; both boxes came from the same maxima + return inner.X0 >= outer.X0-slack && inner.X1 <= outer.X1+slack && + inner.Y0 >= outer.Y0-slack && inner.Y1 <= outer.Y1+slack +} + +// textFraction is how much of a rectangle's area the text inside it covers. +// +// A run is charged by how much of it overlaps, not by whether its centre is +// inside, and that is not a refinement — the centre rule was written first and it +// is wrong by a factor of ten on a real page. [countCellText] can use the centre +// because a cell is at least as wide as the text set in it; a figure is not. Page +// 529 of the sequential manual prints a small diagram 91 units wide under a +// caption line 400 units wide, and the caption's midpoint lands inside the +// diagram's box: charged whole, one line of text covered 95.3% of a picture and +// the text guard rejected it, while the very same diagram on page 550 — the same +// content in another language, with the caption a few units higher — was accepted. +// A guard that decides opposite ways about the same picture is measuring the wrong +// thing. +// +// Run rectangles are still summed without subtracting where two overlap, which can +// only overstate the fraction. That direction is deliberate: it errs towards +// calling a picture a table, and rendering a page of prose as an illustration is +// the worse of the two failures. +func textFraction(area CellRect, text []TextRun) float64 { + size := area.Width() * area.Height() + if size <= 0 { + return 0 + } + var covered float64 + for i := range text { + r := &text[i] + w := math.Min(area.X1, r.X+r.Width) - math.Max(area.X0, r.X) + h := math.Min(area.Y1, r.Y+r.Height) - math.Max(area.Y0, r.Y) + if w > 0 && h > 0 { + covered += w * h + } + } + return covered / size +} + +// trimToPicture pulls a candidate's edges in off a line of text the box has +// REACHED OVER — a line that starts or ends outside the box — and leaves a line +// printed wholly within the artwork alone. +// +// That distinction is the whole function, and it is what was missing. Trimming was +// written as the remedy for the clip this code could not read: an ink box was a +// path's unclipped extent, so a drawing whose artwork ran past its frame reached +// into the text column beside it, and the crop of page 18's first figure contained +// a slab of German prose and the printed D badge. clip.go removed that cause, and +// what was left was a rule that could not tell a picture's own callout from the +// prose next to it. It cut into 13 of the columns manual's 59 drawings to exclude +// 6 lines of prose — page 16 figure 2 lost its right third to the label »click«, +// printed inside the illustration with artwork around it. +// +// # What separates a callout from prose, measured +// +// The signal tried first was ink: a label inside a drawing should have drawn shapes +// on more than one side of it. It does not separate these documents. The »click« of +// page 16 has ink on all four sides, but the same label on pages 24, 26 and 36 sits +// at the drawing's right edge and has ink only to its left and below — while page +// 1's "GEBRAUCHSANLEITUNG", which is prose the box reached over, also has ink on two +// sides. Those four readings are the cases in TestTrimOnlyPullsOffALineItReachedOver. +// +// What does separate them is containment, and it follows from where a candidate's +// box comes from: the box IS the bounding box of the drawn ink. So a line the box +// merely reached over sticks out of it — the edge that touches the line was set by +// a stroke, not by the line — while a label set inside the artwork has ink beyond +// it on the side that fixes that edge, and is therefore wholly inside. What the +// test drops, measured over both whole documents, is every trim that cut a printed +// callout and no trim that excluded a line of prose: +// +// old rule reaching lines only +// columns manual, trims made 13 6 +// ...of which cut a printed callout 7 0 +// lines of prose excluded 6 6 +// sequential manual, trims made 12 11 +// +// The six prose lines are page 1's cover title block and the one line of body text +// above the process diagram on each of pages 52-56, and they are excluded either +// way. The seven that stop being cut are »click« on pages 16, 24 (twice), 26 and 36 +// (twice) and "1,8 l"/"max. 30° C" on page 28. The sequential manual loses one trim +// of its twelve, on page 53, and one other stops short of a label: page 545's box +// held its right edge at x=245, through the leader line running out to "QR コード", +// and now stops at 285 where the Wi-Fi caption genuinely reaches in. +// +// End to end with `manualbox verify`, figures cut off by their crop fall from 15 to +// 3 on the columns manual and from 71 to 70 on the sequential one, with the figure +// count, the pages carrying figures and the blank-band count unmoved on both. +// +// The figure-overlaps-text count that used to be quoted here cannot show this and +// is not quoted any more: it counts any run of five runes or more, so a picture +// keeping its own »click« reads to it exactly like a picture swallowing a +// paragraph. On the columns manual it moves 9 -> 14 while the prose excluded stays +// at 6. TestGuardSweep still prints it, as a bound rather than as a verdict. +// +// Two guards are kept underneath. Only a run of [minTrimRunes] or more is trimmed +// for, because a callout number is one or two characters and page 11's parts diagram +// carries 73 of them; containment already protects those, and the floor is the +// second lock on a diagram whose numbering runs to the frame. And no edge moves by +// more than [maxTrimFraction] of the side it is on, so a candidate that is genuinely +// half prose — page 34's over-merged cluster — is not whittled into a plausible +// picture but left for the text guard to reject. +// +// The edge that costs the least area is chosen each round, among only the edges the +// run actually reaches past, because a line at a corner can be excluded two ways and +// the cheaper one keeps more of the drawing. +func trimToPicture(area CellRect, text []TextRun) CellRect { + const ( + // minTrimRunes is the shortest run worth trimming for. Four rather than one + // because the labels printed inside these documents' diagrams are short — + // "OFF" on page 34 of the columns manual, the digits 1 to 39 on page 11 — + // and a figure is not improved by being cut away from its own labels. + minTrimRunes = 4 + // maxTrimFraction is how much of a side may be trimmed away in total. A + // third: past that the candidate was not a picture with text at its edge, it + // was a region containing both, and shrinking it to fit is fabricating a + // figure boundary the page does not draw. + maxTrimFraction = 1.0 / 3 + ) + minW := area.Width() * (1 - maxTrimFraction) + minH := area.Height() * (1 - maxTrimFraction) + + // Bounded rather than "until clean": each round removes at least one run, and a + // page of these documents holds a few hundred. + for range 32 { + var worst *TextRun + var bestCost float64 + var bestEdge int + for i := range text { + r := &text[i] + if len([]rune(strings.TrimSpace(r.Text))) < minTrimRunes { + continue + } + x0, y0 := math.Max(area.X0, r.X), math.Max(area.Y0, r.Y) + x1, y1 := math.Min(area.X1, r.X+r.Width), math.Min(area.Y1, r.Y+r.Height) + if x1 <= x0 || y1 <= y0 { + continue + } + // Which edges the line reaches past. A line wholly inside reaches past + // none of them and is the picture's own label, so it is left alone; this + // is the test the whole function turns on. Compared exactly rather than + // with a tolerance — see [trimReachSlack] for what a tolerance costs. + reaches := [4]bool{ + r.X < area.X0-trimReachSlack, + r.X+r.Width > area.X1+trimReachSlack, + r.Y < area.Y0-trimReachSlack, + r.Y+r.Height > area.Y1+trimReachSlack, + } + // Four ways to put the run outside: pull in the left, right, top or + // bottom edge to the far side of it. Cost is the area given up. + costs := [4]float64{ + (x1 - area.X0) * area.Height(), // left edge moves right to x1 + (area.X1 - x0) * area.Height(), // right edge moves left to x0 + (y1 - area.Y0) * area.Width(), // top edge moves down to y1 + (area.Y1 - y0) * area.Width(), // bottom edge moves up to y0 + } + for edge, cost := range costs { + if !reaches[edge] { + continue + } + if worst == nil || cost < bestCost { + // Only consider an edge that leaves the figure big enough. + next := area + switch edge { + case 0: + next.X0 = x1 + case 1: + next.X1 = x0 + case 2: + next.Y0 = y1 + case 3: + next.Y1 = y0 + } + if next.Width() < minW || next.Height() < minH { + continue + } + worst, bestCost, bestEdge = r, cost, edge + } + } + } + if worst == nil { + return area + } + x0, y0 := math.Max(area.X0, worst.X), math.Max(area.Y0, worst.Y) + x1, y1 := math.Min(area.X1, worst.X+worst.Width), math.Min(area.Y1, worst.Y+worst.Height) + switch bestEdge { + case 0: + area.X0 = x1 + case 1: + area.X1 = x0 + case 2: + area.Y0 = y1 + case 3: + area.Y1 = y0 + } + } + return area +} + +// growToLabels grows a figure's box to take in the labels its leader lines point +// at, and never takes in a line of prose. +// +// This is [trimToPicture]'s opposite and it exists because the trim was only ever +// half the problem. A figure's box is the bounding box of the drawn ink, and a +// callout number is not ink: it is a text run printed just outside the drawing, at +// the end of a leader line. So the box covers every leader and excludes every label +// they point at, and the user's report is exactly that — "the crop keeps the lines +// and loses every number, so the leaders end in nothing and the diagram cannot be +// read against its parts list." +// +// Measured on page 521 of the sequential manual, the RU product overview, whose +// three drawings carry 31 labels between them: the box's right edge is at 263.0, +// the leader terminators are at 259.6-263.0, and all eleven of that drawing's +// labels start at **266.0**. Three units, every one of them. The box does not need +// to reach the leader's end — it is already there. It needs to cross the gap. +// +// # What says a run is this figure's label, and what says it is prose +// +// The terminator: a small mark, [labelTerminator] units at most, sitting in the +// corridor between the figure's edge and the run, on that run's own midline. Both +// documents draw one at the end of every leader. +// +// It has to be that rather than the gap, and the case that settles it is a document +// rather than an argument. Page 11 of the columns manual prints its parts list — 39 +// numbers and 39 German names, "1 Gehäusedeckel", "2 Tragegriff" — in a column +// 22.3 units to the right of the exploded view. That is INSIDE the range page 521's +// underside diagram holds its own labels at, 20.3 to 35.3 units out: it is not that +// the legend sits further away, it is that no distance separates the two. So any rule +// that grows onto text within some distance swallows the whole parts list, while the +// terminator test refuses all 78 of its runs, because a legend is not pointed at. +// +// The second half of the signal is that **a label wraps**. Its second and third +// lines carry no terminator of their own, and left unclaimed they are obstacles to +// the label they belong to: page 521's lidar drawing claims nine of its eleven +// labels by terminator, and the two continuation lines block the edge from moving at +// all. A run flush with a claimed label, on the next line, alone on its baseline, is +// part of it. Alone matters: "Кнопка сброса" is a label and the five lines under it +// are its description, and what separates them is that a bullet has its text beside +// it while a label's continuation does not. +// +// # What it will not do, which is the conservative half +// +// An edge moves only if everything the growth region touches is a claimed label. +// One line of prose in the way and the edge stays where it is. That is why page +// 521's lid-open drawing keeps its three right-hand labels cropped: the corridor +// holds "Кнопка сброса" and then the five bullet lines that explain it, so growing +// right would drag a paragraph into a picture. Its left edge grows and its right +// does not. +// +// A claimed label may still be cut, and prose may not. The edge goes as far as the +// farthest label it can reach cleanly, which on a page whose two label columns +// interleave in x is not far enough for the longest of them: page 521's lidar +// drawing reaches 397, where its own longest label ends at 469, because the +// neighbouring drawing's labels start at 400. Refusing to cut a label at all was +// measured and costs the whole page — with it, that drawing does not grow, and +// neither does its neighbour's left edge. A leader ending in a word cut short is a +// large improvement on a leader ending in nothing; a picture with a paragraph in it +// is not. +// +// # What it is worth, over both whole documents +// +// columns manual sequential manual +// figures 59 195 +// figures with a label outside them 2 79 +// figures grown 0 55 +// labels taken in 0 233 +// +// Those 229 are labels the crop REACHES, and the difference between reaching and +// containing is the clipping this rule accepts: on page 521 the three drawings reach +// 9, 11 and 14 labels and hold 3, 8 and 12 of them whole, so 23 of the 34 arrive +// uncut. That is the number to watch if this is ever replaced by carrying a label as +// text, which would make it 34. +// +// The columns manual does not move at any setting, which is the same shape of +// evidence [mergeOverlapping] rests on, so this change is the other document's +// entirely. Both of its claims are FALSE and both are blocked, which is worth +// stating plainly because it is what the conservative rule is for: page 1's cover +// figure claims the title `РУКОВОДСТВО ПО ЭКСПЛУАТАЦИИ`, and page 22's claims eight +// lines of German prose about emptying the DryBOX. The terminator signal is not +// precise on its own — a small shape near a line of text will do — and what makes +// it safe is that an edge does not move unless the region it would add holds +// nothing but claims. +// +// Of the sequential manual's 55, **22 are on pages 5 and 6** — the front-matter +// diagram plates, which fall outside every language region and are never converted — +// leaving 33 on pages a reader is served. +// +// The cost is overlapping crops, and it is confined: 11 pairs of grown boxes +// overlap — 9 on page 5 and 2 on page 6 — and none on any page a conversion serves. +// Page 5 is 31 figures on one sheet with labels between them, and one of its nine +// pairs is a crop now wholly inside another crop. The drawn boxes are untouched by +// all of this — measured over both documents, every page, they still overlap in 0 +// pairs and nest in 0, so [mergeOverlapping]'s property holds of the rect it is +// about. Recorded rather than fixed, because no page a +// reader sees is affected and the alternative — arbitrating which of two drawings a +// shared corridor belongs to — would be a rule invented for one plate. +// +// Edges are taken in a fixed order and each one's region is judged against the box +// as already grown, which is what keeps two independently clean edges from admitting +// a run diagonally outside both. +func growToLabels(area CellRect, text []TextRun, drawn []Ink, g figureGuards) CellRect { + if g.growth <= 0 { + return area + } + marks := terminatorMarks(drawn, g) + + out := area + for side := range 4 { + // Claims come from the box the guards judged, so which runs are this + // figure's labels does not depend on the order the edges are taken in. + claimed := claimLabels(area, text, marks, side, g) + if len(claimed) == 0 { + continue + } + // The region and the reach are judged against the box as already grown, + // which is what keeps two independently clean edges from admitting a run + // diagonally outside both. The cap is against the drawing, so an edge's + // allowance does not grow because another edge moved first. + want, ok := labelExtent(out, text, claimed, side) + if !ok { + continue + } + if edgeMove(area, side, want) > g.growth*edgeSpan(area, side) { + continue + } + out = moveEdge(out, side, want) + } + return out +} + +// terminatorMarks picks the leader end-mark candidates out of a page's ink. +// +// Computed once for the page rather than once per run: a leader's mark is small, and +// page 42 of the columns manual draws 82,626 shapes that a per-run scan would walk +// for every label on every figure. +func terminatorMarks(drawn []Ink, g figureGuards) []CellRect { + marks := make([]CellRect, 0, 64) + for i := range drawn { + if r := drawn[i].Rect; r.Width() <= g.terminator && r.Height() <= g.terminator { + marks = append(marks, r) + } + } + return marks +} + +// labelBand is the rectangle a figure is cropped to: the drawing, together with +// every run the claim rule reaches for it, taken whole. +// +// # The paper had already solved this, and re-solving it is what kept going wrong +// +// Three arrangements have been tried for a diagram and its callouts, and the first +// two failed the same way. [growToLabels] widened the crop until it would have +// swallowed a neighbour's text and then stopped, so page 521's lidar drawing reached +// x=397 where its own longest label ends at 469 — 23 of that page's 34 labels held +// whole and the rest cut mid-word. Carrying each label as text with a position and +// letting the reader place it removed the cutting and introduced four defects of its +// own: labels colliding (page 522 overlaps in two places), a wrapped label's tail +// stranded, claims the gate refused left floating mid-page, and every figure placed +// with no knowledge of its neighbour. All four existed only because the reader was +// rebuilding an arrangement the printed page gets right. +// +// So the crop takes the arrangement instead of reconstructing it, and there is +// nothing left to place, to cut, or to collide. +// +// # Every claim, not the gated ones, and the difference is the point +// +// [figureLabels] keeps its gate and this does not have one. A claim [labelExtent] +// refuses is a claim of UNKNOWN kind — page 521 figure 0's right side is three real +// labels plus three lines of the bullet prose under one of them — and the two +// questions have different right answers: +// +// what becomes ALT TEXT only what the gate believes is a label +// how wide the PICTURE is everything the page laid out beside the drawing +// +// Being wrong about the second costs a paragraph printed inside a picture that +// already printed it. Being wrong about the first would tell a screen reader that a +// paragraph of German body prose is a diagram's callout. That asymmetry is why the +// gate stays exactly where it is and this pass has none. +// +// It is what fixes `Кнопка сброса`, `Индикатор Wi-Fi` and `Датчик края`, which +// docs/design/conversion.md recorded as measured-and-unfixable: both ways of DRAWING +// them were refused, and neither refusal applies to cropping them. +// +// # What a band costs, measured over both whole documents +// +// The cost is prose that the crop prints as pixels and the block flow ALSO emits as +// text, so a few lines arrive twice. Counted in runes over the pages that serve a +// crop, against three alternatives: +// +// crop rule columns manual sequential manual +// the drawing alone (before this) 773 1.0% 51 0.3% +// drawing + the GATED labels 773 1.0% 116 0.7% +// drawing + every claim (THIS) 1,481 1.8% 1,714 9.8% +// the page's full width 67,662 84.0% 11,377 65.4% +// +// The full-width band was the first choice and is refused by that last row. 59,618 of +// its 67,662 runes are a NEIGHBOURING COLUMN's text, and on the columns manual a +// neighbouring column is a different language — the one failure [attribute] says the +// funnel may not have. That document also carries 0 labels, all nine of its claims +// being false, so it would pay the whole cost of a feature it cannot use. +// +// This rule's own cost lands on the columns manual's page 22, where the crop takes in +// the eight lines of German prose that make up its nine false claims, and on the +// sequential manual's page 546, its worst at 30% of the page. Page 521 is 8.6%. +// +// # What is not solved +// +// Two crops may overlap — 35 pairs on the sequential manual, 0 on the columns one — +// so a strip of page can be printed by two pictures, and a neighbour's label can be +// cut at an edge: 48 runs against 16 before. Merging the overlapping pair is the +// obvious answer and is REFUSED on the record, by measurement rather than by +// preference: see TestAPlateMergeOnSharedLabelsIsRefused, where the transitive +// closure of overlapping want boxes takes page 521's three drawings into one crop +// 0.629 of the page and joins page 522's front view to a cutaway 328 units away. +func labelBand(area CellRect, text []TextRun, marks []CellRect, g figureGuards) CellRect { + band := area + for side := range 4 { + for _, r := range claimLabels(area, text, marks, side, g) { + band = unionRect(band, runBox(r)) + } + } + return snapBandToLines(band, text) +} + +// snapBandToLines moves a band's top and bottom out until no line of text is cut +// across it. +// +// A rectangle whose edge falls in the middle of a line of type prints the top half of +// the letters, which reads as damage rather than as a crop. The x edges are left +// alone deliberately: a vertical cut lands between two words far more often than not, +// and widening on x is what would reach into the neighbouring column this whole rule +// exists to stay out of. +// +// To a fixpoint, because taking a line in can put the band's new edge across the next +// one; bounded because a document that keeps chaining is one where the answer is the +// whole page and this should stop rather than get there. +func snapBandToLines(band CellRect, text []TextRun) CellRect { + for again, guard := true, 0; again && guard < maxBandSnaps; guard++ { + again = false + for i := range text { + r := &text[i] + if axisOverlap(r.X, r.right(), band.X0, band.X1) <= 0 { + continue + } + if r.Y < band.Y0 && r.bottom() > band.Y0 { + band.Y0, again = r.Y, true + } + if r.Y < band.Y1 && r.bottom() > band.Y1 { + band.Y1, again = r.bottom(), true + } + } + } + return band +} + +// maxBandSnaps bounds [snapBandToLines]. Measured over both documents the deepest +// chain is 2, so 8 is four times the worst case and is a runaway guard rather than a +// threshold. +const maxBandSnaps = 8 + +// unionRect is the smallest rectangle holding both. +func unionRect(a, b CellRect) CellRect { + return CellRect{math.Min(a.X0, b.X0), math.Min(a.Y0, b.Y0), + math.Max(a.X1, b.X1), math.Max(a.Y1, b.Y1)} +} + +// runBox is a run as a rectangle, so it can be compared with one. +func runBox(r *TextRun) CellRect { + return CellRect{r.X, r.Y, r.right(), r.bottom()} +} + +// figureLabels is the callout labels a figure's leaders point at, as the text a +// caller that cannot read pixels needs. +// +// It is the same claim rule [growToLabels] used — [claimLabels], by terminator plus +// continuation — behind the one gate that makes a claim safe to CALL a label. The +// crop no longer depends on it: [labelBand] takes every claim, gated or not, so what +// this decides is only what is said about the picture, never what is in it. +// +// # The gate is KEPT, and the columns manual is why +// +// [claimLabels] on its own is not safe enough to describe a picture with. Measured +// over both whole documents, the parallel-columns manual has 9 claims and EVERY ONE +// IS FALSE: page 1's cover figure claims the book title `РУКОВОДСТВО ПО ЭКСПЛУАТАЦИИ`, +// and page 22's claims eight lines of German body prose about emptying the DryBOX +// ("Drehen Sie dazu die Entriegelung des Grobschmutzbehälters…"). A terminator is a +// small shape near a line of text on that run's midline, and that signal is simply not +// precise on its own — [growToLabels] says so in as many words. +// +// What refuses all nine is [labelExtent]: there is no distance the edge could move to +// whose growth region holds nothing but claims, because real prose sits in the +// corridor. So that test is what this pass gates on, per side. Drop it and a screen +// reader is told that a paragraph of German instructions is a diagram's callout. +// +// The gate's cost is now paid in words rather than in pixels, which is the change. +// Before, a refused side meant a label a reader never saw; page 521's `Кнопка сброса` +// and `Датчик края` floated mid-page and docs/design/conversion.md recorded them as +// unfixable, both alternatives having been measured and refused. Now the crop prints +// them — see [labelBand] — and a refusal costs only their absence from the alt text. +// +// # The side is the unit of the decision and the label is the unit of the answer +// +// Judge a side once, then keep all of its claims, whole. [growToLabels] could not do +// that: one rectangle had to both contain a label and exclude a neighbour's, so it +// moved an edge as far as it could go cleanly and let the rest be cut — page 521's +// lidar drawing reached x=397 where its own longest label ends at 469. +// +// # What that is worth, over both whole documents +// +// columns manual sequential manual +// claims [claimLabels] finds 9 327 +// claims this pass keeps 0 276 +// page 521 — 37 +// +// The 51 the sequential manual does not keep are the gate doing its job, and page 521 +// figure 0's right side is the case to read: its corridor holds `Кнопка сброса` and +// then the bullet lines that explain it, so that side is refused whole. Those runs +// stay in the block flow, where a paragraph belongs, and the picture prints them. +// +// Of the 276, 109 are on PDF pages 5 and 6 — the front-matter plates, served only +// under the neutral-pages opt-in. +// +// # The order is the page's own +// +// Down, then across, over all four sides together. It used to be side-major, because +// the side was carried and a caller hung the label off that edge; nothing does now, so +// the order that remains is the one a reader would read the labels in — which is what +// alt text is. Two runs cannot share a top-left corner, so the two keys are a total +// order and a re-conversion writes the same rows under the same indices. +// +// The second return is where each label was printed, for [Callouts.Mark]. It is not +// stored: see [Figure.LabelBoxes]. +func figureLabels(area CellRect, text []TextRun, marks []CellRect, g figureGuards) ([]string, []CellRect) { + var kept []*TextRun + for side := range 4 { + claimed := claimLabels(area, text, marks, side, g) + if len(claimed) == 0 { + continue + } + // The gate, and the only thing standing between a diagram's callouts and a + // paragraph of prose described as one. See above. + if _, ok := labelExtent(area, text, claimed, side); !ok { + continue + } + for _, r := range claimed { + // A run can be claimed from two sides — a label at a corner is beyond both + // — and it is one label. + if claims(kept, r) { + continue + } + kept = append(kept, r) + } + } + if len(kept) == 0 { + return nil, nil + } + sort.SliceStable(kept, func(i, j int) bool { + if kept[i].Y != kept[j].Y { + return kept[i].Y < kept[j].Y + } + return kept[i].X < kept[j].X + }) + + labels := make([]string, 0, len(kept)) + at := make([]CellRect, 0, len(kept)) + for _, r := range kept { + labels = append(labels, strings.TrimSpace(r.Text)) + at = append(at, runBox(r)) + } + return labels, at +} + +// claimLabels collects the runs beyond one edge that belong to the figure: those a +// leader points at, and the continuation lines of those. +// +// # A LEADER IS JUDGED IN THE FIGURE'S BAND AND A CONTINUATION IS NOT +// +// The first pass asks [runBeyond], which requires the run to sit within the band the +// drawing occupies on the other axis: a leader points out of the drawing, so its +// label is level with some part of it, and that band is what keeps a terminator +// coincidence somewhere else on the page from claiming a run. +// +// The continuation pass asks [runInCorridor], which drops the band. A label wraps +// DOWNWARD, so a label set level with the drawing's last few units runs past its +// bottom edge on its second line — page 521's `Монтажные отверстия для держателя +// насадки для швабры` is level with the underside drawing's foot, and lines 3, 4 and 5 +// of it sit 8 to 47 units below the box. With the band in force they were invisible +// to this loop, and its first two lines were carried alone: the user saw +// `Монтажные отверстия для` drawn on the picture and `держателя`, `насадки для` and +// `швабры` left behind as three floating paragraphs. +// +// The band is not what makes a continuation safe, and there is nothing to widen: a +// continuation must already be flush with a claimed line to 3 units, adjacent to it +// to 6, and alone on its own baseline — [continuesLabel] — which reaches only along +// a chain that starts at a leader. +// +// Measured over both whole documents: the columns manual carries 0 before and 0 after, +// its 9 false claims still refused; the sequential manual goes from 268 to 276, and +// every one of the 8 is a later line of a label whose earlier lines were ALREADY +// carried, so nothing new is claimed and four partial labels become whole: +// +// page 521 Монтажные отверстия для | держателя насадки для швабры +// page 522 Бак для | чистой воды +// page 522 Вентиляционное | отверстие системы автоопорожнения +// page 542 モップパッドホル | ダー取り付け穴 +// +// The eighth is page 546's second line of a bullet caution the pass should not have +// claimed at all. That is the known "7 claims are prose" defect, and this makes the +// claim whole rather than half — which is the point: a partial claim is worse than +// either answer, whichever answer is right. +// +// # What is NOT truncated, which is what makes this the whole of the fix +// +// A chain is grown to a fixpoint over exactly this predicate, so a claimed label +// cannot have an unclaimed later line UNLESS one of two bounds cuts it, and both are +// measured at zero or accounted for: 0 continuations anywhere in either document are +// refused by [labelCorridor], and 2 by [minWrapRunes] — the single digits "4" on page +// 5 and "2" on page 6, which are their own numbered callouts and not any label's +// second line. So there is no case left where dropping a partly-carried label would +// be better than carrying it, which is why the fix is to carry the tail rather than to +// discard the head. +func claimLabels(area CellRect, text []TextRun, marks []CellRect, side int, g figureGuards) []*TextRun { + var claimed []*TextRun + for i := range text { + r := &text[i] + gap, outside := runBeyond(area, r, side) + if !outside || gap > g.corridor { + continue + } + if terminatorAt(marks, area, r, side, g) { + claimed = append(claimed, r) + } + } + if len(claimed) == 0 { + return nil + } + // A wrapped label's later lines, to a fixpoint: a third line continues a second + // that was itself only just claimed. + for again := true; again; { + again = false + for i := range text { + r := &text[i] + if !runInCorridor(area, r, side, g.corridor) || claims(claimed, r) { + continue + } + if len([]rune(strings.TrimSpace(r.Text))) < minWrapRunes { + continue + } + if continuesLabel(claimed, r, side, text, area) { + claimed = append(claimed, r) + again = true + } + } + } + return claimed +} + +// labelExtent is how far the edge may move: the farthest claimed label whose +// growth region holds nothing but claimed labels. +func labelExtent(area CellRect, text []TextRun, claimed []*TextRun, side int) (float64, bool) { + extents := make([]float64, 0, len(claimed)) + for _, r := range claimed { + extents = append(extents, farEdge(r, side)) + } + sort.Float64s(extents) + if side == edgeRight || side == edgeBottom { + slices.Reverse(extents) + } + for _, e := range extents { + if !outward(area, side, e) { + continue // already inside the box, from another edge's growth + } + if unclaimedRun(growthRegion(area, side, e), text, claimed) == nil { + return e, true + } + } + return 0, false +} + +// The four edges, in the order [growToLabels] takes them. +const ( + edgeLeft = iota + edgeRight + edgeTop + edgeBottom +) + +// minWrapRunes is the shortest run that may be claimed as a label's next line. +// Three: "колесо" and "щетки" are continuation lines on page 521 and a bullet's +// "•" is not, and the floor is the second lock on that after [continuesLabel]'s +// own test. A claim by terminator has no floor, because a callout number is one +// character. +const minWrapRunes = 3 + +// runBeyond reports whether a run lies wholly beyond one edge — within the band the +// figure occupies on the other axis — and by how far. +func runBeyond(area CellRect, r *TextRun, side int) (gap float64, ok bool) { + switch side { + case edgeLeft: + if r.right() <= area.X0 && r.bottom() > area.Y0 && r.Y < area.Y1 { + return area.X0 - r.right(), true + } + case edgeRight: + if r.X >= area.X1 && r.bottom() > area.Y0 && r.Y < area.Y1 { + return r.X - area.X1, true + } + case edgeTop: + if r.bottom() <= area.Y0 && r.right() > area.X0 && r.X < area.X1 { + return area.Y0 - r.bottom(), true + } + case edgeBottom: + if r.Y >= area.Y1 && r.right() > area.X0 && r.X < area.X1 { + return r.Y - area.Y1, true + } + } + return 0, false +} + +// runInCorridor reports whether a run lies beyond one edge, within the corridor, at +// any height — [runBeyond] without its band. +// +// This is the predicate a wrapped label's later lines are found by, and the one the +// scan for a run sharing a line asks, so both halves of [continuesLabel] see the same +// set. See [claimLabels] for why the band belongs to the leader and not to the wrap. +func runInCorridor(area CellRect, r *TextRun, side int, corridor float64) bool { + var gap float64 + switch side { + case edgeLeft: + if r.right() > area.X0 { + return false + } + gap = area.X0 - r.right() + case edgeRight: + if r.X < area.X1 { + return false + } + gap = r.X - area.X1 + case edgeTop: + if r.bottom() > area.Y0 { + return false + } + gap = area.Y0 - r.bottom() + default: + if r.Y < area.Y1 { + return false + } + gap = r.Y - area.Y1 + } + return gap <= corridor +} + +// terminatorAt reports whether a leader's end mark sits between the figure's edge +// and this run, on the run's own midline. +// +// The mark may be just inside the edge or out in the corridor, and both happen in +// one document: page 521's lidar drawing ends AT its terminators, which is what +// sets its box edge, while its underside drawing's marks are 28 units outside the +// box because the leader lines running to them are perfectly horizontal and a +// horizontal hairline has no height, so [onPageInk] never saw them. See the note +// at the end of this file. +func terminatorAt(marks []CellRect, area CellRect, r *TextRun, side int, g figureGuards) bool { + midX, midY := r.X+r.Width/2, r.Y+r.Height/2 + for _, s := range marks { + cx, cy := (s.X0+s.X1)/2, (s.Y0+s.Y1)/2 + switch side { + case edgeLeft: + if cx <= area.X0+g.terminator && cx >= r.right()-g.terminator && + math.Abs(cy-midY) <= g.align { + return true + } + case edgeRight: + if cx >= area.X1-g.terminator && cx <= r.X+g.terminator && + math.Abs(cy-midY) <= g.align { + return true + } + case edgeTop: + if cy <= area.Y0+g.terminator && cy >= r.bottom()-g.terminator && + math.Abs(cx-midX) <= g.align { + return true + } + case edgeBottom: + if cy >= area.Y1-g.terminator && cy <= r.Y+g.terminator && + math.Abs(cx-midX) <= g.align { + return true + } + } + } + return false +} + +// continuesLabel reports whether r is the next line of a label already claimed: set +// flush with it, on the adjacent line, and alone on its own baseline. +// +// The scan for a run sharing r's line asks [runInCorridor] with no distance bound, +// which is [runBeyond]'s old reach minus its band. It has to be the same predicate the +// candidate itself passed, or a wrapped label's later line would be judged alone +// against a narrower set of neighbours than the one it was found in — the band-free +// set is the LARGER one, so this makes the test stricter and not weaker. +// +// NO CASE IN EITHER DOCUMENT DISTINGUISHES IT, and that is measured rather than assumed: +// putting the band back on this scan alone, with the candidate still found without it, +// changes no count on either manual. It is here for the consistency, not for a gain — a +// run outside the band that shares a later line's baseline is a shape neither fixture +// prints, and the first document that prints one would otherwise get a label with a +// neighbour's words merged into its chain. +func continuesLabel(claimed []*TextRun, r *TextRun, side int, text []TextRun, area CellRect) bool { + const ( + // flush is how far two lines of one label's near edges may differ. Three: + // page 521 sets "Модуль" and "MopExtend" against a right margin two units + // apart, because the glyphs do not end at the same place. + flush = 3.0 + // step is how far apart two lines of one label may sit. Six: a run is taller + // than the pitch it is set at, so consecutive lines of these labels overlap + // rather than leaving a gap, and this bounds the case where they do not. + step = 6.0 + // beside is how near another run must be to count as sharing this line. A + // bullet's text starts 1 unit after it; the next label column on page 521 + // starts 147 units away and must not count, or a three-line label is blocked + // by a run it has nothing to do with. + beside = 12.0 + // sameLine compares BASELINES, not bands. Two consecutive lines of one label + // overlap vertically, so a band test reports a label's own third line as + // something sharing the second's line, which blocked every growth on the page + // this pass was written for. + sameLine = 2.0 + ) + for _, c := range claimed { + if math.Abs(nearEdge(r, side)-nearEdge(c, side)) > flush { + continue + } + var apart float64 + if side == edgeLeft || side == edgeRight { + apart = math.Max(r.Y-c.bottom(), c.Y-r.bottom()) + } else { + apart = math.Max(r.X-c.right(), c.X-r.right()) + } + if apart > step { + continue + } + alone := true + for i := range text { + o := &text[i] + if o == r || strings.TrimSpace(o.Text) == "" || claims(claimed, o) { + continue + } + if !runInCorridor(area, o, side, math.Inf(1)) { + continue + } + var shares, near bool + if side == edgeLeft || side == edgeRight { + shares = math.Abs(o.Y-r.Y) <= sameLine + near = o.X < r.right()+beside && r.X < o.right()+beside + } else { + shares = math.Abs(o.X-r.X) <= sameLine + near = o.Y < r.bottom()+beside && r.Y < o.bottom()+beside + } + if shares && near { + alone = false + break + } + } + if alone { + return true + } + } + return false +} + +// claims reports whether a run is already claimed. By identity: two runs of a page +// can hold the same text at the same size, and it is this one that is claimed. +func claims(claimed []*TextRun, r *TextRun) bool { + return slices.Contains(claimed, r) +} + +// nearEdge is the run's side facing the figure, farEdge the side away from it. +func nearEdge(r *TextRun, side int) float64 { + switch side { + case edgeLeft: + return r.right() + case edgeRight: + return r.X + case edgeTop: + return r.bottom() + default: + return r.Y + } +} + +func farEdge(r *TextRun, side int) float64 { + switch side { + case edgeLeft: + return r.X + case edgeRight: + return r.right() + case edgeTop: + return r.Y + default: + return r.bottom() + } +} + +// growthRegion is the strip an edge would add by moving out to want. +func growthRegion(area CellRect, side int, want float64) CellRect { + switch side { + case edgeLeft: + return CellRect{want, area.Y0, area.X0, area.Y1} + case edgeRight: + return CellRect{area.X1, area.Y0, want, area.Y1} + case edgeTop: + return CellRect{area.X0, want, area.X1, area.Y0} + default: + return CellRect{area.X0, area.Y1, area.X1, want} + } +} + +// unclaimedRun is the first run inside a region that no label claimed, or nil when +// the region holds nothing else. +func unclaimedRun(region CellRect, text []TextRun, claimed []*TextRun) *TextRun { + for i := range text { + r := &text[i] + if strings.TrimSpace(r.Text) == "" || claims(claimed, r) { + continue + } + if math.Min(region.X1, r.right()) > math.Max(region.X0, r.X) && + math.Min(region.Y1, r.bottom()) > math.Max(region.Y0, r.Y) { + return r + } + } + return nil +} + +// outward reports whether want is further out than the edge already is. +func outward(area CellRect, side int, want float64) bool { + switch side { + case edgeLeft: + return want < area.X0 + case edgeRight: + return want > area.X1 + case edgeTop: + return want < area.Y0 + default: + return want > area.Y1 + } +} + +func edgeMove(area CellRect, side int, want float64) float64 { + switch side { + case edgeLeft: + return area.X0 - want + case edgeRight: + return want - area.X1 + case edgeTop: + return area.Y0 - want + default: + return want - area.Y1 + } +} + +func edgeSpan(area CellRect, side int) float64 { + if side == edgeLeft || side == edgeRight { + return area.Width() + } + return area.Height() +} + +func moveEdge(area CellRect, side int, want float64) CellRect { + switch side { + case edgeLeft: + area.X0 = want + case edgeRight: + area.X1 = want + case edgeTop: + area.Y0 = want + default: + area.Y1 = want + } + return area +} + +// renderFigure renders one figure's rectangle with pdftoppm and fills in its +// bytes, pixel size and digest. +// +// One invocation per figure rather than one render of the page cropped in Go, and +// the cost of that is real: measured on the sequential manual, a crop takes 0.19 s +// and a whole page 0.39 s, so a page carrying 33 figures costs 6 s this way against +// 0.4 s the other, because poppler renders the whole page either way and only the +// output is clipped. It is chosen anyway, for two reasons that outweigh it. The +// bytes are exactly what poppler wrote, so the digest does not depend on Go's PNG +// encoder reproducing poppler's output — and this is a content-addressed store, +// where a re-encoded byte means the same picture stored twice. And the total stays +// the same order as what the stage it belongs to already costs: the columns +// manual's 46 figures take 14 s end to end including the pdftocairo passes, against +// the 8.6 s reading that document's ruled lines takes, and conversion runs only over +// the pages in scope. +func renderFigure(ctx context.Context, path string, fig *Figure) error { + bin, err := extern.Require(extern.PDFToPPM) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(ctx, extractTimeout) + defer cancel() + + // -x -y -W -H are in the rendered image's own pixels, which at figureDPI are + // exactly figureScale times this package's coordinates. Rounded outwards so + // the crop never cuts a stroke the geometry included. + x := int(math.Floor(fig.Rect.X0 * figureScale)) + y := int(math.Floor(fig.Rect.Y0 * figureScale)) + w := int(math.Ceil(fig.Rect.X1*figureScale)) - x + h := int(math.Ceil(fig.Rect.Y1*figureScale)) - y + if x < 0 { + w, x = w+x, 0 + } + if y < 0 { + h, y = h+y, 0 + } + if w <= 0 || h <= 0 { + return fmt.Errorf("doc: figure on page %d has no extent: %v", fig.Page, fig.Rect) + } + + // -png rather than -jpeg: a line drawing is what these are, and JPEG rings + // around a hairline. The output prefix is omitted entirely, which is how + // pdftoppm is told to write the image to stdout — it keeps the bytes out of the + // directory the immutable blob store lives in, and it is not the "-" the other + // poppler tools here take: passing "-" makes pdftoppm write a file called + // "-.png" beside the process's working directory and return nothing, measured. + // #nosec G204 -- see ProbeInfo: bin comes from extern's own tool table, path + // is a blob-store path derived from a validated SHA-256 digest, and every + // other argument is an int. + cmd := exec.CommandContext(ctx, bin, "-png", "-r", strconv.Itoa(figureDPI), + "-f", strconv.Itoa(fig.Page), "-l", strconv.Itoa(fig.Page), + "-x", strconv.Itoa(x), "-y", strconv.Itoa(y), + "-W", strconv.Itoa(w), "-H", strconv.Itoa(h), + "-singlefile", path) + out := &limitedBuffer{limit: maxFigurePNGBytes} + var errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = out, &errOut + if err := cmd.Run(); err != nil { + if errors.Is(err, errOutputTooLarge) { + return fmt.Errorf("%w (limit %d bytes)", errFigureTooLarge, maxFigurePNGBytes) + } + return fmt.Errorf("doc: pdftoppm failed on page %d: %w: %s", + fig.Page, err, redact(strings.TrimSpace(errOut.String()), path)) + } + + png := out.buf.Bytes() + pw, ph, err := pngSize(png) + if err != nil { + return fmt.Errorf("doc: pdftoppm output for page %d: %w", fig.Page, err) + } + sum := sha256.Sum256(png) + fig.PNG, fig.DPI = png, figureDPI + fig.PixelWidth, fig.PixelHeight = pw, ph + fig.Digest = hex.EncodeToString(sum[:]) + return nil +} + +// pngSize reads a PNG's pixel dimensions out of its IHDR chunk. +// +// Eight bytes of a fixed header rather than image/png.DecodeConfig, because +// decoding is the one thing this file must not do: the bytes are stored verbatim +// and the size is the only fact needed about them. It doubles as the check that +// pdftoppm wrote a PNG at all — an empty stdout with a zero exit status would +// otherwise be stored as a figure. +func pngSize(data []byte) (width, height int, err error) { + const sigAndIHDR = 8 + 8 + 8 + if len(data) < sigAndIHDR { + return 0, 0, fmt.Errorf("expected a PNG, got %d bytes", len(data)) + } + if !bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")) { + return 0, 0, errors.New("expected a PNG signature") + } + if !bytes.Equal(data[12:16], []byte("IHDR")) { + return 0, 0, errors.New("expected IHDR as the PNG's first chunk") + } + be := func(b []byte) int { + return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3]) + } + return be(data[16:20]), be(data[20:24]), nil +} + +// parseInk reads every drawn shape's bounding box out of cairo's SVG. +func parseInk(data []byte) ([]Ink, error) { + doc, err := parseSVG(data) + if err != nil { + return nil, err + } + w := &inkWalker{doc: doc, visited: make(map[visitKey]bool)} + // Only the body from the top, entering exclusively through the + // reference that pulls a definition back in — the compositing-group trap + // rules.go's header documents applies here identically, and a figure's box is + // wrong by the filter region's origin without it. + for _, kid := range doc.root.kids { + w.walkBody(kid, identity, clipBox{}, 0) + } + return w.ink, nil +} + +// inkWalker accumulates bounding boxes while walking the tree. The same walk +// [ruleWalker] makes, keeping every shape rather than only the axis-aligned thin +// ones; glyph outlines are already gone, dropped structurally by [parseSVG]. +type inkWalker struct { + doc *svgDoc + ink []Ink + visited map[visitKey]bool +} + +func (w *inkWalker) walkBody(n *svgNode, m matrix, clip clipBox, depth int) { + if n.tag == "defs" { + return + } + w.walk(n, m, clip, depth) +} + +func (w *inkWalker) walk(n *svgNode, m matrix, clip clipBox, depth int) { + if depth > maxSVGDepth || strings.HasPrefix(n.id, "glyph-") { + return + } + key := visitKey{node: n} + for i, v := range m { + key.m[i] = int64(math.Round(v * 1e4)) + } + if w.visited[key] { + return + } + w.visited[key] = true + + m = m.compose(parseTransform(n.transform)) + // The element's clip narrows whatever it inherited, in the user space its own + // transform establishes. A clip admitting nothing means every shape below is + // invisible, so the subtree is abandoned rather than walked and discarded. + if box, ok := w.doc.clipAt(n.clip, m); ok { + clip = clip.intersect(box) + if clip.empty() { + return + } + } + + if id, ok := refID(n.filter); ok { + for _, ref := range w.doc.filterRefs[id] { + if target := w.doc.byID[ref]; target != nil { + w.walk(target, m, clip, depth+1) + } + } + } + if n.tag == "use" && strings.HasPrefix(n.href, "#") { + if target := w.doc.byID[n.href[1:]]; target != nil { + w.walk(target, m, clip, depth+1) + } + } + + painted := func(v string) bool { return v != "" && v != "none" } + switch { + case n.tag == "path" && (painted(n.stroke) || painted(n.fill)): + stroked := painted(n.stroke) + for _, sub := range subpaths(n.d) { + w.add(m, clip, sub, stroked) + } + case n.tag == "rect" && painted(n.fill): + w.add(m, clip, []point{ + {n.x, n.y}, {n.x + n.w, n.y}, {n.x + n.w, n.y + n.h}, {n.x, n.y + n.h}, + }, false) + } + + for _, kid := range n.kids { + w.walkBody(kid, m, clip, depth+1) + } +} + +// add records one shape's visible box: its geometric extent cut back to the clip +// in force where it is drawn. +// +// A shape clipped away entirely is dropped rather than recorded with an empty +// box, because a figure is recognised by how many shapes are inside it — see +// [minFigureInk] — and counting ink the page never paints is the same error as +// including it in the extent. +func (w *inkWalker) add(m matrix, clip clipBox, sub []point, stroked bool) { + if len(sub) == 0 { + return + } + minX, minY := math.Inf(1), math.Inf(1) + maxX, maxY := math.Inf(-1), math.Inf(-1) + for _, p := range sub { + x, y := m.apply(p.x, p.y) + x, y = x*svgPointScale, y*svgPointScale + minX, maxX = math.Min(minX, x), math.Max(maxX, x) + minY, maxY = math.Min(minY, y), math.Max(maxY, y) + } + box, visible := clip.apply(CellRect{X0: minX, Y0: minY, X1: maxX, Y1: maxY}) + if !visible { + return + } + w.ink = append(w.ink, Ink{Rect: box, Stroked: stroked}) +} + +// What this deliberately does not solve, each measured rather than supposed. +// +// **An empty ruled table reads as a picture.** The text guard separates a table +// from a framed illustration by whether the cells hold words, and a blank form has +// none: page 558 of the sequential manual prints two warranty-registration forms +// whose labels sit only in the left column, and both come back as figures — 2 of +// that document's 238. Excluding anything the ruled-table shape guard claims would +// fix it and cost more than it saves, and that number is already recorded in +// docs/design/conversion.md: the shape guard alone passes 12 pages of the columns +// manual, and 2 of those — 22 and 44 — are the grids of framed illustrations this +// file exists to find. +// +// **A picture can still be cut by a caption printed over its artwork.** Not by its +// own labels any more: [trimToPicture] leaves a line the artwork encloses alone, and +// page 16 figure 2 of the columns manual — the case that used to lose its right +// third to the label »click« — now returns whole, which took that document from 15 +// figures cut to 3. What is left is the opposite arrangement, where the drawing +// really does run under the text: page 1's cover art continues behind the title +// block, so excluding the titles cuts it, and that figure is one of the 3. Nothing +// here can have both, because both are one rectangle. +// +// **Page furniture repeated in the same place is not identified as such.** The +// ink guard rejects every logo and badge in these two documents because they are +// two or three shapes, not because they repeat. A vector logo drawn with a hundred +// strokes, on every page, would come back as a figure on every page. What +// identifies furniture is repetition in the same position across pages, which is +// the same conclusion docs/design/conversion.md reaches about a running head and +// the same different input: a pass with the whole document in view. +// +// **A figure has no caption and no reading position within a region.** Both need +// the block work this file deliberately does not touch. The rectangle is here so +// that join can be geometric, which is what conversion.md records it has to be. +// +// **Nothing is stored.** The digest is computed and the bytes are returned; who +// writes them to the content-addressed store, and what row points at them, is the +// storage step. diff --git a/internal/doc/figures_fixture_test.go b/internal/doc/figures_fixture_test.go new file mode 100644 index 0000000..b4b285a --- /dev/null +++ b/internal/doc/figures_fixture_test.go @@ -0,0 +1,988 @@ +package doc_test + +import ( + "context" + "fmt" + "math" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" +) + +// These drive the figure reader against both real manuals, and they are the +// valuable half of its verification: a synthetic PDF cannot reproduce a page of +// framed line drawings, a cover ornament that every area-based guard reads as the +// page's largest picture, or 1,301 gradient-mesh slivers on two pages. +// +// The numbers asserted are the ones counted off renders of the pages, and the +// header of figures.go records where each came from. Where a count is a known +// over- or under-reading it is asserted as it stands, with the reason named, so a +// change that silently moves a different set of figures cannot pass by arriving at +// the same total. + +// figuresOf reads one page's figures. +func figuresOf(t *testing.T, path string, pages []doc.PageRuns, no int) []doc.Figure { + t.Helper() + if !extern.Available(extern.PDFToPPM) { + t.Skipf("%s is not installed", extern.PDFToPPM.Name) + } + figs, err := doc.PageFigures(context.Background(), path, pageOf(t, pages, no)) + if err != nil { + t.Fatalf("PageFigures page %d: %v", no, err) + } + return figs +} + +// areasOf reads one page's figure geometry without rendering anything, which is +// what the whole-document sweeps below use: rendering 201 figures would add half +// a minute to them and the guards are what is under test. +func areasOf(t *testing.T, path string, page *doc.PageRuns) []doc.Figure { + t.Helper() + return inkedBy(t, path, page, doc.FindFigures) +} + +// servedOf is what a conversion would be given for this page: [doc.ServedFigures], +// which is FindFigures with the crops that lie wholly inside another removed. A test +// about what a READER meets asks this one; a test about what the detector finds asks +// areasOf. +func servedOf(t *testing.T, path string, page *doc.PageRuns) []doc.Figure { + t.Helper() + return inkedBy(t, path, page, doc.ServedFigures) +} + +func inkedBy(t *testing.T, path string, page *doc.PageRuns, + find func([]doc.Ink, *doc.PageRuns) []doc.Figure) []doc.Figure { + t.Helper() + ink, err := doc.ExtractInk(context.Background(), path, page.No) + if err != nil { + t.Fatalf("ExtractInk page %d: %v", page.No, err) + } + return find(ink, page) +} + +// TestFiguresOfTheColumnsManualAreItsLineDrawings is the acceptance check on the +// document whose pictures are hardest: every illustration in it is vector, so +// pdfimages returns none of them, and its pages of framed drawings are ruled +// exactly the way its tables are. +func TestFiguresOfTheColumnsManualAreItsLineDrawings(t *testing.T) { + path, pages := rulesFixture(t, "thomas-drybox-amfibia") + + // Counted off renders of the pages. Page 42 prints four framed drawings and + // returns four, and page 22 three for three — both were one short until the + // clip was read, because a path drawn past its frame bridged the gap to the + // next drawing. Page 16 prints four panels and returns four, which is the case + // conversion.md recorded as "1 for 3" and had wrong twice over: it returned one + // figure, and the page prints four rather than three. Page 11 is one framed + // parts diagram with the loose accessory drawings inside the same frame. + want := map[int]int{ + 1: 1, 11: 1, 12: 1, 16: 4, 22: 3, 42: 4, + // The five ruled troubleshooting pages print no illustration at all, though + // each carries the two largest ink clusters in the document. Which guard + // rejects them is not the one it looks like — see + // TestARuledTablePageYieldsNoPicture. + 57: 0, 58: 0, 59: 0, 60: 0, 61: 0, + // Prose and an unruled specification table. + 62: 0, + } + for _, no := range sortedPageNumbers(want) { + figs := areasOf(t, path, pageOf(t, pages, no)) + if len(figs) != want[no] { + t.Errorf("page %d: %d figures, expected %d", no, len(figs), want[no]) + for i := range figs { + t.Logf(" %s", describe(&figs[i])) + } + } + } +} + +// TestFigureCountsOverBothWholeDocuments is the measurement the design rests on, +// and it is a whole-document sweep on purpose: a guard tuned on the pages someone +// looked at is exactly the thing that fires 500 times on the pages nobody did. +func TestFigureCountsOverBothWholeDocuments(t *testing.T) { + for _, tc := range []struct { + name string + pagesWith int + figures int + smallest float64 // the smallest figure's shorter side, in units + leastInk int + mostOnAPage int + maxTextOfReal float64 + }{ + // The columns manual: 59 figures on 27 of 68 pages, 3 to 4 on the pages of + // framed drawings. Its smallest figure's short side is 130 units — this + // document draws nothing small, which is why the size floor decides nothing + // on it at any value from 10 to 120. + // + // It was 46 on the same 27 pages before the clip was read, and the extra 13 + // are drawings that had been merged into a neighbour: the count rises where + // the page count does not, which is what tells a split from a new find. + // Its least-inked figure falls from 28 shapes to 26 for the same reason — a + // merged cluster held both drawings' shapes. + // + // Two of these moved when trimToPicture stopped cutting a drawing away from + // its own labels, and both moved because the old numbers were measuring the + // cut rather than the document. The smallest side was 128, which was page + // 52's process diagram amputated to 128.9 units tall; the real smallest + // drawing is page 48's second panel at 130.4, and no trim has ever touched + // it. The text ceiling rises from 9% to 10% for the same reason: the most + // texted accepted figure is now page 53's Polish process diagram at 9.9%, + // which keeps the three-line label block printed inside it. That is still + // far under maxFigureTextFraction's 15%, which is what this bound is for, + // and page 57's rejected tables are still at 37-39%. + // + // Merging candidate boxes that overlap moved nothing here at all, and that + // is a measurement rather than an omission: this document has no page where + // two candidates overlap, at any merge threshold from 0 to 1. Every number + // on this row is the same before and after, which is what says the merge + // pass cannot lose a picture on the document whose pictures were counted by + // eye. See mergeOverlapping in figures.go. + {"thomas-drybox-amfibia", 27, 59, 130, 26, 4, 0.10}, + // The sequential manual: 195 figures on 23 of 560 pages, and up to 31 on one + // page — its front matter carries two pages that are nothing but grids of + // small diagrams. Every figure in it is in the front matter or the back + // matter: the 34 language sections print prose and ruled tables and no + // illustration at all, which is docs/design/conversion.md's open problem of + // language-neutral content measured from the other side, and it is the reason + // a language-scoped conversion of this document would show a reader no + // pictures at all. + // + // 229 before the clip and 238 after, on the same 23 pages. 195 since + // candidate boxes that overlap are merged: 43 of those 238 were pieces of a + // drawing that had already been found, and the page count does not move, + // which is what tells a merge from a lost picture. Page 522 was rendered and + // counted by eye — 9 printed drawings, 13 figures before and 9 after — and so + // was page 524, which returned the hand out of its own drawing as a separate + // picture and now returns 4 boxes for its 4 drawings. + // + // Three of the columns below move with it, all in the same direction and for + // the same reason: the smallest and leanest candidates were fragments, and + // they are inside something else now. The smallest side rises from 20 units + // to 22, the least-inked figure from 28 shapes to 30, and the most-texted + // accepted figure from 6.0% to 6.3% — a merged box is larger, so a caption + // printed beside the drawing covers more of it. + {"dreame-l40-ultra", 23, 195, 22, 30, 31, 0.07}, + } { + t.Run(tc.name, func(t *testing.T) { + path, pages := rulesFixture(t, tc.name) + + var total, withFigures, mostOnAPage, leastInk int + smallest, maxText := 1e9, 0.0 + for i := range pages { + figs := areasOf(t, path, &pages[i]) + if len(figs) == 0 { + continue + } + withFigures++ + total += len(figs) + if len(figs) > mostOnAPage { + mostOnAPage = len(figs) + } + for j := range figs { + f := &figs[j] + // The drawn box, not the crop. The size floor is a guard and it + // judges the drawing, so the census of what it admitted has to ask + // the same box — since growToLabels this reads 24 off Rect, because + // the smallest drawing grew on one edge, and that number is a + // measurement of the crop rather than of the threshold. + drawn := f.DrawnExtent() + side := drawn.Width() + if drawn.Height() < side { + side = drawn.Height() + } + if side < smallest { + smallest = side + } + if leastInk == 0 || f.Ink < leastInk { + leastInk = f.Ink + } + if f.TextFraction > maxText { + maxText = f.TextFraction + } + } + } + t.Logf("%s: %d figures on %d of %d pages; most on one page %d; "+ + "smallest side %.0f units; least ink %d shapes; most text %.1f%%", + tc.name, total, withFigures, len(pages), mostOnAPage, + smallest, leastInk, 100*maxText) + + if total != tc.figures { + t.Errorf("%d figures, expected %d", total, tc.figures) + } + if withFigures != tc.pagesWith { + t.Errorf("figures on %d pages, expected %d", withFigures, tc.pagesWith) + } + if mostOnAPage != tc.mostOnAPage { + t.Errorf("most figures on one page = %d, expected %d", mostOnAPage, tc.mostOnAPage) + } + // The smallest and least-inked figures are asserted because they are + // what the two shape guards were set from. A change that raises either + // threshold shows up here as a lost figure rather than as a total that + // happens to still add up. + if int(smallest) != int(tc.smallest) { + t.Errorf("smallest figure side = %.0f units, expected %.0f", smallest, tc.smallest) + } + if leastInk != tc.leastInk { + t.Errorf("least ink in a figure = %d shapes, expected %d", leastInk, tc.leastInk) + } + // Every accepted figure is far under the text guard, and the tables it + // rejects are far over it. Asserting the accepted side pins the margin: + // if a table starts being accepted, this moves long before a count does. + if maxText > tc.maxTextOfReal { + t.Errorf("an accepted figure is %.1f%% text, expected all under %.1f%%", + 100*maxText, 100*tc.maxTextOfReal) + } + }) + } +} + +// TestARuledTablePageYieldsNoPicture records which guard actually rejects the +// columns manual's ruled tables, because the first version of this code asserted +// the wrong one. +// +// Page 57's two tables are the largest ink clusters in the document, 398x756 units +// each, and 37-39% of each is text — so the text guard would reject them. It never +// gets the chance: cairo draws each table as about fourteen rectangles, and +// fourteen is under minFigureInk, so the shape guard rejects them first. Taking the +// page's text away therefore changes nothing, which is the assertion here and the +// opposite of what was expected. Where the text guard does decide is measured by +// TestWhatTheTextGuardIsStillWorth. +func TestARuledTablePageYieldsNoPicture(t *testing.T) { + path, pages := rulesFixture(t, "thomas-drybox-amfibia") + page := pageOf(t, pages, 57) + + ink, err := doc.ExtractInk(context.Background(), path, page.No) + if err != nil { + t.Fatalf("ExtractInk: %v", err) + } + if figs := doc.FindFigures(ink, page); len(figs) != 0 { + t.Errorf("page 57 returned %d figures; it prints two tables and no picture", len(figs)) + } + + blind := *page + blind.Runs = nil + if figs := doc.FindFigures(ink, &blind); len(figs) != 0 { + t.Errorf("with no text page 57 returned %d figures; its tables are rejected on "+ + "shape, not on text, so this must not change", len(figs)) + } +} + +// TestRenderedFigureMatchesItsRectangle checks the half of the answer geometry +// cannot check: that the bytes come back, that they are a PNG, and that the pixels +// are the rectangle at the declared resolution. It renders rather than only +// measuring, which is what would catch a crop offset — the failure that produces a +// perfectly valid image of the wrong part of the page. +func TestRenderedFigureMatchesItsRectangle(t *testing.T) { + path, pages := rulesFixture(t, "thomas-drybox-amfibia") + + figs := figuresOf(t, path, pages, 11) + if len(figs) != 1 { + t.Fatalf("page 11 returned %d figures, expected its one parts diagram", len(figs)) + } + f := &figs[0] + if f.DPI != 216 { + t.Errorf("rendered at %d dpi, expected 216", f.DPI) + } + // 216 dpi is twice the 108 the coordinates are in, so the pixel size is the + // rectangle doubled, to within the outward rounding of the crop. + for _, c := range []struct { + name string + units float64 + pixels int + unitsScale float64 + }{ + {"width", f.Rect.Width(), f.PixelWidth, 2}, + {"height", f.Rect.Height(), f.PixelHeight, 2}, + } { + want := c.units * c.unitsScale + if d := float64(c.pixels) - want; d < 0 || d > 2 { + t.Errorf("%s = %d pixels, expected %.0f (the rectangle at %d dpi)", + c.name, c.pixels, want, f.DPI) + } + } + if len(f.Digest) != 64 { + t.Errorf("digest = %q, expected 64 hex characters", f.Digest) + } + if len(f.PNG) < 10000 { + t.Errorf("the parts diagram rendered to %d bytes; it is a full-page drawing", len(f.PNG)) + } + + // Rendering twice must give the same digest, or a content-addressed store + // gets a second copy of the same picture on every re-run of an idempotent job. + again := figuresOf(t, path, pages, 11) + if again[0].Digest != f.Digest { + t.Errorf("two renders of the same figure gave different digests:\n %s\n %s", + f.Digest, again[0].Digest) + } + + // MANUALBOX_FIGURE_DIR=/some/scratch writes the figures out to be looked at. + // Never inside the repository: these are pictures out of someone's copyrighted + // manual, and CI rejects a committed image outright. + if dir := os.Getenv("MANUALBOX_FIGURE_DIR"); dir != "" { + name := filepath.Join(dir, fmt.Sprintf("p%d-%d.png", f.Page, f.Index)) + if err := os.WriteFile(name, f.PNG, 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + t.Logf("wrote %s", name) + } +} + +// TestEveryFigureOfTheColumnsManualRenders is where the size cap comes from, and +// it is the only test that pays for every render: 59 figures, which is also the +// measurement of what a whole document's pictures cost. +// +// Set MANUALBOX_FIGURE_DIR to a scratch directory outside the repository to write +// them all out and look at them. Never inside it — these are pictures out of +// someone's copyrighted manual, and CI rejects a committed image outright. +func TestEveryFigureOfTheColumnsManualRenders(t *testing.T) { + path, pages := rulesFixture(t, "thomas-drybox-amfibia") + if !extern.Available(extern.PDFToPPM) { + t.Skipf("%s is not installed", extern.PDFToPPM.Name) + } + dir := os.Getenv("MANUALBOX_FIGURE_DIR") + + seen := make(map[string]string) + var total, largest int + var largestName string + var count int + for i := range pages { + figs, err := doc.PageFigures(context.Background(), path, &pages[i]) + if err != nil { + t.Fatalf("PageFigures page %d: %v", pages[i].No, err) + } + for j := range figs { + f := &figs[j] + count++ + total += len(f.PNG) + if len(f.PNG) > largest { + largest, largestName = len(f.PNG), describe(f) + } + if f.PixelWidth <= 0 || f.PixelHeight <= 0 { + t.Errorf("%s rendered to %dx%d pixels", describe(f), f.PixelWidth, f.PixelHeight) + } + // Two pictures with the same bytes are the same picture, and the store + // is content-addressed, so a repeat is a saving rather than a bug — but + // a repeat nobody expected is usually furniture that got through. + if prev, ok := seen[f.Digest]; ok { + t.Logf("same bytes as %s: %s", prev, describe(f)) + } + seen[f.Digest] = describe(f) + + if dir != "" { + name := filepath.Join(dir, fmt.Sprintf("p%02d-%d.png", f.Page, f.Index)) + if err := os.WriteFile(name, f.PNG, 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + } + } + t.Logf("%d figures, %d KB in total, largest %d KB (%s)", + count, total/1024, largest/1024, largestName) + if count != 59 { + t.Errorf("rendered %d figures, expected 59", count) + } + // The cap is two orders above the largest measured. If a figure ever gets + // within an order of it, the cap is the thing to revisit rather than this. + if largest > 1<<20 { + t.Errorf("largest figure is %d KB; the largest measured was 353 KB and "+ + "maxFigurePNGBytes was set from it", largest/1024) + } +} + +// TestNoFigureOverlapsAnotherOnEitherManual is the property the merge pass exists +// to establish, asserted over both whole documents rather than on the page the +// fault was reported on. +// +// A picture served twice is the worst thing this stage can do — a reader gets the +// drawing and then a scrap of the same drawing as if it were a second picture — and +// it cannot be caught by a count, because the count of a document nobody has looked +// at is unfalsifiable. This can be: no figure's box may share any area with +// another's on the same page. +// +// Before the merge pass the columns manual had 0 overlapping pairs on that test and +// the sequential one 53, of which 7 were one box wholly inside another. The strict +// containment census is kept separate because it was the case the report named. +// +// # The box the property is about is now the drawn one +// +// It was asserted of Rect, and Rect stopped being the box the merge pass produces +// once growToLabels started moving edges outward. Measured at both rects over both +// whole documents: the drawn boxes overlap in 0 pairs and nest in 0 on either +// document, exactly as before, and it is the rendered crops that overlap — 11 pairs, +// every one of them on page 5 or 6 of the sequential manual, with one crop wholly +// inside another on page 5. +// +// So the merge pass's property is intact and is asserted where it belongs, of the +// drawn box, at zero on both documents and on every page. +// +// # The crops split into a property and a census, and the band is why +// +// This used to demand that no two crops overlap on any page except the two +// front-matter plates. The drawn-box crop got that for free — clusterInk merges +// candidates whose ink touches, so two crops could only overlap once one had grown — +// and the band gives it up by construction: two drawings whose labels interleave have +// bands that overlap. So the two halves are now asserted differently. NESTING stays at +// zero everywhere, enforced by absorbNested rather than hoped for. OVERLAP is a pinned +// per-page census that may only go down. +func TestNoFigureOverlapsAnotherOnEitherManual(t *testing.T) { + for _, tc := range []struct { + name string + // overlapping crop pairs, and the pages they are on. + overlaps int + overlapPages []int + }{ + // Nothing overlaps on the columns manual, before or after the band: it has no + // page where two drawings' claims reach each other, which is the same structural + // fact that makes its 9 false claims harmless. + {"thomas-drybox-amfibia", 0, nil}, + // 26 pairs, of which 15 are on the two front-matter plates. The 11 on served + // pages are the band's stated cost: page 521's two crops each print a fragment + // of the other's label column. + {"dreame-l40-ultra", 26, []int{5, 6, 521, 522, 523, 531, 552, 553}}, + } { + name := tc.name + t.Run(name, func(t *testing.T) { + path, pages := rulesFixture(t, name) + inkOverlaps, inkNested := 0, 0 + cropOverlaps, cropNested := map[int]int{}, map[int]int{} + for i := range pages { + no := pages[i].No + // The SERVED set, because this test is about what a reader meets. The + // detector's own answer may hold a crop inside another; absorbNested is + // what removes it, and this is the assertion that it did. + figs := servedOf(t, path, &pages[i]) + for a := range figs { + // Growth only ever grows. This is the invariant that says a moved edge + // is a crop widened onto a label and never a drawing cut away, which is + // what trimToPicture does and what growth is the opposite of. Checked + // here rather than in its own sweep because this test already walks + // every figure of both documents. + if !within(figs[a].DrawnExtent(), figs[a].Rect) { + t.Errorf("page %d: %s has a crop that does not contain its drawn box "+ + "(%.1f,%.1f)-(%.1f,%.1f)", no, describe(&figs[a]), + figs[a].InkRect.X0, figs[a].InkRect.Y0, figs[a].InkRect.X1, figs[a].InkRect.Y1) + } + for b := range figs { + if a == b { + continue + } + fa, fb := &figs[a], &figs[b] + if a < b && overlaps(fa.DrawnExtent(), fb.DrawnExtent()) { + inkOverlaps++ + t.Errorf("page %d: the drawn boxes of figures %d and %d overlap\n %s\n %s", + no, a, b, describe(fa), describe(fb)) + } + if within(fa.DrawnExtent(), fb.DrawnExtent()) { + inkNested++ + } + if a < b && overlaps(fa.Rect, fb.Rect) { + cropOverlaps[no]++ + } + if within(fa.Rect, fb.Rect) { + cropNested[no]++ + } + } + } + } + t.Logf("%s: drawn boxes %d overlapping pair(s), %d nested; "+ + "crops %v overlapping pair(s) by page, %v nested by page", + name, inkOverlaps, inkNested, cropOverlaps, cropNested) + + if inkNested != 0 { + t.Errorf("%d drawn box(es) sit wholly inside another; a box inside a box "+ + "is a fragment of that drawing, served to a reader as a second picture", + inkNested) + } + // NESTING IS STILL ZERO EVERYWHERE, INCLUDING THE PLATES, and that is the + // half of this test the band did not get to weaken. A crop wholly inside + // another is a scrap of a drawing served as a picture of its own, and + // [absorbNested] removes it at the source by giving its labels to the crop + // that swallowed it. Before that pass the band produced 6 of them, on pages + // 529, 532, 546 and 550 — page 529 figure 7's band was figure 4's band with + // the top cut off, because both claimed the numbered step underneath. + if n := censusTotal(cropNested); n != 0 { + t.Errorf("%d crop(s) sit wholly inside another (%v); absorbNested exists "+ + "to make this impossible, so a non-zero count is that pass failing "+ + "rather than a cost of the band", n, cropNested) + } + // OVERLAP IS NOT ZERO ANY MORE, AND IT IS A COST RATHER THAN A DEFECT. This + // used to demand none off the plates, which the drawn-box crop got for free: + // clusterInk merges candidates whose ink touches, so two crops could only + // overlap once one had GROWN. The crop is now the band the page prints, and + // two drawings whose labels interleave have bands that overlap by + // construction — page 521's lidar and robot each print a fragment of the + // other's label column at their facing edges. + // + // The alternative is to merge the pair, and that is refused on the record by + // TestAPlateMergeOnSharedLabelsIsRefused: the transitive closure of + // overlapping want boxes takes page 521's three drawings into one crop 0.629 + // of the page. So the count is pinned instead, per page, and it may only go + // down. See docs/design/conversion.md. + if !slices.Equal(sortedPageNumbers(cropOverlaps), tc.overlapPages) || + censusTotal(cropOverlaps) != tc.overlaps { + t.Errorf("%d crop overlap(s) on pages %v, measured at %d on %v", + censusTotal(cropOverlaps), sortedPageNumbers(cropOverlaps), + tc.overlaps, tc.overlapPages) + } + }) + } +} + +// TestTheReportedPageKeepsItsCalloutLabels is the fault the user reported, on the +// page it was reported on and against the three drawings it was reported against: +// PDF page 521 of the sequential manual, the RU product overview, whose crops kept +// every leader line and lost the labels those leaders point at — so the leaders ended +// in nothing and the diagram could not be read against its parts. +// +// THIS TEST HAS ASSERTED THREE DIFFERENT DESIGNS AND THE SEQUENCE IS THE ARGUMENT. +// It began by asserting GROWN CROPS: widening reached 34 of the page's 40 claims and +// held only 23 of them whole, because a crop is a rectangle and these labels sit in +// two columns either side of each drawing. Then it asserted CARRIED LABELS, drawn by +// the reader beside a crop that was the drawn box exactly, and all 34 arrived whole — +// then 37, once a label wrapping past the bottom of its own drawing stopped losing its +// last three lines. +// +// It now asserts a BAND: the crop is the drawing together with everything the claim +// rule reaches, so the picture prints the labels itself and nothing is re-laid out. +// The drawn boxes below have not moved through any of that and are the same numbers +// this test always asserted. What moved is the crop, which is no longer one of them. +// +// The assertion that inverted is the one worth reading: a carried label used to have to +// be OUTSIDE the crop, or the reader would draw it over a picture that already printed +// it. It now has to be INSIDE, because the alt text says the picture prints it. +func TestTheReportedPageKeepsItsCalloutLabels(t *testing.T) { + path, pages := rulesFixture(t, "dreame-l40-ultra") + page := pageOf(t, pages, 521) + + figs := areasOf(t, path, page) + if len(figs) != 3 { + t.Fatalf("page 521 returned %d figures; it prints three drawings", len(figs)) + } + for i := range figs { + f := &figs[i] + t.Logf("figure %d: drawn (%.4f,%.4f)-(%.4f,%.4f) crop (%.4f,%.4f)-(%.4f,%.4f) %d label(s)", + f.Index, f.InkRect.X0, f.InkRect.Y0, f.InkRect.X1, f.InkRect.Y1, + f.Rect.X0, f.Rect.Y0, f.Rect.X1, f.Rect.Y1, len(f.Labels)) + } + total := 0 + for i, c := range []struct { + ink doc.CellRect + labels int + }{ + // The base station, seen from the front. Its nine labels are the two columns + // printed over the station's own footprint. Its right-hand six are NOT here and + // that is the conservative rule working — see below. + {doc.CellRect{X0: 539.0625, Y0: 96.0820, X1: 748.3594, Y1: 277.5645}, 9}, + // The lidar drawing, the one figures.go measured: its box ends at 263.0, its + // leader terminators are the marks at 259.6-263.0 that set that edge, and all + // eleven of its labels begin at 266.0. Three units, every one of them. Under the + // old pass the crop stopped at 397 and cut the label ending at 469, because the + // neighbouring drawing's labels start at 400. Carried as text, all eleven are + // whole and nothing has to arbitrate that corridor. + {doc.CellRect{X0: 65.9355, Y0: 116.9121, X1: 263.0098, Y1: 364.4355}, 11}, + // The underside, whose labels sit on both sides of it, and which carries the most. + // + // 14 UNTIL A WRAPPED LABEL STOPPED LOSING ITS TAIL, and this figure is the one + // the user photographed: `Монтажные отверстия для держателя насадки для швабры` + // is level with the drawing's foot, so lines 3, 4 and 5 of it sit below the box + // and were outside the band [runBeyond] claims in. Its first two lines were drawn + // on the picture and `держателя`, `насадки для` and `швабры` were left in the + // prose as three floating paragraphs. All five are carried now — see + // [doc.claimLabels] and TestNoLabelIsCarriedWithoutItsLaterLines. + {doc.CellRect{X0: 579.2813, Y0: 359.4258, X1: 765.1171, Y1: 522.7383}, 17}, + } { + f := &figs[i] + // A thousandth of a unit, which is two orders tighter than any difference these + // numbers are about. + if !sameBox(f.InkRect, c.ink, 0.001) { + t.Errorf("figure %d's drawn box is (%.4f,%.4f)-(%.4f,%.4f), measured at "+ + "(%.4f,%.4f)-(%.4f,%.4f)", i, + f.InkRect.X0, f.InkRect.Y0, f.InkRect.X1, f.InkRect.Y1, + c.ink.X0, c.ink.Y0, c.ink.X1, c.ink.Y1) + } + // THE CROP CONTAINS THE DRAWING AND IS WIDER THAN IT. Equal would mean the + // band took in nothing, which on a page whose every drawing is labelled means + // the claim rule found nothing to bring along. + if !holds(f.Rect, c.ink) { + t.Errorf("figure %d's crop (%.4f,%.4f)-(%.4f,%.4f) does not contain its drawn "+ + "box (%.4f,%.4f)-(%.4f,%.4f)", i, + f.Rect.X0, f.Rect.Y0, f.Rect.X1, f.Rect.Y1, + c.ink.X0, c.ink.Y0, c.ink.X1, c.ink.Y1) + } + if sameBox(f.Rect, c.ink, 0.001) { + t.Errorf("figure %d's crop is exactly its drawn box; every drawing on this "+ + "page is labelled, so the band must reach past it", i) + } + if len(f.Labels) != c.labels { + t.Errorf("figure %d carries %d label(s), measured at %d", i, len(f.Labels), c.labels) + for _, l := range f.Labels { + t.Logf(" %q", l) + } + } + total += len(f.Labels) + } + // 37, and read the sequence 23 -> 34 -> 37 rather than the last value. The grown crop + // REACHED 34 of this page's claims and held 23 of them whole; carrying each label as + // text held all 34, because a text run cannot be cut by a rectangle; and 37 is what + // the page prints, once a label that wraps past the bottom of its own drawing keeps + // the three lines that fall outside the box. + if total != 37 { + t.Errorf("page 521 carries %d labels in all, measured at 37 — the crop reached "+ + "34 and held 23 of them whole, and the last three are the tail of the wrapped "+ + "label the user reported", total) + } + + // The side that is refused, which is the conservative rule working — and THE + // PICTURE PRINTS IT ANYWAY, which is what the band bought. + // + // Figure 0's right corridor holds the label "Кнопка сброса" and then the lines of + // bullet description explaining it, so the gate refuses that side: naming a + // paragraph as a diagram's callout would be a lie to a screen reader, and it is the + // same rule that refuses all nine of the columns manual's false claims. Under the + // old design that refusal cost a reader the label entirely, and + // docs/design/conversion.md recorded these as measured-and-unfixable after both ways + // of DRAWING them had been refused. Neither refusal was about CROPPING, and the band + // takes the whole corridor: so they are absent from the alt text and present in the + // picture, which is the right answer to a claim of unknown kind. + for _, absent := range []string{"Кнопка сброса", "Индикатор Wi-Fi", "Датчик края"} { + for i := range figs { + for _, l := range figs[i].Labels { + if strings.Contains(l, absent) { + t.Errorf("figure %d carries %q in its labels; that whole side is "+ + "refused because its corridor holds bullet description", i, l) + } + } + } + r := runContaining(t, page, absent) + printed := false + for i := range figs { + if holds(figs[i].Rect, boxOf(r)) { + printed = true + } + } + if !printed { + t.Errorf("%q is in no figure's crop; the gate refuses to CALL it a label and "+ + "the band is what still shows it to a reader", absent) + } + } + + // AND THE FOURTH ONE IS STILL LOST, which is the cost this test exists to keep + // visible and a correction to what conversion.md recorded. + // + // `Датчики перепада высоты` was written down as a claim of figure 2 whose side the + // gate refused. It is not. It is claimed by NOTHING, so no gate ever sees it, the + // band cannot print what was never claimed, and the leaders in figure 2's top edge + // end in empty paper — which is how the user found it. + // + // figure 0, below 40.44 units past its box, against labelCorridor = 40 + // figure 2, above 27.43 units, within reach, and a REAL terminator that + // labelAlign refuses: 3.3x3.3 at (665.3, 339.3), 14.7 units + // off the midpoint of a label 140 units wide + // + // The second is the finding, and it is an asymmetry rather than a threshold: + // labelAlign is 4 units on both axes, a one-line label is 13-14 units TALL so +-4 of + // its midline is most of it, and a top-edge label is as WIDE as its phrase. Asking + // the mark to fall inside the run's own extent on that axis, instead of near its + // middle, is the rule that would claim this. It is unbuilt because it changes + // claimLabels, which every count in this package is expressed in. + // + // labelCorridor is deliberately NOT moved to 41 for the first row: a bound set from + // the one sample it has to admit is the fitted threshold minFigureWidth refuses on + // the record, and this label does not need it. + stranded := runContaining(t, page, "Датчики перепада высоты") + for i := range figs { + if holds(figs[i].Rect, boxOf(stranded)) { + t.Errorf("figure %d's crop now prints %q. That is an improvement, not a "+ + "failure — but it was measured as unreachable by 0.44 units and this "+ + "test is the record of that cost, so update the record", i, + "Датчики перепада высоты") + } + } + + // And the labels that ARE carried, named rather than counted: a count alone would + // pass if the pass claimed 34 of something else. Each of these was measured as + // outside its figure's drawn box, which is what made it invisible before. + for _, c := range []struct { + figure int + label string + }{ + {0, "Разъемы"}, + {1, "Микрофон"}, + {1, "Крышка лидара"}, + {2, "Датчик ковра"}, + // The one the old pass CUT: its far edge is at 469 and the crop stopped at 397. + {1, "Вспомогательная светодиодная подсветка"}, + // The tail of the wrapped label the user reported. `Монтажные` and + // `отверстия для` are level with the drawing's last few units and were always + // carried; these three sit 8, 20 and 32 units BELOW its box, which is why they + // were left in the prose while the head of their own label was drawn. + {2, "держателя"}, + {2, "насадки для"}, + {2, "швабры"}, + } { + r := runContaining(t, page, c.label) + f := &figs[c.figure] + if reached(f.InkRect, r) { + t.Errorf("%q is already inside figure %d's drawn box; it is meant to be a "+ + "label the crop had lost", c.label, c.figure) + } + found := false + for _, l := range f.Labels { + if strings.TrimSpace(l) == c.label { + found = true + } + } + // And it is INSIDE the crop, which is the inversion this design turns on: the + // alt text names it, so the picture has to print it. + if !holds(f.Rect, boxOf(r)) { + t.Errorf("%q is outside figure %d's crop (%.1f,%.1f)-(%.1f,%.1f); the alt "+ + "text names a label the picture does not show", c.label, c.figure, + f.Rect.X0, f.Rect.Y0, f.Rect.X1, f.Rect.Y1) + } + if !found { + t.Errorf("figure %d does not carry %q; it is one of the labels the leaders "+ + "point at", c.figure, c.label) + } + } +} + +// TestTheCropDoesNotChangeWhichLanguageAPictureBelongsTo is the funnel's promise +// applied to the one thing growth could break: a box grown sideways onto a label +// must not reach out of its own language column and be served to every household. +// attribute asks Figure.DrawnExtent for exactly that reason, and this checks the +// answer through doc.Convert rather than through the geometry. +// +// It is honest about what these two documents can and cannot show, because the +// answer is not what it looks like. The only fixture with side-by-side language +// columns is the columns manual, and it grows nothing at all — so on these fixtures +// the failure DrawnExtent prevents is unreachable, and this test cannot refute an +// attribute that read Rect. What it can do is pin that: the count is asserted +// together with "no served figure of that document is grown", so if a future change +// makes the columns manual grow, this stops being a vacuous pass and the counts move +// with it. The refutation that does not depend on a document behaving this way is +// TestAGrownCropDoesNotChangeAFiguresLanguage in convert_internal_test.go, which +// builds the case these fixtures cannot supply: page 14's real German and Polish +// columns and a figure whose drawn box is German while its crop reaches 76 units +// into Polish. The two tests deliberately say different things about the same rule — +// that one can fail, this one pins that the real documents still come back at the +// numbers they came back at. +// +// The sequential manual is the other side of the same fact: 14 of the 65 figures its +// Russian conversion serves ARE grown, so these counts are the counts of a corpus +// that actually contains grown boxes. Its regions are whole-page, though, which is +// why growth cannot move its attribution either. +// +// One note for whoever reads these numbers next. The 54, 53, 52 and 51 below are +// measured; conversion.md and CLAUDE.md carried 41, 40, 39 and 38 for a while, and +// those were stale from before the clip was read and candidate boxes were merged — +// convert_fixture_test.go already asserted the post-merge 53 for German alone and 65 +// for Russian while the docs still said 40 and 81. Growth moved none of them. +// wantWidenedColumns and wantWidenedSequential are how many served pictures have a +// crop reaching past their own drawing, which is how many the claim rule found +// something to bring along for. Measured; see TestABandCropDuplicatesThisMuchProse for +// the same fact counted in runes. +const ( + wantWidenedColumns = 1 + wantWidenedSequential = 25 +) + +func TestTheCropDoesNotChangeWhichLanguageAPictureBelongsTo(t *testing.T) { + // The de+uk conversion of the columns manual, which is the case that has no test + // elsewhere: a household reading two of its five languages. 54 figures, of which + // German sees 53 and Ukrainian 52, overlapping in the 51 that sit inside no + // region of their page — page 14's two photographs among them, which the render + // shows belong to neither text column. + both := convertFixture(t, "thomas-drybox-amfibia", "de", "uk") + neutral := 0 + for i := range both.Figures { + if both.Figures[i].Neutral { + neutral++ + } + } + if len(both.Figures) != 54 || len(both.FiguresFor("de")) != 53 || + len(both.FiguresFor("uk")) != 52 || neutral != 51 { + t.Errorf("de+uk stores %d figures, German sees %d and Ukrainian %d, %d neutral; "+ + "measured at 54, 53, 52 and 51", len(both.Figures), len(both.FiguresFor("de")), + len(both.FiguresFor("uk")), neutral) + } + // THE CROP IS NO LONGER THE DRAWN BOX, AND ATTRIBUTION DOES NOT NOTICE. That is + // the property this test is named for and the one that had to survive the band: + // [attribute] asks Figure.DrawnExtent, which is the ink alone, so widening the crop + // cannot move a picture into a language it does not belong to. The counts above + // were measured before the crop widened and must not move. + // + // This used to assert the opposite — Rect == DrawnExtent on every figure of both + // documents — with a note saying that if nothing ever grew, the counts stopped + // saying anything about growth. The band makes that equality false by design, so + // what replaces it is the distinction that actually matters: the drawn box is + // PRESENT, and it is the smaller of the two. + widenedColumns := 0 + for i := range both.Figures { + f := &both.Figures[i] + if f.InkRect == (doc.CellRect{}) { + t.Errorf("page %d figure %d carries no drawn box; attribute asked DrawnExtent "+ + "and would have fallen back to the crop, which is now wider than the "+ + "picture", f.Page, f.Index) + } + if !holds(f.Rect, f.DrawnExtent()) { + t.Errorf("page %d figure %d has a crop that does not contain its own drawing", + f.Page, f.Index) + } + if f.Rect != f.DrawnExtent() { + widenedColumns++ + } + } + + // The sequential manual's Russian. Its 65 and its page range 517-538 are asserted + // by TestConvertTheSequentialManualForRussian, as the columns manual's German 53 is + // by TestConvertTheColumnManualForGerman, and neither is restated here. + ru := convertFixture(t, "dreame-l40-ultra", "ru") + widenedSequential := 0 + for i := range ru.Figures { + f := &ru.Figures[i] + if f.InkRect == (doc.CellRect{}) { + t.Errorf("page %d figure %d carries no drawn box; attribute asked DrawnExtent "+ + "and would have fallen back to the crop", f.Page, f.Index) + } + if !holds(f.Rect, f.DrawnExtent()) { + t.Errorf("page %d figure %d has a crop that does not contain its own drawing, "+ + "(%.1f,%.1f)-(%.1f,%.1f) against (%.1f,%.1f)-(%.1f,%.1f)", + f.Page, f.Index, + f.Rect.X0, f.Rect.Y0, f.Rect.X1, f.Rect.Y1, + f.DrawnExtent().X0, f.DrawnExtent().Y0, f.DrawnExtent().X1, f.DrawnExtent().Y1) + } + if f.Rect != f.DrawnExtent() { + widenedSequential++ + } + } + + // AND THE HAZARD IS LIVE RATHER THAN ABSENT, which is the honest reading and the + // reason the two boxes are kept apart at all. "The crop equals the drawing" is no + // longer available as the reason attribution is safe; what makes it safe is that + // attribute never reads the crop. If either of these reaches 0 the loops above have + // stopped covering anything. + t.Logf("crops wider than their drawing: %d of %d on the columns manual, %d of %d "+ + "on the sequential manual's Russian", widenedColumns, len(both.Figures), + widenedSequential, len(ru.Figures)) + if widenedColumns != wantWidenedColumns || widenedSequential != wantWidenedSequential { + t.Errorf("crops wider than their drawing: %d and %d, measured at %d and %d", + widenedColumns, widenedSequential, wantWidenedColumns, wantWidenedSequential) + } + + // THE SAME HAZARD MOVED FROM THE RECTANGLE TO THE TEXT, and it is stated here + // rather than checked, because neither document can produce it and a check that + // cannot run is worse than a note that says so. + // + // A label is claimed from the corridor around a drawing, up to labelCorridor units + // out. On a parallel-columns page a figure sitting near a column edge could in + // principle claim a run from the NEIGHBOURING language's column and show it to a + // household that does not read that language — the one failure the funnel may not + // have, restated for text instead of for a crop. + // + // It is unreachable on these two documents for a structural reason and not because + // a rule prevents it. The parallel-columns manual is the only one with columns and + // it carries NO labels at all, asserted below; the sequential manual has labels and + // its regions are whole-page, so there is no neighbouring column to reach into. A + // third document with columns AND callouts is the first real test of it, and + // whoever adds one should expect to need a rule here. + for i := range both.Figures { + if n := len(both.Figures[i].Labels); n != 0 { + t.Errorf("page %d figure %d of the columns manual carries %d label(s); all "+ + "nine of that document's claims are false and every one must stay "+ + "refused — and it is that fact, not a rule, which makes the cross-column "+ + "hazard above unreachable today", + both.Figures[i].Page, both.Figures[i].Index, n) + } + } +} + +func describe(f *doc.Figure) string { + return fmt.Sprintf("page %d figure %d (%.1f,%.1f)-(%.1f,%.1f) %.0fx%.0f ink=%d text=%.1f%%", + f.Page, f.Index, f.Rect.X0, f.Rect.Y0, f.Rect.X1, f.Rect.Y1, + f.Rect.Width(), f.Rect.Height(), f.Ink, 100*f.TextFraction) +} + +// overlaps reports whether two boxes share any area. +func overlaps(a, b doc.CellRect) bool { + return math.Min(a.X1, b.X1) > math.Max(a.X0, b.X0) && + math.Min(a.Y1, b.Y1) > math.Max(a.Y0, b.Y0) +} + +// within reports whether the first box lies wholly inside the second. +func within(inner, outer doc.CellRect) bool { + return inner.X0 >= outer.X0 && inner.X1 <= outer.X1 && + inner.Y0 >= outer.Y0 && inner.Y1 <= outer.Y1 +} + +// sameBox compares two boxes edge by edge, to a tolerance. +func sameBox(a, b doc.CellRect, tol float64) bool { + return near(a.X0, b.X0, tol) && near(a.Y0, b.Y0, tol) && + near(a.X1, b.X1, tol) && near(a.Y1, b.Y1, tol) +} + +// censusTotal sums a per-page census. +func censusTotal(m map[int]int) int { + var n int + for _, v := range m { + n += v + } + return n +} + +// holds reports whether the outer box contains every part of the inner one, to the +// tolerance the boxes written down in this file are quoted at. The band design's +// central invariant is stated with it: a label the alt text names is inside the crop +// that is supposed to print it. +func holds(outer, inner doc.CellRect) bool { + const tol = 0.001 + return inner.X0 >= outer.X0-tol && inner.Y0 >= outer.Y0-tol && + inner.X1 <= outer.X1+tol && inner.Y1 <= outer.Y1+tol +} + +func boxOf(r *doc.TextRun) doc.CellRect { + return doc.CellRect{X0: r.X, Y0: r.Y, X1: r.X + r.Width, Y1: r.Y + r.Height} +} + +// reached reports whether a box takes in any part of a run. +// +// Its companion `boxed`, and `labelsTakenIn` with it, are gone with the pass they +// measured. Both existed to count what GROWING a crop bought — "reaches the run" rather +// than "holds all of it", which was the difference between 34 labels on page 521 and 23 +// — and that distinction is exactly what carrying a label as text removes: a text run +// has no rectangle, so there is no partial case left to count. The 34 and the 23 are +// kept where they mean something, in TestTheReportedPageKeepsItsCalloutLabels and +// TestALabelReachesAReaderWhole. +func reached(box doc.CellRect, r *doc.TextRun) bool { return overlaps(box, boxOf(r)) } + +// runContaining finds the one run holding a piece of text, and fails if the page +// does not print it exactly once — a label asserted by name is worth nothing if the +// name matches two runs or none. +func runContaining(t *testing.T, page *doc.PageRuns, text string) *doc.TextRun { + t.Helper() + var found *doc.TextRun + n := 0 + for i := range page.Runs { + if strings.Contains(page.Runs[i].Text, text) { + found = &page.Runs[i] + n++ + } + } + if n != 1 { + t.Fatalf("page %d prints %q in %d runs, expected exactly one", page.No, text, n) + } + return found +} + +func sortedPageNumbers(m map[int]int) []int { + out := make([]int, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Ints(out) + return out +} diff --git a/internal/doc/figures_internal_test.go b/internal/doc/figures_internal_test.go new file mode 100644 index 0000000..913139b --- /dev/null +++ b/internal/doc/figures_internal_test.go @@ -0,0 +1,2122 @@ +package doc + +import ( + "context" + "fmt" + "maps" + "math" + "os" + "slices" + "sort" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" +) + +// TestWhatTheTextGuardIsStillWorth removes the guard over both whole documents and +// records what changes, because twice now the answer has not been what was +// expected and both times the wrong belief was written down first. +// +// It was first claimed to be what rejects page 57 of the columns manual — two ruled +// tables, the largest ink clusters in that document. It is not: cairo draws each of +// those tables as about fourteen rectangles, under [minFigureInk], so the shape +// guard rejects them and the text guard never sees them. +// +// Then [trimToPicture] arrived and took most of the rest. Before it, the guard +// decided 1 cluster of the columns manual and 4 of the sequential one; after it, 0 +// and 1 — because a candidate that had reached over a text column now has that +// column trimmed off instead of being thrown away whole, which is the better of the +// two outcomes. What is left is one cluster in 275 across both documents. +// +// Narrowing the trim to lines the box has reached over did not move that: the trims +// it stopped making are the ones that cut a label out of the middle of a drawing, +// and a drawing keeping its own label is nowhere near [maxFigureTextFraction]. The +// counts below are the same on both documents before and after. +// +// Reading the clip moved both totals — 46 to 59 figures on the columns manual and +// 229 to 238 on the sequential one, because drawings that had been merged into a +// neighbour are now separate — and merging candidate boxes that overlap moved the +// sequential one back to 195, because pieces of one drawing are one drawing again. +// Neither moved the verdict: the guard still decides nothing on the columns manual +// and still decides page 53 alone on the other. +// +// So this test asserts the measured numbers rather than "the guard does something", +// and the guard is kept on the reasoning [ruleWalker.filled] sets out: the shape it +// handles — a ruled table with more parts than minFigureInk and cells full of +// text — is an ordinary thing for a document to contain, and the next manual gets no +// say in which cases this code understands. +func TestWhatTheTextGuardIsStillWorth(t *testing.T) { + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixtures and run this", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFToHTML, extern.PDFToCairo} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + + for _, tc := range []struct { + name string + with, none int + pages []int + }{ + {"thomas-drybox-amfibia", 59, 59, nil}, + // Page 53 prints the French recycling label — a picture with a paragraph + // set inside it, 34.7% text, which is exactly the case the guard cannot + // tell from a table and the reason its remaining decision is a loss rather + // than a save. + {"dreame-l40-ultra", 195, 196, []int{53}}, + } { + name := tc.name + t.Run(name, func(t *testing.T) { + m, err := fixture.Load("../../testdata/fixtures", name) + if err != nil { + t.Fatalf("load manifest: %v", err) + } + path, err := m.Fetch(context.Background()) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + pages, err := ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + + var withGuard, withoutGuard int + var gained []int + for i := range pages { + p := &pages[i] + ink, err := ExtractInk(context.Background(), path, p.No) + if err != nil { + t.Fatalf("ExtractInk page %d: %v", p.No, err) + } + noText := defaultGuards + noText.maxText = 1 + on := len(findFigures(ink, p, defaultGuards)) + off := len(findFigures(ink, p, noText)) + withGuard += on + withoutGuard += off + if off > on { + gained = append(gained, p.No) + for _, f := range findFigures(ink, p, noText) { + if f.TextFraction > maxFigureTextFraction { + t.Logf(" page %d rejected %v ink=%d text=%.1f%%", + p.No, f.Rect, f.Ink, 100*f.TextFraction) + } + } + } + } + t.Logf("%s: %d figures with the text guard, %d without; "+ + "the extra ones are on pages %v", name, withGuard, withoutGuard, gained) + if withGuard != tc.with || withoutGuard != tc.none { + t.Errorf("%d figures with the guard and %d without, expected %d and %d", + withGuard, withoutGuard, tc.with, tc.none) + } + if len(gained) != len(tc.pages) { + t.Errorf("the guard decides pages %v, expected %v", gained, tc.pages) + } + for i := range tc.pages { + if i < len(gained) && gained[i] != tc.pages[i] { + t.Errorf("the guard decides pages %v, expected %v", gained, tc.pages) + break + } + } + }) + } +} + +// loadFigureInk reads every page's ink once, so a sweep over thresholds does not +// re-run pdftocairo 560 times per value. +func loadFigureInk(t *testing.T, name string) (pages []PageRuns, ink [][]Ink) { + t.Helper() + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixtures and run this", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFToHTML, extern.PDFToCairo} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + m, err := fixture.Load("../../testdata/fixtures", name) + if err != nil { + t.Fatalf("load manifest: %v", err) + } + path, err := m.Fetch(context.Background()) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + pages, err = ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + ink = make([][]Ink, len(pages)) + for i := range pages { + ink[i], err = ExtractInk(context.Background(), path, pages[i].No) + if err != nil { + t.Fatalf("ExtractInk page %d: %v", pages[i].No, err) + } + } + return pages, ink +} + +// TestGuardSweep prints how each threshold behaves over both whole documents, one +// constant at a time with the rest held at their defaults. It is the measurement +// the constants in figures.go are set from, and it is a test rather than a script +// so that a later change can re-run it instead of trusting the numbers written +// down. +// +// It asserts only the one thing a sweep can assert: that the chosen value is not +// on a cliff. A threshold one step either side of the default must not move the +// figure count by more than a tenth. +func TestGuardSweep(t *testing.T) { + for _, name := range []string{"thomas-drybox-amfibia", "dreame-l40-ultra"} { + t.Run(name, func(t *testing.T) { + pages, ink := loadFigureInk(t, name) + count := func(g figureGuards) int { + var n int + for i := range pages { + n += len(findFigures(ink[i], &pages[i], g)) + } + return n + } + base := count(defaultGuards) + t.Logf("%s: %d figures at the defaults", name, base) + + for _, v := range []int{2, 5, 10, 15, 20, 25, 30, 40, 60, 100, 200} { + g := defaultGuards + g.minInk = v + t.Logf(" minFigureInk=%-3d -> %d figures", v, count(g)) + } + for _, v := range []float64{10, 20, 30, 40, 50, 60, 80, 120} { + g := defaultGuards + g.minWidth, g.minHeight = v, v + t.Logf(" minFigureSize=%-4.0f -> %d figures", v, count(g)) + } + for _, v := range []float64{0.02, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5, 1} { + g := defaultGuards + g.maxText = v + t.Logf(" maxFigureTextFraction=%-4.2f -> %d figures", v, count(g)) + } + + // How many figures overlap a line of text at all. Zero would be the + // ideal, and the shortfall is this file's main known cost — see + // [minFigureWidth]'s neighbours in figures.go for the cause. + var overlapping, figures int + for i := range pages { + var dropped DroppedRuns + runs := usableRuns(pages[i].Runs, pages[i].Width, pages[i].Height, &dropped) + for _, f := range findFigures(ink[i], &pages[i], defaultGuards) { + figures++ + for j := range runs { + r := &runs[j] + if len([]rune(strings.TrimSpace(r.Text))) < 5 { + continue + } + if textFraction(f.Rect, runs[j:j+1]) > 0 { + overlapping++ + break + } + } + } + } + t.Logf(" %d of %d figures overlap a line of five characters or more", + overlapping, figures) + + if os.Getenv("MANUALBOX_FIGURE_SHOW") != "" { + small := withSize(defaultGuards, 20) + for i := range pages { + for _, f := range findFigures(ink[i], &pages[i], small) { + if f.Rect.Width() < minFigureWidth || f.Rect.Height() < minFigureHeight { + t.Logf(" under the size floor: page %d %v %.0fx%.0f ink=%d", + f.Page, f.Rect, f.Rect.Width(), f.Rect.Height(), f.Ink) + } + } + } + } + + // The ink guard sits on a plateau and that is asserted, because it is the + // guard that separates a picture from page furniture and a value on a + // cliff there would be a value fitted to these two documents. + for _, step := range []struct { + name string + g figureGuards + }{ + {"minInk one lower", withInk(defaultGuards, minFigureInk-5)}, + {"minInk one higher", withInk(defaultGuards, minFigureInk+5)}, + } { + got := count(step.g) + if d := got - base; d > base/10 || -d > base/10 { + t.Errorf("%s changes the count from %d to %d, more than a tenth; "+ + "the default is on a cliff", step.name, base, got) + } + } + + // The size floor deliberately is not asserted that way, because it has no + // plateau to sit on: see [minFigureWidth]. What is asserted is the one + // measured property of it — that on the columns manual it decides nothing + // at all, so a change to it can only be a change to the other document. + if name == "thomas-drybox-amfibia" { + for _, v := range []float64{10, 40, 120} { + if got := count(withSize(defaultGuards, v)); got != base { + t.Errorf("the size floor at %.0f gives %d figures where the default "+ + "gives %d; on this document it used to decide nothing", v, got, base) + } + } + } + }) + } +} + +// TestMergeThresholdSweep is the evidence behind [figureMergeOverlap], and the +// evidence is that there is nothing for a threshold to separate. +// +// It sweeps how much of the smaller of two candidate boxes may lie inside the other +// before they are read as one picture, from 1 — which disables the pass, since no +// overlap can exceed it — down to 0, and prints the count over each whole document. +// Two things are asserted rather than only printed. +// +// The first is that the parallel-columns manual does not move at any value. It has +// no page where two candidates overlap, so this whole change is the other document's +// and the manual whose pictures were counted by eye cannot lose one to it. +// +// The second is that the sequential manual sits on a plateau at the bottom of the +// range rather than on a cliff: 0, 0.01, 0.05 and 0.1 give 195, 194, 197 and 196 +// figures, against 213 at 0.5 and 229 at containment. That is the claim the value +// rests on. The 53 overlapping pairs on that +// document run 1.00, 0.96, 0.91 … 0.11, 0.10, 0.01 with no gap, every one of them +// was rendered as a crop of the two boxes' union and looked at, and every one is a +// single printed drawing that clustered in pieces — so a threshold anywhere in that +// range would be deciding a case that does not exist, and the counts say the same +// thing from the other side. +// +// The counts are NOT monotonic in the threshold and that is expected rather than a +// fault: merging happens before the guards, so two candidates that were each under +// [minFigureInk] can merge into one that passes, and a document can gain a figure by +// merging. The sequential manual does, at 0.75. +func TestMergeThresholdSweep(t *testing.T) { + for _, name := range []string{"thomas-drybox-amfibia", "dreame-l40-ultra"} { + t.Run(name, func(t *testing.T) { + pages, ink := loadFigureInk(t, name) + count := func(v float64) int { + g := defaultGuards + g.mergeOverlap = v + var n int + for i := range pages { + n += len(findFigures(ink[i], &pages[i], g)) + } + return n + } + off := count(1) + base := count(figureMergeOverlap) + t.Logf("%s: %d figures with the merge off, %d at the default", name, off, base) + for _, v := range []float64{0.999, 0.9, 0.75, 0.5, 0.25, 0.1, 0.05, 0.01, 0} { + t.Logf(" mergeOverlap=%-5.3g -> %d figures", v, count(v)) + } + + if name == "thomas-drybox-amfibia" { + for _, v := range []float64{0, 0.25, 0.5, 0.999} { + if got := count(v); got != off { + t.Errorf("at %.3f this document gives %d figures against %d with "+ + "the merge off; it has no overlapping candidates and must not move", + v, got, off) + } + } + } + lo, hi := base, base + for _, v := range []float64{0.01, 0.05, 0.1} { + got := count(v) + lo, hi = min(lo, got), max(hi, got) + } + // A twentieth, the same shape of bound TestGuardSweep puts on the ink + // guard. Measured spread on this document is 194..197 against a default + // of 195, which is under 2%. + if hi-lo > base/20 { + t.Errorf("between 0 and 0.1 the count ranges over %d..%d against a "+ + "default of %d; the default is on a cliff, and it was chosen "+ + "because there is no case in that range for a threshold to decide", + lo, hi, base) + } + }) + } +} + +func withInk(g figureGuards, v int) figureGuards { g.minInk = v; return g } +func withSize(g figureGuards, v float64) figureGuards { g.minWidth, g.minHeight = v, v; return g } + +// TestTrimOnlyPullsOffALineItReachedOver drives [trimToPicture] with the four +// arrangements measured on the columns manual, at their real coordinates, so the +// rule is pinned without a PDF. +// +// The four are the whole argument for the rule and each one is a page: +// +// page 16 fig 2 »click« printed inside the panel, artwork on all four sides +// page 24 fig 0 the same label at the drawing's RIGHT EDGE, no artwork past it +// page 52 fig 0 a line of body text above the diagram, reaching in from outside +// page 1 fig 0 the cover title block, reaching in from outside AND above +// +// Page 24 is why the rule is containment and not ink on more than one side: that +// »click« has ink only to its left and below, exactly like page 1's title, and the +// two must come out opposite ways. What separates them is that one is inside the box +// and the other is not. +func TestTrimOnlyPullsOffALineItReachedOver(t *testing.T) { + for _, tc := range []struct { + name string + area CellRect + text []TextRun + want CellRect + }{{ + // The label sits at 209-238 within a panel running to 288. It used to cost + // the drawing everything past x=209. + name: "a label the panel encloses is left alone", + area: CellRect{42.8, 466.5, 288.4, 643.4}, + text: []TextRun{{X: 209, Y: 530, Width: 29, Height: 17, Text: "»click«"}}, + want: CellRect{42.8, 466.5, 288.4, 643.4}, + }, { + // The same label flush against the drawing's right edge, inside it by half a + // unit. Ink cannot tell this from prose; containment can. + name: "a label at the very edge is still inside", + area: CellRect{42.8, 197.0, 288.4, 373.0}, + text: []TextRun{{X: 262, Y: 304, Width: 25, Height: 14, Text: "»click«"}}, + want: CellRect{42.8, 197.0, 288.4, 373.0}, + }, { + // One line of German body text ending just inside the diagram's top edge. + // The top comes down off it and nothing else moves. + name: "a line of prose reaching in from above is trimmed off", + area: CellRect{323.1, 379.3, 582.5, 567.5}, + text: []TextRun{ + {X: 323, Y: 363, Width: 31, Height: 17, Text: "erzielen:"}, + // The diagram's own labels, which used to go with it. + {X: 357, Y: 471, Width: 37, Height: 13, Text: "Absaugen und"}, + {X: 390, Y: 554, Width: 60, Height: 13, Text: "Lösen und Auswaschen"}, + }, + want: CellRect{323.1, 380.0, 582.5, 567.5}, + }, { + // The cover: the whole title block, five lines stepping down and to the + // right out of the art. The cheaper edge is taken each round, which is why + // this needs all five rather than a sample — the order they come off in is + // the behaviour. + name: "the cover title block is trimmed off two edges", + area: CellRect{37.7, 324.0, 663.0, 819.2}, + text: []TextRun{ + {X: 55, Y: 301, Width: 387, Height: 24, + Text: "29924_Saugerbeschriftungen_DryBoxAmfibia.ind"}, + {X: 534, Y: 321, Width: 163, Height: 34, Text: "GEBRAUCHSANLEITUNG"}, + {X: 568, Y: 354, Width: 152, Height: 34, Text: "INSTRUKCJA OBSŁUGI"}, + {X: 602, Y: 386, Width: 248, Height: 34, Text: "РУКОВОДСТВО ПО ЭКСПЛУАТАЦИИ"}, + {X: 636, Y: 418, Width: 201, Height: 34, Text: "ІНСТРУКЦІЯ З ЕКСПЛУАТАЦІЇ"}, + }, + want: CellRect{37.7, 355.0, 568.0, 819.2}, + }, { + // The floor under the rule: a run of three runes is never trimmed for, even + // when it does reach over the edge. Page 11's diagram numbers its parts 1 to + // 39 and several sit against the frame. + name: "a short run is not trimmed for even when it reaches over", + area: CellRect{100, 100, 300, 300}, + text: []TextRun{{X: 60, Y: 150, Width: 50, Height: 14, Text: "12"}}, + want: CellRect{100, 100, 300, 300}, + }, { + // And the cap above it, which the two rules enforce together: the line reaches + // over the LEFT edge only, so the left edge is the only one that may move, and + // moving it past a third of the side is refused. The candidate is left whole + // for the text guard to reject rather than whittled into a plausible picture. + // Before the reach rule this trimmed the top instead — an edge the line never + // crossed — and that is what "whittled" meant. + name: "no edge moves by more than a third of its side", + area: CellRect{100, 100, 300, 300}, + text: []TextRun{{X: 0, Y: 150, Width: 200, Height: 14, Text: "a whole line of prose"}}, + want: CellRect{100, 100, 300, 300}, + }} { + t.Run(tc.name, func(t *testing.T) { + got := trimToPicture(tc.area, tc.text) + if got != tc.want { + t.Errorf("trimToPicture(%v) = %v, expected %v", tc.area, got, tc.want) + } + }) + } +} + +// TestPNGSizeReadsTheHeader covers the one piece of byte-level parsing here +// without a PDF, including the two malformed cases that would otherwise be stored +// as a figure: an empty stdout with a zero exit status, and output that is not a +// PNG at all. +func TestPNGSizeReadsTheHeader(t *testing.T) { + // A 3x2 PNG, IHDR and all, written out by hand: signature, then the IHDR + // chunk's length, type, width, height and the rest. + good := []byte("\x89PNG\r\n\x1a\n" + + "\x00\x00\x00\rIHDR" + + "\x00\x00\x00\x03\x00\x00\x00\x02\x08\x06\x00\x00\x00") + w, h, err := pngSize(good) + if err != nil { + t.Fatalf("pngSize: %v", err) + } + if w != 3 || h != 2 { + t.Errorf("pngSize = %dx%d, expected 3x2", w, h) + } + + for _, tc := range []struct { + name string + data []byte + }{ + {"empty", nil}, + {"truncated", good[:12]}, + {"not a png", []byte("%PDF-1.4\nnot an image at all really")}, + {"png but no IHDR first", append([]byte("\x89PNG\r\n\x1a\n"), + []byte("\x00\x00\x00\rpHYs\x00\x00\x00\x03\x00\x00\x00\x02\x08")...)}, + } { + if _, _, err := pngSize(tc.data); err == nil { + t.Errorf("%s: pngSize accepted it", tc.name) + } + } +} + +// TestOnPageInkDropsWhatCannotBePartOfAPicture pins the two structural filters +// with the real coordinates that motivated them. +func TestOnPageInkDropsWhatCannotBePartOfAPicture(t *testing.T) { + const w, h = 892, 850 + ink := []Ink{ + // Cairo's compositing rect, from page 57 of the columns manual. + {Rect: CellRect{-196.4, -187.1, 1089.4, 1037.5}}, + // One of the columns manual's full-width section bands. + {Rect: CellRect{0, 311.5, 891.8, 508.3}}, + // A real drawing, from page 42. + {Rect: CellRect{42.8, 45.5, 288.4, 241.4}}, + // A degenerate shape: a moveto with no extent. + {Rect: CellRect{100, 100, 100, 100}}, + } + kept := onPageInk(ink, w, h) + if len(kept) != 1 { + t.Fatalf("kept %d of 4 shapes, expected only the drawing", len(kept)) + } + if kept[0].Rect.X0 != 42.8 { + t.Errorf("kept %v, expected the drawing at x=42.8", kept[0].Rect) + } +} + +// TestFindFiguresAppliesBothGuards drives the geometry with hand-built ink, so the +// guards are checked without poppler and without a PDF. The shapes are the measured +// ones: a language badge is three shapes, a picture is many. +func TestFindFiguresAppliesBothGuards(t *testing.T) { + page := &PageRuns{No: 7, Width: 892, Height: 850} + + // A picture: 40 small strokes chained into a 200x200 area at (100,100). + var ink []Ink + for i := range 40 { + x := 100 + float64(i)*5 + ink = append(ink, Ink{Rect: CellRect{x, 100, x + 6, 300}, Stroked: true}) + } + // A badge: three shapes in a 67x60 area, far from the picture. + for range 3 { + ink = append(ink, Ink{Rect: CellRect{600, 20, 667, 80}}) + } + // A thin rule: long, but under the height floor. + ink = append(ink, Ink{Rect: CellRect{100, 700, 500, 702}}) + + figs := FindFigures(ink, page) + if len(figs) != 1 { + t.Fatalf("found %d figures, expected 1", len(figs)) + return + } + got := figs[0] + if got.Page != 7 || got.Index != 0 { + t.Errorf("figure is page %d index %d, expected page 7 index 0", got.Page, got.Index) + } + if got.Ink != 40 { + t.Errorf("ink = %d, expected the 40 strokes", got.Ink) + } + if got.Rect != (CellRect{100, 100, 301, 300}) { + t.Errorf("rect = %v, expected the strokes' bounding box", got.Rect) + } + + // Now fill the same area with text. It must stop being a picture. + page.Runs = []TextRun{ + {X: 105, Y: 105, Width: 190, Height: 190, Text: "a whole paragraph's worth"}, + } + if figs := FindFigures(ink, page); len(figs) != 0 { + t.Errorf("an area %.0f%% covered by text came back as a figure", + 100*figs[0].TextFraction) + } +} + +// TestFiguresAreInReadingOrder pins the order, because a figure's whole value to +// the reader is landing in the right place. +func TestFiguresAreInReadingOrder(t *testing.T) { + page := &PageRuns{No: 1, Width: 892, Height: 850} + // Three pictures: two side by side at the top, one below. Built bottom-right + // first so the sort has something to do. + corners := [][2]float64{{500, 500}, {500, 100}, {100, 100}} + var ink []Ink + for _, c := range corners { + for i := range 25 { + x := c[0] + float64(i)*4 + ink = append(ink, Ink{Rect: CellRect{x, c[1], x + 5, c[1] + 120}}) + } + } + figs := FindFigures(ink, page) + if len(figs) != 3 { + t.Fatalf("found %d figures, expected 3", len(figs)) + } + want := [][2]float64{{100, 100}, {500, 100}, {500, 500}} + for i := range figs { + if figs[i].Rect.X0 != want[i][0] || figs[i].Rect.Y0 != want[i][1] { + t.Errorf("figure %d starts at (%.0f,%.0f), expected (%.0f,%.0f)", + i, figs[i].Rect.X0, figs[i].Rect.Y0, want[i][0], want[i][1]) + } + if figs[i].Index != i { + t.Errorf("figure at position %d carries index %d", i, figs[i].Index) + } + } +} + +// TestAFragmentDrawnInsideADrawingIsNotItsOwnPicture is the fault the user +// reported, at the coordinates it was reported at. +// +// Page 524 of the sequential manual draws a hand holding a pin over the robot's +// underside. The hand's strokes touch none of the robot's, so the shape-level pass +// clusters it alone, and it was served as a picture of its own: a duplicate scrap +// of the drawing it came out of. The two boxes are the measured ones, x=279-412 +// y=134-256 for the robot and x=375-416 y=146-184 for the hand — which is 90.8% of +// the hand inside the robot and NOT containment. The pin pokes 4 units past the +// robot's right edge, which is why a containment test alone would not have fixed +// the case it was reported on. +func TestAFragmentDrawnInsideADrawingIsNotItsOwnPicture(t *testing.T) { + page := &PageRuns{No: 524, Width: 918, Height: 631} + + // The robot, drawn as chains of overlapping strokes along the top, the bottom + // and the left of x=279-412 y=134-256. Deliberately open on the right between + // y=146 and y=184, so no shape of the robot is anywhere near the hand. + ink := chainX(279, 412, 134, 140, 7) + ink = append(ink, chainX(279, 412, 250, 256, 7)...) + ink = append(ink, chainY(134, 256, 279, 285, 7)...) + // The hand: a chain of its own, dense enough to clear the ink guard by itself — + // which is what made it a picture — reaching 4 units past the robot's right edge + // and touching nothing the robot drew. + hand := chainY(146, 184, 375, 416, 1.5) + if len(hand) < minFigureInk { + t.Fatalf("the hand is %d shapes; it has to pass the ink guard alone", len(hand)) + } + ink = append(ink, hand...) + + figs := FindFigures(ink, page) + if len(figs) != 1 { + t.Fatalf("found %d figures, expected the drawing and its hand to be one", len(figs)) + } + // The merged box is the union, so the parent keeps everything it had and gains + // only what the fragment reached past it. + if got := figs[0].Rect; got != (CellRect{279, 134, 416, 256}) { + t.Errorf("rect = %v, expected the union x=279-416 y=134-256", got) + } + if figs[0].Ink != len(ink) { + t.Errorf("ink = %d, expected all %d shapes counted inside the merged box", + figs[0].Ink, len(ink)) + } +} + +// TestBoxesThatOnlyTouchAreNotMerged is the other side of the rule, and it is what +// keeps two drawings printed side by side apart. +// +// Exactly touching is not overlapping: the shape-level pass has already joined +// everything whose boxes meet, so a second pass that merged on contact would only +// undo its own answer. The gaps that carry two real drawings apart on these +// documents are much wider than this — page 524's two halves are 23 units apart and +// page 522's two mop pads 46 — so this pins the boundary at its tightest. +func TestBoxesThatOnlyTouchAreNotMerged(t *testing.T) { + for _, tc := range []struct { + name string + boxes []CellRect + want int + }{ + {"sharing a vertical edge", + []CellRect{{100, 100, 200, 200}, {200, 100, 300, 200}}, 2}, + {"sharing a horizontal edge", + []CellRect{{100, 100, 200, 200}, {100, 200, 200, 300}}, 2}, + {"meeting at a corner", + []CellRect{{100, 100, 200, 200}, {200, 200, 300, 300}}, 2}, + {"a unit apart", + []CellRect{{100, 100, 200, 200}, {201, 100, 300, 200}}, 2}, + // Overlapping on one axis only is not overlapping: two drawings printed + // side by side share a horizontal band and are still two drawings. + {"overlapping on one axis only", + []CellRect{{100, 100, 200, 200}, {201, 150, 300, 250}}, 2}, + {"overlapping by one unit on both axes", + []CellRect{{100, 100, 200, 200}, {199, 199, 300, 300}}, 1}, + } { + got := mergeOverlapping(append([]CellRect(nil), tc.boxes...), figureMergeOverlap) + if len(got) != tc.want { + t.Errorf("%s: %d box(es), expected %d — %v", tc.name, len(got), tc.want, got) + } + } +} + +// TestMergingRunsToAFixpoint covers the case one pass cannot: a merged box is +// bigger than either of its parts and can reach a third that neither part reached. +func TestMergingRunsToAFixpoint(t *testing.T) { + // Three boxes on a diagonal, each overlapping only the next. Built out of + // order, because the merge must not depend on the order they arrive in. + boxes := []CellRect{{280, 280, 380, 380}, {100, 100, 200, 200}, {190, 190, 290, 290}} + got := mergeOverlapping(boxes, figureMergeOverlap) + if len(got) != 1 { + t.Fatalf("%d boxes, expected the chain to collapse to one: %v", len(got), got) + } + if got[0] != (CellRect{100, 100, 380, 380}) { + t.Errorf("box = %v, expected the whole chain x=100-380 y=100-380", got[0]) + } +} + +// TestFindFiguresIsReproducible is here because it was not, and it is asserted at a +// threshold the shipped one does not use, on purpose. +// +// [clusterInk] collects its groups in a map, so they come out in a random order. At +// [figureMergeOverlap]'s zero that cannot matter — merging only grows a box, so it +// never destroys an intersection, and the answer is the connected components of the +// overlap relation whatever order they are visited in. Above zero it matters a +// great deal, because a merged box is wider and the smaller box's share of it falls: +// TestMergeThresholdSweep, with the boxes left in map order, reported 195, 197 and +// 196 figures at 0.01, 0.05 and 0.1 and then 194 to 200 for the same three +// thresholds later in the same run. +// +// So this drives the merge at 0.5, where the order decides the outcome, and pins +// that twenty runs agree. Reproducibility is not a nicety here: these bytes go into +// a content-addressed store, and a page that clusters differently on a re-run stores +// the same picture twice. +func TestFindFiguresIsReproducible(t *testing.T) { + page := &PageRuns{No: 1, Width: 892, Height: 850} + g := defaultGuards + g.mergeOverlap = 0.5 + + // A row of overlapping corners of different sizes, which is the arrangement + // where a merge can drop a later pair below the threshold. + var ink []Ink + for i := range 8 { + x := 100 + float64(i)*55 + y := 100 + float64(i)*9 + w := 60 + float64(i)*12 + ink = append(ink, chainX(x, x+w, y, y+5, 5)...) + ink = append(ink, chainY(y, y+w, x, x+5, 5)...) + } + + first := findFigures(ink, page, g) + if len(first) < 2 { + t.Fatalf("the arrangement collapsed to %d figure(s); it has to leave several "+ + "for the order to decide between", len(first)) + } + for range 20 { + got := findFigures(ink, page, g) + if len(got) != len(first) { + t.Fatalf("%d figures on one run and %d on another", len(first), len(got)) + } + for i := range got { + if got[i].Rect != first[i].Rect { + t.Fatalf("figure %d is %v on one run and %v on another", + i, first[i].Rect, got[i].Rect) + } + } + } +} + +// chainX lays overlapping strokes along a horizontal line from lo to hi, ending +// exactly on hi, so that they cluster into one shape group. A picture's strokes +// meet, which is what [clusterInk] turns on, and a hand-built test that forgets +// that measures nothing. +func chainX(lo, hi, y0, y1, step float64) []Ink { + var ink []Ink + for x := lo; x < hi; x += step { + end := x + step + 1 + if end > hi { + end = hi + } + ink = append(ink, Ink{Rect: CellRect{x, y0, end, y1}, Stroked: true}) + } + return ink +} + +// chainY is [chainX] down the page. +func chainY(lo, hi, x0, x1, step float64) []Ink { + var ink []Ink + for y := lo; y < hi; y += step { + end := y + step + 1 + if end > hi { + end = hi + } + ink = append(ink, Ink{Rect: CellRect{x0, y, x1, end}, Stroked: true}) + } + return ink +} + +// TestBoxOverlapOnADegenerateAxis pins the case an area ratio cannot answer: a +// single hairline clusters alone and its box has zero height. +func TestBoxOverlapOnADegenerateAxis(t *testing.T) { + for _, tc := range []struct { + name string + a, b CellRect + want float64 + }{ + {"a flat rule inside a box", CellRect{10, 50, 90, 50}, CellRect{0, 0, 100, 100}, 1}, + {"a flat rule half inside", CellRect{50, 50, 150, 50}, CellRect{0, 0, 100, 100}, 0.5}, + {"a flat rule above the box", CellRect{10, 150, 90, 150}, CellRect{0, 0, 100, 100}, 0}, + {"touching along an edge", CellRect{100, 0, 200, 100}, CellRect{0, 0, 100, 100}, 0}, + {"one box inside the other", CellRect{10, 10, 20, 20}, CellRect{0, 0, 100, 100}, 1}, + } { + if got := boxOverlap(tc.a, tc.b); got != tc.want { + t.Errorf("%s: overlap = %.3f, expected %.3f", tc.name, got, tc.want) + } + if got := boxOverlap(tc.b, tc.a); got != tc.want { + t.Errorf("%s, the other way round: overlap = %.3f, expected %.3f", + tc.name, got, tc.want) + } + } +} + +// The two fixtures, by what they are rather than by their filenames: which of them +// a number belongs to is the whole point of every assertion below. +const ( + columnsManual = "thomas-drybox-amfibia" + sequentialManual = "dreame-l40-ultra" +) + +// frontMatterPlates are the sequential manual's two diagram plates. They fall +// outside every language region, so no conversion ever serves them, and they are +// where every cost this pass has lands. +var frontMatterPlates = []int{5, 6} + +// grownGuards is [defaultGuards] with the growth pass switched back on, at the +// [maxLabelGrowth] it shipped at. +// +// IT IS NOT THE SHIPPED CONFIGURATION and every test that uses it is testing the +// alternative this package MEASURED AND REPLACED, not what a reader is served. The +// crop stopped growing when [Figure.Labels] started carrying each label as text, +// because a crop that also contained them would print every label twice. +// +// The tests are kept, and pointed here rather than deleted, for the reason +// TestAPlateMergeOnSharedLabelsIsRefused is kept: growing the crop is a good idea that +// the documents refuse, and the record of exactly how far it got is what stops it +// being proposed again. Note what would happen without this variable — a test whose +// assertion is "nothing grew" would go on passing against a pass that CANNOT grow, and +// pass for a reason that has nothing to do with what it is checking. +var grownGuards = func() figureGuards { + g := defaultGuards + g.growth = maxLabelGrowth + return g +}() + +// TestGrowSweep prints what each of the growth rule's four numbers does over both +// whole documents, one at a time with the rest at their defaults, in the same shape +// TestGuardSweep and TestMergeThresholdSweep use. It is the measurement +// [labelTerminator], [labelAlign] and [labelCorridor] are set from — three thresholds +// that are still live, because [figureLabels] claims with the same rule. +// +// THE GROWTH IT SWEEPS IS SWITCHED OFF IN THE SHIPPED CONFIGURATION. Everything below +// runs from [grownGuards], and what it measures is the pass that carrying a label as +// text replaced: see that variable for why the measurement is kept rather than +// deleted. The one number here that is NOT still live is [maxLabelGrowth]. +// +// What it asserts is only what is measured and stable. +// +// **The columns manual does not move at any setting.** 0 of its 59 figures grow, for +// every value of every one of the four. Both of its claims are blocked by prose in +// the corridor, so this pass is the other document's entirely and the manual whose +// pictures were counted by eye cannot lose one to it. That is the same shape of +// evidence [mergeOverlapping] rests on and it is the strongest safety property this +// change has. +// +// **Growth never changes the figure COUNT**, on either document: 59 and 195 with the +// pass on and off. It moves edges, and it runs after both guards, so a page cannot +// gain or lose a picture to it. +// +// **The shipped values give 55 grown figures and 229 labels taken in** on the +// sequential manual, 22 of the 55 on the plate pages. +// +// **The cap is a smooth continuum with no cliff**, so its shape is asserted rather +// than a gap: 18/51, 32/107, 55/229, 63/255 and 64/262 figures/labels at 0.25, 0.5, +// 1, 2 and no cap at all. +// +// **The overlapping crops are confined to the plates**: 11 pairs on pages 5 and 6, +// 0 on every other page of either document. Measured both ways — counting every +// overlapping pair of grown boxes and counting only the pairs whose drawings do not +// themselves overlap gives 11 either way, because no two of these figures' drawings +// overlap at all. +func TestGrowSweep(t *testing.T) { + for _, name := range []string{columnsManual, sequentialManual} { + t.Run(name, func(t *testing.T) { + pages, ink := loadFigureInk(t, name) + show := func(label string, g figureGuards) growStats { + s := growSweepStats(pages, ink, g) + t.Logf(" %-16s -> %3d figures, %2d grown, %3d labels, "+ + "overlapping pairs on %v", + label, s.figures, s.grown, s.labels, growPages(s.pairsOn)) + return s + } + base := show("growth at 1", grownGuards) + t.Logf(" grown per page: %v", base.grownOn) + t.Logf(" overlapping pairs per page: %v", base.pairsOn) + + var swept []growStats + for _, s := range []struct { + name string + vals []float64 + set func(*figureGuards, float64) + }{ + {"terminator", []float64{4, 6, 8, 12}, + func(g *figureGuards, v float64) { g.terminator = v }}, + {"align", []float64{2, 4, 6, 8}, + func(g *figureGuards, v float64) { g.align = v }}, + {"corridor", []float64{20, 40, 60, 80}, + func(g *figureGuards, v float64) { g.corridor = v }}, + {"growth", []float64{0, 0.25, 0.5, 1, 2, math.Inf(1)}, + func(g *figureGuards, v float64) { g.growth = v }}, + } { + for _, v := range s.vals { + // From grownGuards, not defaultGuards: the shipped configuration + // does not grow, so sweeping the other three thresholds from it + // would report 0 grown at every value and measure nothing. + g := grownGuards + s.set(&g, v) + swept = append(swept, show(fmt.Sprintf("%s=%g", s.name, v), g)) + } + } + + // The figure count is growth's invariant, at every setting of every one + // of the four: the pass runs after both guards and only moves edges. + for i := range swept { + if swept[i].figures != base.figures { + t.Errorf("a swept value gives %d figures against %d at the "+ + "defaults; growth may move an edge and never decide a picture", + swept[i].figures, base.figures) + } + } + + if name == columnsManual { + // The safety property. Not "few" and not "no regression": none, at + // every value of every threshold. + for i := range swept { + if swept[i].grown != 0 || swept[i].labels != 0 { + t.Errorf("this document grew %d figures and took %d labels at "+ + "some swept value; both of its claims are blocked by prose "+ + "in the corridor and it must not move at any setting", + swept[i].grown, swept[i].labels) + break + } + } + if base.figures != 59 { + t.Errorf("%d figures, expected 59", base.figures) + } + return + } + + if base.figures != 195 || base.grown != 55 || base.labels != 229 { + t.Errorf("%d figures, %d grown, %d labels; expected 195, 55 and 229", + base.figures, base.grown, base.labels) + } + if plates := growTotal(base.grownOn) - growTotal(base.grownOn, frontMatterPlates...); plates != 22 { + t.Errorf("%d grown figures on pages %v, expected 22 — the front-matter "+ + "plates carry most of what this pass does and none of what a "+ + "reader is served", plates, frontMatterPlates) + } + + // The cap, as a continuum rather than a gap. Each step gains figures and + // labels over the one below it, and no step is a cliff. + for _, tc := range []struct { + growth float64 + grown, labels int + }{ + {0.25, 18, 51}, {0.5, 32, 107}, {1, 55, 229}, {2, 63, 255}, + {math.Inf(1), 64, 262}, + } { + g := defaultGuards + g.growth = tc.growth + s := growSweepStats(pages, ink, g) + if s.grown != tc.grown || s.labels != tc.labels { + t.Errorf("growth=%g gives %d grown and %d labels, expected %d and %d", + tc.growth, s.grown, s.labels, tc.grown, tc.labels) + } + } + + // The cost, and the reason it is recorded rather than fixed: it is not on + // a page any conversion serves. Page 5 is 31 figures on one sheet with + // labels between them. + if off := growTotal(base.pairsOn, frontMatterPlates...); off != 0 { + t.Errorf("%d overlapping pairs of grown boxes off pages %v (%v); every "+ + "one of them used to be on a plate, and a reader is served none of "+ + "those pages", off, frontMatterPlates, base.pairsOn) + } + if all := growTotal(base.pairsOn); all != 11 { + t.Errorf("%d overlapping pairs in total, expected 11 on the plates", all) + } + }) + } +} + +// TestALabelReachesAReaderWhole is the number this whole change exists for, and it +// replaces the residual count that used to stand here. +// +// The old test counted labels a leader points at that the final CROP did not hold — +// 2 on the columns manual and 88 over 41 figures on the sequential one — and it is +// gone because its question no longer means anything. No crop grows now, so every +// label is outside every crop, and "labels outside the crop" would report all of them +// while a reader sees all of them too. The question that replaced it is the one the +// user actually asked: HOW MANY LABELS REACH A READER, WHOLE. +// +// The count runs the same direction as the old one and for a stronger reason. A +// carried label is never clipped — a text run has no rectangle — so this is exactly +// the number [figureLabels] carries, and a change that carries fewer has regressed. +// +// Converted pages are every page except the two front-matter plates, which are the +// only pages either document has that fall outside every language region. Those are +// counted separately rather than dropped, because they are served under the +// neutral-pages opt-in and their labels are the ONLY text they contribute. +func TestALabelReachesAReaderWhole(t *testing.T) { + for _, tc := range []struct { + name string + // labels carried off the plates, and on them. + served, plates int + // what the crop held whole before this change, off the plates. + wasWhole int + }{ + // ZERO, AND THAT IS THE SAFETY PROPERTY. All nine of this document's claims + // are false — page 1's cover title and eight lines of page 22's German body + // prose — and every one is refused by [figureLabels]'s clean-region gate. The + // manual whose pictures were counted by eye must not gain a label. + {columnsManual, 0, 0, 0}, + // 167 labels on pages a reader is served, plus 109 on the two plates. Against + // 186 held whole by the old grown crop DOCUMENT-WIDE, of which the plates held + // most: off the plates the crop held 99. So a served reader gains 68 labels, + // and the plates go from a crop that held them to text that carries them. + // + // It was 159 until a wrapped label stopped losing its tail. The 8 are the later + // lines of four labels whose earlier lines were already here — pages 521, 522 + // twice and 542 — plus the second line of page 546's bullet caution, which is one + // of the prose claims TestASentenceShapedClaimIsTheResidual counts and is now + // wrong WHOLE rather than wrong by halves. None of the 8 lands on a plate, which + // is why that column does not move. See [claimLabels]. + {sequentialManual, 167, 109, 99}, + } { + t.Run(tc.name, func(t *testing.T) { + pages, ink := loadFigureInk(t, tc.name) + + carried := map[int]int{} + wasWhole := map[int]int{} + for i := range pages { + p := &pages[i] + var dropped DroppedRuns + text := usableRuns(p.Runs, p.Width, p.Height, &dropped) + drawn := onPageInk(ink[i], p.Width, p.Height) + marks := marksOf(drawn) + for _, f := range findFigures(ink[i], p, defaultGuards) { + carried[p.No] += len(f.Labels) + // Every carried label must be INSIDE the crop, which is the exact + // INVERSE of what this asserted before. The crop used to be the + // drawing alone and a label was drawn beside it, so a label inside + // the crop would have printed twice; the crop is now the band the + // page laid the drawing and its labels out in, so a label OUTSIDE it + // is one the alt text says is in a picture that does not print it. + // See [labelBand]. + for _, r := range claimedRuns(f.InkRect, text, marks) { + if boxOverlap(runBox(r), f.Rect) < 1 { + t.Errorf("page %d: the label %q is not inside the crop %v "+ + "that is supposed to print it", p.No, + strings.TrimSpace(r.Text), f.Rect) + } + } + } + // What the replaced pass held whole, for the same figures. The grown box + // is computed here rather than read off Figure.Rect, which is now the + // band and does not depend on grownGuards at all. + for _, f := range findFigures(ink[i], p, grownGuards) { + grown := growToLabels(f.InkRect, text, drawn, grownGuards) + for _, r := range claimSetOf(f.InkRect, text, marks, grownGuards) { + if boxOverlap(runBox(r), grown) >= 1 { + wasWhole[p.No]++ + } + } + } + } + + served := growTotal(carried, frontMatterPlates...) + plates := growTotal(carried) - served + was := growTotal(wasWhole, frontMatterPlates...) + t.Logf("%s: %d labels reach a reader whole on served pages (%d more on the "+ + "plates), against %d the grown crop held whole", tc.name, served, plates, was) + t.Logf(" carried per page: %v", carried) + + if served != tc.served || plates != tc.plates { + t.Errorf("%d labels served and %d on the plates, expected %d and %d; "+ + "this number can only go up, so a change that carries fewer has "+ + "regressed", served, plates, tc.served, tc.plates) + } + if was != tc.wasWhole { + t.Errorf("the replaced pass held %d whole, expected %d", was, tc.wasWhole) + } + // The claim this change is measured against: it must beat the crop. + if served < was { + t.Errorf("carrying labels as text serves %d and the crop served %d; "+ + "the replacement must not be worse than what it replaced", served, was) + } + }) + } +} + +// TestASentenceShapedClaimIsTheResidual counts what this pass gets WRONG, so the +// number is visible rather than discovered. +// +// A callout label should be a word or a short noun phrase. A few claims are neither: +// they are lines of body prose that happen to sit in a figure's corridor with a small +// shape on their midline, and whose whole side passed [labelExtent] because the rest of +// that corridor was claims too. Page 529 is the clearest — `6. Для возврата робота на +// базовую станцию используйте приложение` and its continuation `или один раз нажмите +// кнопку на роботе.` are step 6 of a numbered list, and they are now drawn beside the +// picture and taken out of the prose. Pages 531, 546, 550, 551 and 553 have the same +// shape, four of them in Japanese. +// +// # THERE IS NO THRESHOLD UNDER THIS AND ONE WAS MEASURED FOR +// +// Sorted by length, the claims run +// ... 27, 32, 34, 35, 37, **38, 39**, 39, 40, 42, 42, 48, 64 runes. +// 38 is `Вспомогательная светодиодная подсветка`, a real label on page 521. 39 is a +// Japanese sentence. ONE RUNE apart, and everything above 38 is prose while everything +// below it is a label — which is a coincidence and not a gap, and cutting there would +// be exactly the fitted threshold [minFigureWidth] refuses on the record. The other +// candidate signal is worse: only 2 of the 268 end in a full stop, because Japanese +// sentences here end in a wrapped clause and a numbered step ends mid-phrase. +// +// So nothing is guarded and the cost is stated instead. It is bounded, and it is a +// misplacement rather than a loss: every one of these is still shown to the reader and +// still counted by `verify.checkCoverage` — it is beside the picture instead of in the +// list it belongs to. The columns manual has NONE, because it carries no labels at all. +// +// The number can only go down. +func TestASentenceShapedClaimIsTheResidual(t *testing.T) { + for _, tc := range []struct { + name string + wrong int + pages []int + }{ + {columnsManual, 0, nil}, + // Page 551 has one of the same shape at 35 runes — `3. 新しい紙パックを取り付けます。…` + // — which this bound does not reach, and that is the bound being a description + // rather than a rule. It is the same defect and it is not counted here. + {sequentialManual, 7, []int{529, 531, 546, 550, 553}}, + } { + t.Run(tc.name, func(t *testing.T) { + pages, ink := loadFigureInk(t, tc.name) + // A DESCRIPTION FOR COUNTING, NOT A GUARD. Nothing in figures.go consults + // this; see the note above for why it must not. + sentenceShaped := func(s string) bool { + return len([]rune(s)) >= 39 || strings.HasSuffix(strings.TrimSpace(s), ".") + } + var n int + on := map[int]int{} + var longest int + for i := range pages { + p := &pages[i] + for _, f := range findFigures(ink[i], p, defaultGuards) { + for _, l := range f.Labels { + if sentenceShaped(l) { + n++ + on[p.No]++ + t.Logf(" p%d: %q", p.No, l) + continue + } + longest = max(longest, len([]rune(l))) + } + } + } + t.Logf("%s: %d sentence-shaped claims on pages %v; the longest real label is "+ + "%d runes", tc.name, n, growPages(on), longest) + if n != tc.wrong { + t.Errorf("%d sentence-shaped claims, expected %d; this number can only "+ + "go down", n, tc.wrong) + } + if got := growPages(on); tc.pages != nil && !slices.Equal(got, tc.pages) { + t.Errorf("on pages %v, expected %v", got, tc.pages) + } + // The one-rune non-gap, asserted so that a change which opens a REAL gap is + // noticed and can be turned into a guard. + if tc.name == sequentialManual && longest != 38 { + t.Errorf("the longest real label is %d runes, expected 38; the reason "+ + "there is no guard here is that 38 and 39 are one rune apart, so a "+ + "move in this number may mean a threshold has become defensible", + longest) + } + }) + } +} + +// TestNoLabelIsCarriedWithoutItsLaterLines is the invariant behind the second fault +// the user reported off page 521, and it is an invariant rather than a count because +// what went wrong there has no total that describes it. +// +// A LABEL IS A UNIT. `Монтажные отверстия для держателя насадки для швабры` was drawn +// on the underside diagram as its first two lines, and `держателя`, `насадки для` and +// `швабры` were left in the prose as three floating paragraphs — worse than either +// answer, because the reader saw half a label on the picture and the other half adrift +// in the text. The cause was the band: [runBeyond] only sees a run level with the +// drawing, and a label set against the drawing's foot wraps BELOW its box. +// +// So this asks the question directly, of every carried side of every figure of both +// documents: is there any run left in the flow that [continuesLabel] says is the next +// line of something carried? Two bounds could still cut a chain, and the answer is +// measured rather than assumed: +// +// refused by labelCorridor 0 on either document +// refused by minWrapRunes 0 on the columns manual, 2 on the sequential one +// +// The 2 are the single digits "4" on plate page 5 and "2" on page 6, which are their +// own numbered callouts and not any label's second line — [minWrapRunes] exists for +// exactly that case and names those pages. They are asserted by name, so a chain that +// really is cut short cannot hide inside the allowance. +// +// It is deliberately NOT a count of carried labels: TestALabelReachesAReaderWhole owns +// that number, and a test that asserted both would move for two reasons at once. +func TestNoLabelIsCarriedWithoutItsLaterLines(t *testing.T) { + for _, tc := range []struct { + name string + // the runs a wrap floor keeps out of a carried chain, as page -> text. + floor map[int]string + }{ + {columnsManual, map[int]string{}}, + {sequentialManual, map[int]string{5: "4", 6: "2"}}, + } { + t.Run(tc.name, func(t *testing.T) { + pages, ink := loadFigureInk(t, tc.name) + floor := map[int]string{} + for i := range pages { + p := &pages[i] + var dropped DroppedRuns + text := usableRuns(p.Runs, p.Width, p.Height, &dropped) + drawn := onPageInk(ink[i], p.Width, p.Height) + marks := marksOf(drawn) + for _, f := range findFigures(ink[i], p, defaultGuards) { + for side := range 4 { + claimed := claimLabels(f.InkRect, text, marks, side, defaultGuards) + if len(claimed) == 0 { + continue + } + // A refused side carries nothing, so a truncated chain on it is + // not a label the reader sees half of. + if _, ok := labelExtent(f.InkRect, text, claimed, side); !ok { + continue + } + for k := range text { + r := &text[k] + if claims(claimed, r) || strings.TrimSpace(r.Text) == "" { + continue + } + if !runInCorridor(f.InkRect, r, side, math.Inf(1)) || + !continuesLabel(claimed, r, side, text, f.InkRect) { + continue + } + switch { + case !runInCorridor(f.InkRect, r, side, defaultGuards.corridor): + t.Errorf("page %d figure %d: %q is the next line of a "+ + "carried label and sits past labelCorridor, so the "+ + "label is drawn on the picture without it", + p.No, f.Index, strings.TrimSpace(r.Text)) + case len([]rune(strings.TrimSpace(r.Text))) < minWrapRunes: + floor[p.No] = strings.TrimSpace(r.Text) + default: + t.Errorf("page %d figure %d: %q continues a carried label "+ + "and is not claimed, and no bound explains it — the "+ + "fixpoint in claimLabels should have reached it", + p.No, f.Index, strings.TrimSpace(r.Text)) + } + } + } + } + } + if !maps.Equal(floor, tc.floor) { + t.Errorf("the wrap floor keeps %v out of a carried chain, expected %v; "+ + "each of these must be a callout of its own rather than a label's "+ + "second line", floor, tc.floor) + } + }) + } +} + +// claimSetOf is every run claimed for a figure over all four sides, deduped. The +// unfiltered claim, before [figureLabels]'s gate — used to measure what the replaced +// growth pass reached. +func claimSetOf(area CellRect, text []TextRun, marks []CellRect, g figureGuards) []*TextRun { + var out []*TextRun + for side := range 4 { + for _, r := range claimLabels(area, text, marks, side, g) { + if !claims(out, r) { + out = append(out, r) + } + } + } + return out +} + +// TestAPlateMergeOnSharedLabelsIsRefused is the measurement behind a rule this +// package deliberately does NOT have, and it is kept as a test because the idea is a +// good one that the documents refuse. +// +// The idea: [growToLabels] cannot widen a crop onto its longest label when a +// neighbouring drawing's labels sit in the way — page 521's lidar drawing reaches +// x=397 where its own label ends at 469, because the lid-open drawing's labels start +// at 400 — so if two drawings' label claims interleave, read the page as having laid +// them out as ONE plate with a shared label field, and serve them as one picture. +// One merge criterion here instead of carrying a label as text through the schema, +// the API and the reader. +// +// It is refused on three measurements, and the crops were rendered and looked at +// rather than scored. +// +// # The criterion has no threshold to sit on +// +// Give every figure a want box — its drawn extent plus every run [claimLabels] +// claims for it, which is the crop it would have if nothing were in the way — and +// merge the figures whose want boxes overlap. Over both documents that is 35 +// overlapping pairs, all on the sequential manual, and sorted by how much of the +// smaller want box lies inside the larger there is no gap and no ordering that +// helps: +// +// 1.000 page 546 figs 0+3 two drawings 99 units apart, merged text 0.189 +// 0.965 page 553 figs 7+9 three drawings and a whole three-line note, 0.256 +// 0.148 page 521 figs 0+1 THE case this idea is for +// 0.063 page 522 figs 7+8 two columns' drawings, 328 units apart, 0.316 +// 0.001 page 521 figs 1+2 a 14x5-unit corner touch +// +// The gap between the drawn boxes orders the two page-521 and page-522 cases the +// wrong way round to be usable as the bound: the pair that should merge is 276 units +// apart and the pair that must not is 328, and any bound between them also admits +// page 546's 99, page 552's 131 and page 553's 174 and 223. +// +// # It cascades on the page it was invented for, and not on the page expected +// +// Page 521 is three drawings. Merging the pair the idea is about does produce one +// crop holding both drawings and all 20 of their labels uncut — rendered, and a real +// improvement. But figure 1's want box also clips figure 2's by 14x5 units, so the +// transitive closure takes all three: a crop 0.629 of the page holding two columns +// of button-description prose, cut mid-line. +// +// Page 522 was the expected disaster and is not one: seven printed pictures come back +// as 9 figures, 6 of which DO grow — the brief that proposed this had it that growth +// declines that page — and only one pair merges, so nothing tiles. That pair is the +// worse failure. It joins the base station's front view in the left column to its +// cutaway in the right, 328 units apart, and the crop carries a fragment of the +// page's `Примечание.` line and a fragment of a bullet list while STILL clipping +// labels at its right edge. A reader is served that instead of two crops that each +// lose a word. +// +// # The version with real evidence under it helps two plate pages and nothing else +// +// "A shared label field" has a literal reading that is a much stronger signal than +// interleaving: a run claimed by BOTH figures. It is 0 on page 521's pair and 0 on +// page 522's. At two shared runs or more it fires on 4 groups, every one of them on +// PDF pages 5 and 6 — the front-matter plates, which are converted only under the +// neutral-pages opt-in — and it takes 18 more labels in. That is the honest size of +// the safe rule: 18 labels on two plates, 0 on any page the reported defect is on, +// and one crop over the text guard to pay for them. +// +// The floor of two is not pinned by these documents and mutation testing said so: 2 +// and 3 give the identical answer, because the five pairs that qualify share 3, 3, 5, +// 6 and 8 runs. 1 admits three content-page pairs and 4 drops two plate pairs, so the +// plateau is 2..3 and the number is recorded rather than chosen. +// +// So the complete answer stays the one docs/design/conversion.md already names: +// carry a label as text. +func TestAPlateMergeOnSharedLabelsIsRefused(t *testing.T) { + for _, tc := range []struct { + name string + // today, interleaving want boxes, and two or more shared claimed runs. + today, interleave, shared plateStats + }{ + // The columns manual does not move under any of the three, which is the same + // safety property [mergeOverlapping] and [growToLabels] rest on: it has no + // page where two want boxes overlap at all. Its 9 claims are the runs behind + // its two FALSE claims, and no crop holds one whole either way. + {columnsManual, + plateStats{crops: 59, claimed: 9, whole: 0}, + plateStats{crops: 59, claimed: 9, whole: 0}, + plateStats{crops: 59, claimed: 9, whole: 0}}, + // The sequential manual is where all of it happens. Interleaving takes 76 more + // labels in and costs 9 crops over [maxFigureTextFraction] and one crop 0.680 + // of its page; shared runs take 18 and cost ONE over the text guard. + // + // THESE NUMBERS WERE STALE BEFORE THE BAND, and the staleness is worth knowing + // rather than quietly corrected. The wrapped-label tail fix moved the claim + // count 319 -> 327 and page 521's 40 -> 43, and this test was not re-run: it + // was already failing at 3532d4b with exactly the numbers below. Fixture tests + // need MANUALBOX_TEST_FIXTURES=1 and so do not run in the default suite, which + // is how a red test shipped. The band does not move any of them — it takes no + // part in the claim rule — and the refusal is unchanged and slightly stronger: + // the shared-run rule now also produces one crop over the text guard. + {sequentialManual, + plateStats{crops: 195, claimed: 327, whole: 186}, + plateStats{crops: 162, groups: 17, claimed: 327, whole: 262, overText: 9}, + plateStats{crops: 189, groups: 5, claimed: 327, whole: 204, overText: 1}}, + } { + t.Run(tc.name, func(t *testing.T) { + pages, ink := loadFigureInk(t, tc.name) + for _, r := range []struct { + label string + want plateStats + got plateStats + }{ + {"today", tc.today, plateMergeStats(pages, ink, plateNoMerge)}, + {"interleaving want boxes", tc.interleave, plateMergeStats(pages, ink, plateInterleave)}, + {"two shared claimed runs", tc.shared, plateMergeStats(pages, ink, plateShared)}, + } { + t.Logf("%-24s %d crops, %d merged groups, %d of %d claimed labels held "+ + "whole, largest merged crop %.3f of its page, %d over the text guard, "+ + "page 521 %d of %d", + r.label, r.got.crops, r.got.groups, r.got.whole, r.got.claimed, + r.got.largest, r.got.overText, r.got.whole521, r.got.claimed521) + if r.got.crops != r.want.crops || r.got.groups != r.want.groups || + r.got.claimed != r.want.claimed || r.got.whole != r.want.whole || + r.got.overText != r.want.overText { + t.Errorf("%s: %+v, expected crops=%d groups=%d claimed=%d whole=%d "+ + "overText=%d", r.label, r.got, r.want.crops, r.want.groups, + r.want.claimed, r.want.whole, r.want.overText) + } + } + + if tc.name != sequentialManual { + return + } + // The two numbers the refusal turns on, asserted as bounds because they are + // what a reader would be served: interleaving reaches most of a page, and + // the version with evidence under it does not reach a sixteenth of one. + if got := plateMergeStats(pages, ink, plateInterleave); got.largest < 0.6 { + t.Errorf("the largest interleaved crop is %.3f of its page; it used to be "+ + "0.629, a crop holding two columns of prose", got.largest) + } + // The safe version was recorded as reaching under a sixteenth of a page. + // It reaches 0.118 now, on the same claim-count move that made the numbers + // above stale, so the bound is stated where it actually is — and with it the + // last thing that made the shared-run rule look free is gone: it costs one + // crop over the text guard. + if got := plateMergeStats(pages, ink, plateShared); got.largest > 0.12 { + t.Errorf("the largest shared-label crop is %.3f of its page, expected "+ + "under 0.12", got.largest) + } + // Page 521 itself, which is the whole point of the idea and the number that + // says the safe version is not worth its risk: 23 of its 43 claims arrive + // whole under the pass this measures, all 43 do if the three drawings are + // merged into most of the page, and the shared-run rule leaves it exactly + // where it was. 43 was 40 before the wrapped-label tail was claimed. + for _, r := range []struct { + label string + rule plateRule + whole int + }{ + {"today", plateNoMerge, 23}, + {"interleaving want boxes", plateInterleave, 43}, + {"two shared claimed runs", plateShared, 23}, + } { + got := plateMergeStats(pages, ink, r.rule) + if got.claimed521 != 43 || got.whole521 != r.whole { + t.Errorf("page 521 holds %d of %d claims whole under %s, expected %d of 43", + got.whole521, got.claimed521, r.label, r.whole) + } + } + }) + } +} + +// plateStats is what one merge rule does to a whole document. crops counts what a +// reader would be served: one per merged group, one per unmerged figure. +type plateStats struct { + crops, groups int + claimed, whole int + overText int + largest float64 + whole521, claimed521 int +} + +// The three rules TestAPlateMergeOnSharedLabelsIsRefused compares. Each answers +// whether two figures are one plate, given their want boxes and their claims. +type plateRule func(a, b CellRect, ca, cb []*TextRun) bool + +func plateNoMerge(CellRect, CellRect, []*TextRun, []*TextRun) bool { return false } + +func plateInterleave(a, b CellRect, _, _ []*TextRun) bool { return boxOverlap(a, b) > 0 } + +func plateShared(_, _ CellRect, ca, cb []*TextRun) bool { + var n int + for _, r := range ca { + if claims(cb, r) { + n++ + } + } + return n >= 2 +} + +// plateMergeStats runs one merge rule over every page of a document. +// +// A merged crop is the union of its members' WANT boxes rather than a re-grown union +// of their drawings, which is the variant most favourable to the idea: re-growing +// re-applies [growToLabels]'s conservative half to the merged box, and on page 521 +// that loses the lid-open drawing's right-hand labels entirely because the merged +// box's right corridor holds a bullet description. Judging the idea on its best +// variant is the point. +func plateMergeStats(pages []PageRuns, ink [][]Ink, rule plateRule) plateStats { + // grownGuards, NOT the shipped configuration, and this is what keeps the comparison + // meaningful. The question this test answers is "would merging plates have been a + // better answer than GROWING THE CROP", which was the state of the world when it was + // asked. Run from the shipped guards no crop grows, so the "today" column would + // report 0 labels held whole and the three rules would be compared against a + // baseline that does not exist. The real answer to the question both columns lost to + // is [figureLabels]; see TestALabelReachesAReaderWhole for that number. + g := grownGuards + var s plateStats + for i := range pages { + p := &pages[i] + var dropped DroppedRuns + text := usableRuns(p.Runs, p.Width, p.Height, &dropped) + drawn := onPageInk(ink[i], p.Width, p.Height) + marks := marksOf(drawn) + figs := findFigures(ink[i], p, g) + + wants := make([]CellRect, len(figs)) + claimed := make([][]*TextRun, len(figs)) + for j := range figs { + wants[j] = figs[j].InkRect + for side := range 4 { + for _, r := range claimLabels(figs[j].InkRect, text, marks, side, g) { + wants[j] = plateUnion(wants[j], runBox(r)) + claimed[j] = append(claimed[j], r) + } + } + } + + // Connected components of the rule, which is where a cascade shows up: the + // relation is fixed before any merge, so this is its transitive closure. + parent := make([]int, len(figs)) + for j := range parent { + parent[j] = j + } + find := func(x int) int { + for parent[x] != x { + parent[x] = parent[parent[x]] + x = parent[x] + } + return x + } + for a := range figs { + for b := a + 1; b < len(figs); b++ { + if !rule(wants[a], wants[b], claimed[a], claimed[b]) { + continue + } + if ra, rb := find(a), find(b); ra != rb { + parent[ra] = rb + } + } + } + groups := map[int][]int{} + for j := range figs { + groups[find(j)] = append(groups[find(j)], j) + } + + for _, members := range groups { + // The GROWN box, computed rather than read off Figure.Rect: the crop is + // [labelBand] now and takes no notice of g.growth, so reading it would + // compare the merge against the wrong baseline entirely. + crop := growToLabels(figs[members[0]].InkRect, text, drawn, g) + if len(members) > 1 { + crop = wants[members[0]] + for _, m := range members[1:] { + crop = plateUnion(crop, wants[m]) + } + s.groups++ + s.largest = math.Max(s.largest, + crop.Width()*crop.Height()/(p.Width*p.Height)) + if textFraction(crop, text) > g.maxText { + s.overText++ + } + } + s.crops++ + for _, m := range members { + for _, r := range claimed[m] { + s.claimed++ + whole := boxOverlap(runBox(r), crop) >= 1 + if whole { + s.whole++ + } + if p.No == 521 { + s.claimed521++ + if whole { + s.whole521++ + } + } + } + } + } + } + return s +} + +func plateUnion(a, b CellRect) CellRect { + return CellRect{math.Min(a.X0, b.X0), math.Min(a.Y0, b.Y0), + math.Max(a.X1, b.X1), math.Max(a.Y1, b.Y1)} +} + +// growStats is what one setting of the growth rule does to a whole document. +type growStats struct { + figures int + grown int + // labels is how many runs the crops took in: a run that intersects a figure's + // grown box and does not touch the drawing itself. A label the edge cut short + // counts, because it WAS taken in — see [growToLabels] on why a claimed label may + // be cut and prose may not. + labels int + // grownOn is how many grown figures each page carries. + grownOn map[int]int + // pairsOn is how many pairs of grown boxes overlap on each page. Counted over + // pairs whose drawings do not themselves overlap, so the number is growth's own + // doing; on these two documents no two drawings overlap, so it is also simply + // every overlapping pair. + pairsOn map[int]int + // residualOn and residualLabels are the figures each page still has with a label + // outside the final crop, and how many such labels. + residualOn map[int]int + residualLabels map[int]int +} + +// growSweepStats runs one setting over every page of a document. +// IT CALLS growToLabels ITSELF, and it has to. The shipped crop is [labelBand], which +// does not read g.growth at all, so reading Figure.Rect here would report the same +// numbers at every setting and the sweep would measure nothing. The grown box is +// computed beside the figure instead, which is what makes this a record of the +// replaced pass rather than a broken test of the current one. +func growSweepStats(pages []PageRuns, ink [][]Ink, g figureGuards) growStats { + s := growStats{grownOn: map[int]int{}, pairsOn: map[int]int{}, + residualOn: map[int]int{}, residualLabels: map[int]int{}} + for i := range pages { + p := &pages[i] + var dropped DroppedRuns + text := usableRuns(p.Runs, p.Width, p.Height, &dropped) + drawn := onPageInk(ink[i], p.Width, p.Height) + marks := marksOf(drawn) + figs := findFigures(ink[i], p, g) + s.figures += len(figs) + grown := make([]CellRect, len(figs)) + for j := range figs { + f := &figs[j] + grown[j] = growToLabels(f.InkRect, text, drawn, g) + if n := clippedLabels(f, grown[j], text, marks, g); n > 0 { + s.residualOn[p.No]++ + s.residualLabels[p.No] += n + } + if grown[j] == f.InkRect { + continue + } + s.grown++ + s.grownOn[p.No]++ + for k := range text { + box := runBox(&text[k]) + if boxOverlap(box, f.InkRect) == 0 && boxOverlap(box, grown[j]) > 0 { + s.labels++ + } + } + } + for a := range figs { + for b := a + 1; b < len(figs); b++ { + if boxOverlap(grown[a], grown[b]) > 0 && + boxOverlap(figs[a].InkRect, figs[b].InkRect) == 0 { + s.pairsOn[p.No]++ + } + } + } + } + return s +} + +// clippedLabels is how many of a figure's labels a leader points at and the final +// crop still does not hold whole. Terminator claims only: a continuation line +// without its first line is not a label this can report on. +func clippedLabels(f *Figure, crop CellRect, text []TextRun, marks []CellRect, g figureGuards) int { + var n int + for side := range 4 { + for i := range text { + r := &text[i] + gap, outside := runBeyond(f.InkRect, r, side) + if !outside || gap > g.corridor { + continue + } + if terminatorAt(marks, f.InkRect, r, side, g) && boxOverlap(runBox(r), crop) < 1 { + n++ + } + } + } + return n +} + +// growPages is the pages a map of per-page counts covers, in order. +func growPages(m map[int]int) []int { + out := make([]int, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Ints(out) + return out +} + +// growTotal sums a map of per-page counts, optionally excluding some pages. +func growTotal(m map[int]int, except ...int) int { + var n int + for k, v := range m { + if !slices.Contains(except, k) { + n += v + } + } + return n +} + +// growthDrawing lays a drawing's border as chained strokes so the cluster's +// bounding box is exactly r, with no stroke small enough to be read as a leader's +// terminator: every rect is 10 units thick, and a mark is admitted only when BOTH +// sides are [labelTerminator] or under. That matters more than it looks. Built with +// 6-unit strokes the border's own 8x6 pieces are terminator candidates, and one of +// them lands on any label's midline — which would make TestAPartsListIsRefused pass +// for the wrong reason. +func growthDrawing(r CellRect) []Ink { + ink := chainX(r.X0, r.X1, r.Y0, r.Y0+10, 10) + ink = append(ink, chainX(r.X0, r.X1, r.Y1-10, r.Y1, 10)...) + ink = append(ink, chainY(r.Y0, r.Y1, r.X0, r.X0+10, 10)...) + ink = append(ink, chainY(r.Y0, r.Y1, r.X1-10, r.X1, 10)...) + return ink +} + +// leaderMark is a leader's end mark at its measured size: the open circles on page +// 521 of the sequential manual are 3.3 to 3.4 units square. +func leaderMark(cx, cy float64) Ink { + return Ink{Rect: CellRect{cx - 1.7, cy - 1.7, cx + 1.7, cy + 1.7}} +} + +// runMidY is a run's midline, which is what a leader points at. +func runMidY(r TextRun) float64 { return r.Y + r.Height/2 } + +// marksOf picks the terminator candidates out of a drawing the same way +// [growToLabels] does, for the tests that call [claimLabels] directly. +func marksOf(drawn []Ink) []CellRect { + var marks []CellRect + for i := range drawn { + if r := drawn[i].Rect; r.Width() <= labelTerminator && r.Height() <= labelTerminator { + marks = append(marks, r) + } + } + return marks +} + +// TestALabelALeaderPointsAtIsCarriedAsText drives the whole pass through FindFigures +// at page 521's measured geometry: the drawn box's right edge is at 263.0, the +// terminator sits at 259.6-263.0 and SETS that edge, and the label starts at 266.0. +// +// What the label reaches a reader by is the assertion. THE CROP DOES NOT MOVE — it is +// the drawing, and only the drawing — and the label arrives on [Figure.Labels] with a +// position relative to that crop. The position is greater than 1 on x, which is not a +// defect but the entire point: the label is outside the picture, which is why widening +// the picture was the only way to reach it before. +func TestALabelALeaderPointsAtIsCarriedAsText(t *testing.T) { + const drawn = 263.0 + crop := CellRect{100, 100, drawn, 300} + label := TextRun{X: 266, Y: 200, Width: 10, Height: 13, Text: "12"} + page := &PageRuns{No: 521, Width: 918, Height: 631, Runs: []TextRun{label}} + + ink := growthDrawing(crop) + ink = append(ink, leaderMark(261.3, runMidY(label))) + + figs := FindFigures(ink, page) + if len(figs) != 1 { + t.Fatalf("found %d figures, expected 1", len(figs)) + } + f := &figs[0] + // THE CROP IS THE BAND, so it holds the drawing AND the label the leader points + // at. That is the inverse of what this asserted while the reader placed labels + // itself, and the label's own box is what the right edge now reaches. + band := CellRect{crop.X0, crop.Y0, label.right(), crop.Y1} + if f.Rect != band { + t.Errorf("Rect = %v, expected the drawing and its label %v", f.Rect, band) + } + if f.InkRect != crop { + t.Errorf("InkRect = %v, expected the drawing alone %v; the language question is "+ + "asked of the drawing and must not move with the crop", f.InkRect, crop) + } + if len(f.Labels) != 1 { + t.Fatalf("carried %d labels, expected the one the leader points at", len(f.Labels)) + } + if f.Labels[0] != "12" { + t.Errorf("label = %q, expected %q", f.Labels[0], "12") + } + // The label is inside the crop, which is the invariant that replaced its position: + // what the alt text names, the picture prints. + if o := boxOverlap(runBox(&label), f.Rect); o < 1 { + t.Errorf("the label lies %.3f inside the crop, expected all of it", o) + } +} + +// claimedRuns is every run [figureLabels] keeps for a figure, as runs rather than as +// strings, so a test can ask where they were printed. It applies the same gate, so it +// is the same set in the same order. +func claimedRuns(area CellRect, text []TextRun, marks []CellRect) []*TextRun { + var kept []*TextRun + for side := range 4 { + claimed := claimLabels(area, text, marks, side, defaultGuards) + if len(claimed) == 0 { + continue + } + if _, ok := labelExtent(area, text, claimed, side); !ok { + continue + } + for _, r := range claimed { + if !claims(kept, r) { + kept = append(kept, r) + } + } + } + return kept +} + +// TestLabelOrderIsStableWhateverOrderTheRunsArriveIn pins the sort, which is not +// cosmetic: the label's position in this slice IS the index it is stored under, so an +// order that depends on the order the page's runs happen to be listed in would make a +// re-conversion rewrite the same labels onto different rows. +// +// The runs below are given to the page in DESCENDING y, which is the order +// claimLabels would return them in — it walks the text slice — and the opposite of the +// order they must come out in. A left-side label is included because SIDE IS NO LONGER +// A KEY: it used to sort first, so `left` came out ahead of all three; the order is now +// the page's own, so it falls between `mid` and `low` where its baseline puts it. That +// is the order alt text should read in, which is what the labels are now for. +// +// This is what a mutation test found nothing else asserting: deleting the sort left +// every other test in this package passing. +func TestLabelOrderIsStableWhateverOrderTheRunsArriveIn(t *testing.T) { + area := CellRect{100, 100, 300, 400} + // Right side, listed bottom to top. + low := TextRun{X: 303, Y: 330, Width: 30, Height: 13, Text: "low"} + mid := TextRun{X: 303, Y: 230, Width: 30, Height: 13, Text: "mid"} + high := TextRun{X: 303, Y: 130, Width: 30, Height: 13, Text: "high"} + // Left side, and its baseline is what places it: between mid and low. + left := TextRun{X: 60, Y: 280, Width: 30, Height: 13, Text: "left"} + + runs := []TextRun{low, mid, high, left} + ink := growthDrawing(area) + for _, r := range runs { + ink = append(ink, leaderMark(map[bool]float64{true: 96, false: 296}[r.X < area.X0], runMidY(r))) + } + page := &PageRuns{No: 521, Width: 918, Height: 631, Runs: runs} + + figs := FindFigures(ink, page) + if len(figs) != 1 { + t.Fatalf("found %d figures, expected 1", len(figs)) + } + got := figs[0].Labels + want := []string{"high", "mid", "left", "low"} + if !slices.Equal(got, want) { + t.Errorf("labels came out %v, expected %v — down, then across", got, want) + } + + // And the answer does not depend on the order the runs were listed in. Reversing + // the slice reverses what claimLabels returns and must not reach the result. + slices.Reverse(runs) + again := FindFigures(ink, &PageRuns{No: 521, Width: 918, Height: 631, Runs: runs}) + regot := again[0].Labels + if !slices.Equal(regot, got) { + t.Errorf("listing the same runs in the other order gave %v against %v; the "+ + "stored index would move with it", regot, got) + } +} + +// TestWithoutATerminatorNoLabelIsCarried is the same geometry with the mark taken out, +// and it is the whole signal: a run three units from the edge is not a label unless +// something points at it. +func TestWithoutATerminatorNoLabelIsCarried(t *testing.T) { + const drawn = 263.0 + label := TextRun{X: 266, Y: 200, Width: 10, Height: 13, Text: "12"} + page := &PageRuns{No: 521, Width: 918, Height: 631, Runs: []TextRun{label}} + + figs := FindFigures(growthDrawing(CellRect{100, 100, drawn, 300}), page) + if len(figs) != 1 { + t.Fatalf("found %d figures, expected 1", len(figs)) + } + if len(figs[0].Labels) != 0 { + t.Errorf("carried %d labels with no mark on the page; nothing points at that run", + len(figs[0].Labels)) + } + if figs[0].Rect != figs[0].InkRect { + t.Errorf("Rect = %v against InkRect = %v; the crop never moves now", + figs[0].Rect, figs[0].InkRect) + } +} + +// TestAPartsListIsRefused is the case that rules a distance rule out, and it is a +// document rather than an argument. +// +// Page 11 of the columns manual prints its parts list — 39 numbers and 39 German +// names — in a column 22.3 units to the right of the exploded view, with no +// terminator anywhere: a legend is not pointed at. 22.3 sits INSIDE the range page +// 521's underside diagram holds its own labels at, 20.3 to 35.3 units out — not +// further away than them, which is the stronger fact: no "grow onto text within N +// units" rule can take one and refuse the other, at any N. +func TestAPartsListIsRefused(t *testing.T) { + area := CellRect{100, 100, 300, 400} + const listX = 322.3 // 22.3 units right of the drawing's edge + text := []TextRun{ + {X: listX, Y: 110, Width: 6, Height: 13, Text: "1"}, + {X: listX + 12, Y: 110, Width: 80, Height: 13, Text: "Gehäusedeckel"}, + {X: listX, Y: 128, Width: 6, Height: 13, Text: "2"}, + {X: listX + 12, Y: 128, Width: 62, Height: 13, Text: "Tragegriff"}, + {X: listX, Y: 146, Width: 6, Height: 13, Text: "3"}, + {X: listX + 12, Y: 146, Width: 70, Height: 13, Text: "Saugschlauch"}, + } + got := growToLabels(area, text, growthDrawing(area), grownGuards) + if got != area { + t.Errorf("grew to %v; a parts list is not pointed at and nothing may move", got) + } +} + +// TestProseInTheCorridorStopsTheEdgeDead is the conservative half of the rule, and +// it is a decision rather than a detail: an edge moves only if everything the growth +// region touches is a claimed label. +// +// Page 521's lid-open drawing is the case. Its corridor holds "Кнопка сброса" and +// then the five bullet lines that explain it, so growing right would drag a +// paragraph into a picture; its left edge grows and its right does not. +func TestProseInTheCorridorStopsTheEdgeDead(t *testing.T) { + area := CellRect{100, 100, 300, 300} + label := TextRun{X: 303, Y: 190, Width: 12, Height: 13, Text: "12"} + prose := TextRun{X: 305, Y: 140, Width: 60, Height: 13, Text: "a whole line of prose"} + drawn := append(growthDrawing(area), leaderMark(296, runMidY(label))) + + got := growToLabels(area, []TextRun{label, prose}, drawn, grownGuards) + if got != area { + t.Errorf("grew to %v; one line of prose in the corridor and the edge stays", got) + } +} + +// TestAWrappedLabelIsTakenWhole covers the second half of the signal: a label's +// later lines carry no terminator of their own, and left unclaimed they are +// obstacles to the label they belong to. Page 521's lidar drawing claims nine of its +// eleven labels by terminator, and its two continuation lines block the edge from +// moving at all. +// +// The counter-case is what stops the rule swallowing a bulleted description. What +// separates them is that a bullet has its text beside it on the same baseline and a +// continuation line does not. +func TestAWrappedLabelIsTakenWhole(t *testing.T) { + area := CellRect{100, 100, 300, 300} + first := TextRun{X: 303, Y: 190, Width: 40, Height: 13, Text: "Модуль"} + second := TextRun{X: 303, Y: 201, Width: 30, Height: 13, Text: "на основе ИИ"} + drawn := append(growthDrawing(area), leaderMark(296, runMidY(first))) + + got := growToLabels(area, []TextRun{first, second}, drawn, grownGuards) + if got != (CellRect{100, 100, 343, 300}) { + t.Errorf("grew to %v, expected the right edge at the first line's far edge 343 "+ + "with the second line claimed rather than blocking it", got) + } + + // The counter-case. The bullet is 1.5 units wide and its text starts 1 unit past + // it on the same baseline, so the description IS flush with the label above it + // and IS on the adjacent line — the only thing left to refuse it is that it is + // not alone on its own baseline. + bullet := TextRun{X: 303, Y: 201, Width: 1.5, Height: 13, Text: "•"} + desc := TextRun{X: 305.5, Y: 201, Width: 60, Height: 13, Text: "Нажмите кнопку сброса"} + text := []TextRun{first, bullet, desc} + marks := []CellRect{leaderMark(296, runMidY(first)).Rect} + claimed := claimLabels(area, text, marks, edgeRight, defaultGuards) + if len(claimed) != 1 || claimed[0] != &text[0] { + got := make([]string, 0, len(claimed)) + for _, c := range claimed { + got = append(got, c.Text) + } + t.Errorf("claimed %q, expected only the label; a bullet's description is not "+ + "its continuation", got) + } + if grown := growToLabels(area, text, drawn, grownGuards); grown != area { + t.Errorf("grew to %v; the description is unclaimed and sits in the region", grown) + } +} + +// TestGrowthComparesBaselinesNotBands pins the reading a band comparison gets wrong. +// Two consecutive lines of one label overlap vertically, because a run is taller +// than the pitch it is set at — 13 units of height on a 10-unit pitch here — so a +// band test reports a label's own third line as something sharing the second's line, +// and that blocked every growth on the page this pass was written for. +func TestGrowthComparesBaselinesNotBands(t *testing.T) { + area := CellRect{100, 100, 300, 300} + text := []TextRun{ + {X: 303, Y: 190, Width: 40, Height: 13, Text: "Модуль"}, + {X: 303, Y: 200, Width: 36, Height: 13, Text: "на основе"}, + {X: 303, Y: 210, Width: 32, Height: 13, Text: "3D-датчики"}, + } + // Each line's band overlaps the next by 3 units, and only the first is pointed at. + for i := range text[:len(text)-1] { + if text[i].bottom() <= text[i+1].Y { + t.Fatalf("line %d ends at %.1f before line %d starts at %.1f; the point of "+ + "this test is that they overlap", i, text[i].bottom(), i+1, text[i+1].Y) + } + } + drawn := append(growthDrawing(area), leaderMark(296, runMidY(text[0]))) + + if got := growToLabels(area, text, drawn, grownGuards); got != (CellRect{100, 100, 343, 300}) { + t.Errorf("grew to %v, expected the right edge at 343; a three-line label whose "+ + "lines overlap must still grow", got) + } +} + +// TestTheCapIsAgainstTheDrawing covers [maxLabelGrowth] from both sides, and then +// the trap underneath it: the cap is measured against the drawing, so an edge's +// allowance does not grow because another edge moved first. +func TestTheCapIsAgainstTheDrawing(t *testing.T) { + // 100 wide, so at maxLabelGrowth of 1 the right edge may move 100 units. + area := CellRect{100, 100, 200, 300} + grow := func(labelWidth float64) CellRect { + label := TextRun{X: 203, Y: 190, Width: labelWidth, Height: 13, Text: "Bezeichnung"} + drawn := append(growthDrawing(area), leaderMark(196, runMidY(label))) + return growToLabels(area, []TextRun{label}, drawn, grownGuards) + } + // Far edge at 299: the edge moves 99 of its allowed 100. + if got := grow(96); got != (CellRect{100, 100, 299, 300}) { + t.Errorf("just inside the cap: grew to %v, expected the edge at 299", got) + } + // Far edge at 301: 101 units, and the whole growth is refused rather than + // clipped back to the cap. A label cut at an arbitrary line is not the point. + if got := grow(98); got != area { + t.Errorf("just outside the cap: grew to %v, expected no move at all", got) + } + + // Two edges. The left label needs 80 units and is allowed; the right label needs + // 150, which is over the drawing's own 100 and must stay refused even though the + // box is 180 wide by the time the right edge is judged. + left := TextRun{X: 20, Y: 190, Width: 60, Height: 13, Text: "links"} + right := TextRun{X: 203, Y: 220, Width: 147, Height: 13, Text: "rechts"} + drawn := append(growthDrawing(area), + leaderMark(96, runMidY(left)), leaderMark(204, runMidY(right))) + got := growToLabels(area, []TextRun{left, right}, drawn, grownGuards) + if got != (CellRect{20, 100, 200, 300}) { + t.Errorf("grew to %v, expected the left edge out to 20 and the right edge "+ + "unmoved; the first edge's growth must not enlarge the second's allowance", got) + } +} + +// TestAClaimedLabelMayBeCutAndProseMayNot is the asymmetry that makes the pass work +// on a page whose two label columns interleave in x. Page 521's lidar drawing +// reaches 397 where its own longest label ends at 469, because the neighbouring +// drawing's labels start at 400. Refusing to cut a label at all was measured and +// costs the whole page. +func TestAClaimedLabelMayBeCutAndProseMayNot(t *testing.T) { + area := CellRect{100, 100, 300, 300} + short := TextRun{X: 303, Y: 190, Width: 27, Height: 13, Text: "Deckel"} + long := TextRun{X: 303, Y: 220, Width: 97, Height: 13, Text: "Absaugen und Lösen"} + prose := TextRun{X: 340, Y: 150, Width: 50, Height: 13, Text: "the next column's prose"} + drawn := append(growthDrawing(area), + leaderMark(296, runMidY(short)), leaderMark(296, runMidY(long))) + + got := growToLabels(area, []TextRun{short, long, prose}, drawn, grownGuards) + if got != (CellRect{100, 100, 330, 300}) { + t.Errorf("grew to %v, expected the edge at the shorter label's 330 — cutting "+ + "the longer label rather than reaching over the prose at 340", got) + } +} + +// TestEachEdgeIsJudgedAgainstTheBoxAsAlreadyGrown is a real trap rather than a +// hypothetical: a prototype that computed all four edges from the original box +// admitted a run diagonally outside two of them on 2 figures. +// +// The run below is beyond neither edge on its own — it clears the left edge but does +// not reach the figure's vertical band, and clears the top edge but not its +// horizontal band — so it is never a candidate label. It only lands in the way once +// the left edge has moved, which is why the top edge must be judged against the +// grown box. +func TestEachEdgeIsJudgedAgainstTheBoxAsAlreadyGrown(t *testing.T) { + area := CellRect{100, 100, 300, 300} + left := TextRun{X: 60, Y: 190, Width: 35, Height: 13, Text: "links"} + top := TextRun{X: 190, Y: 60, Width: 40, Height: 13, Text: "oben"} + corner := TextRun{X: 65, Y: 70, Width: 25, Height: 13, Text: "diagonal"} + for _, side := range []int{edgeLeft, edgeRight, edgeTop, edgeBottom} { + if _, outside := runBeyond(area, &corner, side); outside { + t.Fatalf("the corner run is beyond edge %d; it has to be beyond none of "+ + "them for this test to mean anything", side) + } + } + // Each mark on its own label's midline: 96 is the left label's, and 210 is the + // top label's — a leader points AT its label, and [labelAlign] is 4 units. + drawn := append(growthDrawing(area), + leaderMark(96, runMidY(left)), leaderMark(top.X+top.Width/2, 96)) + + claimedTop := claimLabels(area, []TextRun{left, top, corner}, + marksOf(drawn), edgeTop, defaultGuards) + if len(claimedTop) != 1 { + t.Fatalf("the top edge claimed %d labels, expected the one; without a claim "+ + "there is nothing for the growth region to be judged against", len(claimedTop)) + } + + got := growToLabels(area, []TextRun{left, top, corner}, drawn, grownGuards) + if got != (CellRect{60, 100, 300, 300}) { + t.Errorf("grew to %v, expected the left edge out to 60 and the top refused; "+ + "the corner run is only in the way once the left edge has moved", got) + } +} + +// TestTheGuardsJudgeTheDrawingNotTheCrop pins the order the pass runs in, and the +// band made it load-bearing rather than defensive. A diagram's own labels are text, so +// the band that takes them in is legitimately over [maxFigureTextFraction] — page +// 521's lidar diagram reaches 0.162 with its eleven labels — and re-testing the CROP +// would reject the very pictures the labels complete. +// +// While the crop was the drawing this test had to construct the box that would have +// been judged. It no longer does: the box that would fail the guard is now the box +// that is rendered, so the assertion is direct. +func TestTheGuardsJudgeTheDrawingNotTheCrop(t *testing.T) { + drawn := CellRect{100, 100, 200, 200} + var runs []TextRun + ink := growthDrawing(drawn) + for _, y := range []float64{105, 120, 135, 150, 165, 180} { + r := TextRun{X: 203, Y: y, Width: 85, Height: 13, Text: "Bezeichnung"} + runs = append(runs, r) + ink = append(ink, leaderMark(196, runMidY(r))) + } + page := &PageRuns{No: 521, Width: 918, Height: 631, Runs: runs} + + figs := FindFigures(ink, page) + if len(figs) != 1 { + t.Fatalf("found %d figures, expected 1", len(figs)) + } + got := figs[0] + band := CellRect{100, 100, 288, 200} + if got.Rect != band { + t.Fatalf("Rect = %v, expected the drawing and its six labels %v", got.Rect, band) + } + if len(got.Labels) != len(runs) { + t.Fatalf("carried %d labels, expected all %d", len(got.Labels), len(runs)) + } + // The crop that WAS rendered is over the text guard. That is the case the ordering + // exists for: had the guards run on it, this figure would not exist at all. + if crop := textFraction(got.Rect, runs); crop <= maxFigureTextFraction { + t.Fatalf("the crop is %.3f text, under the %.2f guard; this test needs a crop "+ + "the guard would have rejected", crop, maxFigureTextFraction) + } + if got.InkRect != drawn { + t.Errorf("InkRect = %v, expected the drawing %v", got.InkRect, drawn) + } + if want := textFraction(drawn, runs); got.TextFraction != want { + t.Errorf("TextFraction = %.3f, expected %.3f — the drawing's, not the crop's", + got.TextFraction, want) + } + // Every shape, the six terminators included: a leader's mark is what SETS the + // edge it sits on, so it is inside the drawing's box, not out in the corridor. + if got.Ink != len(ink) { + t.Errorf("Ink = %d, expected all %d shapes of the drawing", got.Ink, len(ink)) + } +} + +// TestDrawnExtentFallsBackToRect covers the two callers that legitimately have no +// ink box: a figure read back out of the database, where the drawn extent is not +// stored, and a figure built by hand in a test. Before this pass the two rects were +// one rect, which is why the fallback is right rather than an error. +func TestDrawnExtentFallsBackToRect(t *testing.T) { + crop := CellRect{100, 100, 276, 300} + drawn := CellRect{100, 100, 263, 300} + stored := Figure{Rect: crop} + if got := stored.DrawnExtent(); got != crop { + t.Errorf("DrawnExtent = %v with no InkRect, expected Rect %v", got, crop) + } + fresh := Figure{Rect: crop, InkRect: drawn} + if got := fresh.DrawnExtent(); got != drawn { + t.Errorf("DrawnExtent = %v, expected InkRect %v", got, drawn) + } +} diff --git a/internal/doc/figures_pdf_test.go b/internal/doc/figures_pdf_test.go new file mode 100644 index 0000000..ebe4b49 --- /dev/null +++ b/internal/doc/figures_pdf_test.go @@ -0,0 +1,200 @@ +package doc_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// These run the whole figure path — pdftocairo, the guards, pdftoppm — against a +// PDF this package writes, so the default suite covers it offline with nothing +// committed. +// +// What they cannot cover is what the fixture tests are for. A generated drawing is +// clean: no clip path inflating its box, no gradient mesh of 80,000 hairlines, no +// language badge repeated on 110 pages, no ruled table shaped exactly like a framed +// illustration. Every constant in figures.go came from the real manuals, and these +// tests check the plumbing between them rather than the judgement. +// +// testpdf grew a Drawings field for this. It wrote text only, and a document with +// no vector graphics exercises none of this — which is also the finding that +// justified generating vector rather than embedding a raster: over 628 pages of the +// two real manuals, pdfimages yields no illustration at all. + +func figurePDF(t *testing.T, d testpdf.Doc) string { + t.Helper() + for _, tool := range []extern.Tool{extern.PDFToHTML, extern.PDFToCairo, extern.PDFToPPM} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + path := filepath.Join(t.TempDir(), "generated.pdf") + if err := os.WriteFile(path, d.Build(), 0o600); err != nil { + t.Fatalf("write the generated PDF: %v", err) + } + return path +} + +// TestAGeneratedDrawingComesBackAsAFigure is the end-to-end check: a drawing is +// written into a PDF, and the same drawing comes back with a rectangle in the +// right place and bytes that are a PNG of the right size. +func TestAGeneratedDrawingComesBackAsAFigure(t *testing.T) { + // 200x150 points at (100, 400) on the 612x792 page, with 40 strokes inside — + // well over minFigureInk. Poppler's coordinates are 1.5x and measured down from + // the top, so the frame arrives at x 150-450 and y 1.5*(792-550)=363 to + // 1.5*(792-400)=588. + path := figurePDF(t, testpdf.Doc{Pages: []testpdf.Page{{ + Lines: []string{"A page with a picture on it"}, + Drawings: []testpdf.Drawing{{X: 100, Y: 400, W: 200, H: 150, Strokes: 40}}, + }}}) + + pages, err := doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + figs, err := doc.PageFigures(context.Background(), path, &pages[0]) + if err != nil { + t.Fatalf("PageFigures: %v", err) + } + if len(figs) != 1 { + t.Fatalf("found %d figures, expected the one drawing", len(figs)) + } + f := &figs[0] + + // The rectangle, in the 1.5-scaled space. A stroke has width, so the box runs a + // fraction outside the frame's centre line; a point of slack covers that. + for _, c := range []struct { + name string + got, want float64 + slack float64 + }{ + {"left", f.Rect.X0, 150, 2}, + {"right", f.Rect.X1, 450, 2}, + {"top", f.Rect.Y0, 363, 2}, + {"bottom", f.Rect.Y1, 588, 2}, + } { + if d := c.got - c.want; d > c.slack || d < -c.slack { + t.Errorf("%s edge = %.1f, expected about %.0f", c.name, c.got, c.want) + } + } + + // The frame plus its 40 strokes. + if f.Ink != 41 { + t.Errorf("ink = %d, expected the frame and its 40 strokes", f.Ink) + } + if f.Page != 1 || f.Index != 0 { + t.Errorf("figure is page %d index %d, expected page 1 index 0", f.Page, f.Index) + } + + // The bytes: a PNG, at 216 dpi, the rectangle doubled. + if f.DPI != 216 { + t.Errorf("rendered at %d dpi, expected 216", f.DPI) + } + if f.PixelWidth < 599 || f.PixelWidth > 606 { + t.Errorf("width = %d pixels, expected about 600", f.PixelWidth) + } + if f.PixelHeight < 449 || f.PixelHeight > 456 { + t.Errorf("height = %d pixels, expected about 450", f.PixelHeight) + } + if len(f.Digest) != 64 || len(f.PNG) == 0 { + t.Errorf("digest %q over %d bytes; expected 64 hex characters over a PNG", + f.Digest, len(f.PNG)) + } + if string(f.PNG[:4]) != "\x89PNG" { + t.Errorf("the bytes do not start with a PNG signature") + } +} + +// TestAPageOfTextAloneHasNoFigures is the negative, and it is the one that would +// catch the worst failure this code can have: reporting the page itself as a +// picture. Nothing here draws, so nothing here is a figure. +func TestAPageOfTextAloneHasNoFigures(t *testing.T) { + path := figurePDF(t, testpdf.TaggedSections([]string{"EN", "DE"}, 2, true)) + + pages, err := doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + for i := range pages { + figs, err := doc.PageFigures(context.Background(), path, &pages[i]) + if err != nil { + t.Fatalf("PageFigures page %d: %v", pages[i].No, err) + } + if len(figs) != 0 { + t.Errorf("page %d of a text-only document returned %d figures: %v", + pages[i].No, len(figs), figs[0].Rect) + } + } +} + +// TestTwoDrawingsAreTwoFiguresInReadingOrder checks the plumbing the real +// documents cannot: separated drawings staying separate, and numbered down the +// page. On the columns manual this is exactly what a clip path defeats. +func TestTwoDrawingsAreTwoFiguresInReadingOrder(t *testing.T) { + path := figurePDF(t, testpdf.Doc{Pages: []testpdf.Page{{ + Drawings: []testpdf.Drawing{ + // Lower on the page first, so the sort has something to do. In PDF + // points Y counts up, so Y=120 is below Y=500. + {X: 100, Y: 120, W: 180, H: 120, Strokes: 30}, + {X: 100, Y: 500, W: 180, H: 120, Strokes: 30}, + }, + }}}) + + pages, err := doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + figs := doc.FindFigures(mustInk(t, path, 1), &pages[0]) + if len(figs) != 2 { + t.Fatalf("found %d figures, expected 2", len(figs)) + } + if figs[0].Rect.Y0 >= figs[1].Rect.Y0 { + t.Errorf("figures came back bottom-first: %.0f then %.0f", + figs[0].Rect.Y0, figs[1].Rect.Y0) + } + if figs[0].Index != 0 || figs[1].Index != 1 { + t.Errorf("indexes are %d and %d, expected 0 and 1", figs[0].Index, figs[1].Index) + } +} + +// TestASimpleShapeIsNotAFigure is the shape guard end to end. Three strokes is a +// logo; the guard is what stops every page badge in a real manual becoming a +// picture. +func TestASimpleShapeIsNotAFigure(t *testing.T) { + path := figurePDF(t, testpdf.Doc{Pages: []testpdf.Page{{ + Drawings: []testpdf.Drawing{{X: 100, Y: 400, W: 200, H: 150, Strokes: 2}}, + }}}) + + pages, err := doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + if figs := doc.FindFigures(mustInk(t, path, 1), &pages[0]); len(figs) != 0 { + t.Errorf("a frame with two strokes in it came back as a figure: %v", figs[0].Rect) + } +} + +// TestExtractInkRejectsANonPage covers the argument check, which is the one error +// path a caller can hit without poppler being involved. +func TestExtractInkRejectsANonPage(t *testing.T) { + if _, err := doc.ExtractInk(context.Background(), "irrelevant.pdf", 0); err == nil { + t.Error("page 0 was accepted") + } + if _, err := doc.PageFigures(context.Background(), "irrelevant.pdf", nil); err == nil { + t.Error("a nil page was accepted") + } +} + +func mustInk(t *testing.T, path string, page int) []doc.Ink { + t.Helper() + ink, err := doc.ExtractInk(context.Background(), path, page) + if err != nil { + t.Fatalf("ExtractInk page %d: %v", page, err) + } + return ink +} diff --git a/internal/doc/fixture_test.go b/internal/doc/fixture_test.go new file mode 100644 index 0000000..6072b0a --- /dev/null +++ b/internal/doc/fixture_test.go @@ -0,0 +1,321 @@ +package doc_test + +import ( + "context" + "os" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" +) + +// fixturesDir is relative to this package. +const fixturesDir = "../../testdata/fixtures" + +// These tests run the whole free pipeline against a real 560-page, 34-language +// appliance manual. They are the only honest check that the language map works: +// a synthetic PDF cannot reproduce a printed index that contradicts itself, a +// back cover in the wrong language, or sibling languages a detector confuses. +// +// The document is fetched on demand and is not committed — it is 15 MB of someone +// else's copyrighted manual. Without MANUALBOX_TEST_FIXTURES=1 these skip, so the +// default suite stays hermetic and offline. + +func loadFixture(t *testing.T) (manifest *fixture.Manifest, path string) { + t.Helper() + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixture and run the real-document tests", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFInfo, extern.PDFToText} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + + m, err := fixture.Load(fixturesDir, "dreame-l40-ultra") + if err != nil { + t.Fatalf("load manifest: %v", err) + } + cached, err := m.Fetch(context.Background()) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + return m, cached +} + +func analyzeFixture(t *testing.T) (manifest *fixture.Manifest, result *doc.Result) { + t.Helper() + m, cached := loadFixture(t) + res, err := doc.Analyze(context.Background(), cached) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + return m, res +} + +// TestProbeMatchesManifest checks stage 0 and stage 1 against facts measured +// independently and recorded in the manifest. +func TestProbeMatchesManifest(t *testing.T) { + m, res := analyzeFixture(t) + + if res.Info.Pages != m.Pages { + t.Errorf("page count = %d, manifest says %d", res.Info.Pages, m.Pages) + } + if res.Info.Encrypted { + t.Error("document reported as encrypted; the manifest describes an open PDF") + } + if res.HasTextLayer != m.HasTextLayer { + t.Errorf("has text layer = %t, manifest says %t", res.HasTextLayer, m.HasTextLayer) + } + + // The manifest records the median in runes. It was previously recorded in + // bytes, which for a document full of Cyrillic, Greek, Hebrew, Arabic and CJK + // is a third larger — hence the tolerance being tight rather than generous. + if got, want := res.MedianChars, m.MedianCharsPerPage; abs(got-want) > 20 { + t.Errorf("median chars per page = %d, manifest says %d", got, want) + } + + if res.ContentStart != m.ContentStartsOnPDFPage { + t.Errorf("content starts on page %d, manifest says %d", res.ContentStart, m.ContentStartsOnPDFPage) + } +} + +// TestLanguageMapMatchesManifest is the central assertion of the ingest pipeline: +// every language section, its boundaries, and its page count, on a real document. +func TestLanguageMapMatchesManifest(t *testing.T) { + m, res := analyzeFixture(t) + + summaries := res.Languages() + if len(summaries) != len(m.Sections) { + t.Errorf("found %d languages, manifest records %d", len(summaries), len(m.Sections)) + for _, s := range summaries { + t.Logf(" found %-6s %-12s pages %d, first page %d", s.Code, s.Lang, s.Pages, s.FirstPage) + } + } + + byCode := make(map[string]doc.LanguageSummary, len(summaries)) + for _, s := range summaries { + byCode[s.Code] = s + } + + for _, want := range m.Sections { + got, ok := byCode[want.Code] + if !ok { + t.Errorf("%s: not found; manifest expects pages %d-%d", want.Code, want.PDFStart, want.PDFEnd) + continue + } + if got.FirstPage != want.PDFStart { + t.Errorf("%s: starts at page %d, manifest says %d", want.Code, got.FirstPage, want.PDFStart) + } + // The end page is asserted separately from the page total on purpose. A + // section wrongly split into two spans can still total the right number of + // pages — that happened during development and a totals-only check missed + // it entirely. + if got.LastPage != want.PDFEnd { + t.Errorf("%s: ends at page %d, manifest says %d", want.Code, got.LastPage, want.PDFEnd) + } + if got.Pages != want.Pages { + t.Errorf("%s: %d pages, manifest says %d", want.Code, got.Pages, want.Pages) + } + if got.Runs != 1 { + t.Errorf("%s: split across %d spans; each section in this document is contiguous", + want.Code, got.Runs) + } + } +} + +// TestEveryContentPageIsLabelled asserts the property that actually matters +// downstream: no page of real content is left without a language, because an +// unlabelled page cannot be included in or excluded from a translation scope. +func TestEveryContentPageIsLabelled(t *testing.T) { + m, res := analyzeFixture(t) + + // Unlabelled is not expected to be zero, and demanding that it were is what + // made this assertion weak. A cover and a colophon carry text and belong to no + // language section, so the honest expectation is "a handful, all of them + // furniture" — the magnitude is the signal, and it is what says whether a + // statistical detector would earn its 118 MB on this document. + if res.Unlabelled > 8 { + t.Errorf("%d pages carry text but no language label; on this document only "+ + "front matter and the back cover should", res.Unlabelled) + } + + // Unlabelled == 0 is not sufficient, and the name of this test used to promise + // more than it delivered. countUnlabelled bounds itself by the content range, + // which is derived from the runs themselves — so a section lost at either end + // of the document simply shrinks the range and still reports zero. Verified: + // deleting the entire English section left this test green. + // + // So count the pages actually covered and require every content page of the + // document to be among them. + covered := make(map[int]bool, m.Pages) + for _, run := range res.Runs { + for p := run.Start; p <= run.End; p++ { + covered[p] = true + } + } + + wantPages := 0 + for _, s := range m.Sections { + wantPages += s.Pages + } + if len(covered) != wantPages { + t.Errorf("%d pages carry a language, but the manifest accounts for %d", len(covered), wantPages) + } + + var missing []int + for _, s := range m.Sections { + for p := s.PDFStart; p <= s.PDFEnd; p++ { + if !covered[p] { + missing = append(missing, p) + } + } + } + if len(missing) > 0 { + show := missing + if len(show) > 12 { + show = show[:12] + } + t.Errorf("%d content pages are in no run at all, e.g. %v", len(missing), show) + } + + // Every page that carries text but no label must lie outside every section. + // That is the property "unlabelled means furniture" actually asserts; a bare + // count cannot distinguish a cover from a lost section. + var strays []int + for i := range res.Pages { + p := &res.Pages[i] + if covered[p.No] || p.Chars < 50 { + continue + } + for _, s := range m.Sections { + if p.No >= s.PDFStart && p.No <= s.PDFEnd { + strays = append(strays, p.No) + break + } + } + } + if len(strays) > 0 { + t.Errorf("pages %v are inside a language section but carry no label", strays) + } +} + +// TestPageTagSignalIsExact records what the printed per-page tag achieves on this +// document, because it is the measurement that justified building the pipeline +// around it. If a change degrades it, this test says so specifically rather than +// leaving a general language-map failure to be diagnosed. +func TestPageTagSignalIsExact(t *testing.T) { + m, res := analyzeFixture(t) + + tagRuns := res.BySource[doc.SourcePageTag] + if len(tagRuns) != len(m.Sections) { + t.Fatalf("page tag produced %d runs, expected %d sections", len(tagRuns), len(m.Sections)) + } + + expected := make(map[int]string, m.Pages) + for _, s := range m.Sections { + for p := s.PDFStart; p <= s.PDFEnd; p++ { + expected[p] = s.Code + } + } + + wrong := 0 + for _, run := range tagRuns { + for p := run.Start; p <= run.End; p++ { + if want, ok := expected[p]; ok && want != run.Code { + if wrong < 5 { + t.Errorf("page %d: tag says %s, manifest says %s", p, run.Code, want) + } + wrong++ + } + } + } + if wrong > 0 { + t.Errorf("%d pages disagree with the manifest", wrong) + } +} + +// TestContentsPagesDoNotBecomeSections guards the measured false-positive case: a +// manual's contents pages list every language code in the same position the +// per-page tab occupies. Without the run-length guard this document gains three +// bogus single-page sections. +func TestContentsPagesDoNotBecomeSections(t *testing.T) { + m, res := analyzeFixture(t) + + for _, run := range res.Runs { + for _, indexPage := range m.IndexPages { + if run.Contains(indexPage) { + t.Errorf("contents page %d was absorbed into a %s section (pages %d-%d)", + indexPage, run.Code, run.Start, run.End) + } + } + } +} + +// TestScopeIsASmallFractionOfTheDocument is the premise of the whole design: a +// household that reads three languages should be asked to process a few per cent +// of a 34-language manual, not all of it. +func TestScopeIsASmallFractionOfTheDocument(t *testing.T) { + _, res := analyzeFixture(t) + + scope := res.ScopeFor([]string{"de", "uk", "en"}) + if len(scope.Languages) != 3 { + t.Errorf("expected 3 household languages in scope, got %d", len(scope.Languages)) + for _, l := range scope.Languages { + t.Logf(" in scope: %s (%s), %d pages", l.Code, l.Lang, l.Pages) + } + } + if scope.Fraction() > 0.15 { + t.Errorf("scope is %.1f%% of the document; the design expects roughly 10%%", + 100*scope.Fraction()) + } + if len(scope.OtherLanguages) == 0 { + t.Error("no other languages reported; the user must be able to see what else is in the document") + } + t.Logf("scope: %d of %d pages (%.1f%%), %d chars, %d other languages available", + scope.Pages, scope.TotalPages, 100*scope.Fraction(), scope.Chars, len(scope.OtherLanguages)) +} + +// TestIndexDisagreementIsSurfaced checks that the manual's own contents table +// being wrong is reported rather than silently accepted or silently corrected. +// This document's index misplaces several sections. +func TestIndexDisagreementIsSurfaced(t *testing.T) { + _, res := analyzeFixture(t) + + indexRuns := res.BySource[doc.SourceIndex] + if len(indexRuns) == 0 { + t.Fatal("the printed index was not parsed at all") + } + + titled := 0 + for _, r := range indexRuns { + if r.Title != "" { + titled++ + } + } + if titled < 30 { + t.Errorf("only %d index entries carry a section title; the index supplies titles no other signal can", titled) + } + + // Titles must survive into the reconciled view, since that is what the UI + // shows. + withTitle := 0 + for _, r := range res.Runs { + if r.Title != "" { + withTitle++ + } + } + if withTitle == 0 { + t.Error("no reconciled run carries a printed section title") + } + t.Logf("index parsed: %d entries, %d with titles; %d reconciled runs carry a title", + len(indexRuns), titled, withTitle) +} + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} diff --git a/internal/doc/fonts_fixture_test.go b/internal/doc/fonts_fixture_test.go new file mode 100644 index 0000000..806cdc7 --- /dev/null +++ b/internal/doc/fonts_fixture_test.go @@ -0,0 +1,236 @@ +package doc_test + +import ( + "context" + "fmt" + "sort" + "testing" + "unicode/utf8" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" +) + +// This is the measurement the heading rule will be built on, taken from both real +// manuals rather than from a generated one — a generated document has as many +// fonts as its generator was told to write, which settles nothing. +// +// It asserts only the properties that must not silently change, and logs the +// distribution. The line between the two is deliberate: the share of characters +// in a given face is a fact about someone else's typesetting and asserting it +// would break on a manifest bump, but a run losing its font, or either weight +// signal going to zero, is a defect here. + +// fontStat accumulates one font's share of a document. +type fontStat struct { + size float64 + family string + weight doc.Weight + oblique bool + runs int + chars int + markedBold int + markedItalic int +} + +// fontProfile summarizes every run of a document by the font it is set in. +// +// Characters are counted in runes, not bytes: half of the sequential manual is +// Cyrillic, Greek, Hebrew, Arabic or CJK, where bytes run a third higher and +// would make those sections look like more of the document than they are. +type fontProfile struct { + pages, runs, chars int + unresolved int + byFont map[string]*fontStat + bySize map[float64]int + byWeight map[doc.Weight]int +} + +func profileFonts(pages []doc.PageRuns) *fontProfile { + p := &fontProfile{ + pages: len(pages), + byFont: make(map[string]*fontStat), + bySize: make(map[float64]int), + byWeight: make(map[doc.Weight]int), + } + for i := range pages { + for j := range pages[i].Runs { + r := &pages[i].Runs[j] + n := utf8.RuneCountInString(r.Text) + p.runs++ + p.chars += n + if r.Font.Family == "" && r.Font.Size == 0 { + p.unresolved++ + continue + } + p.bySize[r.Font.Size] += n + p.byWeight[r.Font.Weight] += n + + key := fmt.Sprintf("%g|%s", r.Font.Size, r.Font.Family) + s := p.byFont[key] + if s == nil { + s = &fontStat{ + size: r.Font.Size, family: r.Font.Family, + weight: r.Font.Weight, oblique: r.Font.Oblique, + } + p.byFont[key] = s + } + s.runs++ + s.chars += n + if r.Font.MarkedBold { + s.markedBold++ + } + if r.Font.MarkedItalic { + s.markedItalic++ + } + } + } + return p +} + +// report logs the distribution in descending share of characters. +func (p *fontProfile) report(t *testing.T, name string) { + t.Helper() + t.Logf("%s: %d pages, %d runs, %d chars; %d runs with no resolvable font", + name, p.pages, p.runs, p.chars, p.unresolved) + + stats := make([]*fontStat, 0, len(p.byFont)) + for _, s := range p.byFont { + stats = append(stats, s) + } + sort.Slice(stats, func(i, j int) bool { return stats[i].chars > stats[j].chars }) + + // Characters per run is logged because it is what separates a heading from + // emphasis at the same size and weight: a heading is a short run, a bold lead-in + // or a table label is shorter still, and body copy is long. + t.Logf(" %-8s %-40s %-9s %8s %6s %8s %7s", + "size", "family", "name-says", "chars", "share", "runs", "ch/run") + shown := 0 + for _, s := range stats { + if shown == 14 { + t.Logf(" ... and %d more fonts", len(stats)-shown) + break + } + shown++ + says := s.weight.String() + if s.oblique { + says += "+obl" + } + marks := "" + if s.markedBold > 0 { + marks += fmt.Sprintf(" on %d/%d", s.markedBold, s.runs) + } + if s.markedItalic > 0 { + marks += fmt.Sprintf(" on %d/%d", s.markedItalic, s.runs) + } + t.Logf(" %-8g %-40s %-9s %8d %5.1f%% %8d %7.1f%s", + s.size, s.family, says, s.chars, 100*float64(s.chars)/float64(p.chars), + s.runs, float64(s.chars)/float64(s.runs), marks) + } + + sizes := make([]float64, 0, len(p.bySize)) + for size := range p.bySize { + sizes = append(sizes, size) + } + sort.Slice(sizes, func(i, j int) bool { return p.bySize[sizes[i]] > p.bySize[sizes[j]] }) + line := "" + for i, size := range sizes { + if i == 8 { + break + } + line += fmt.Sprintf(" %g: %.1f%%", size, 100*float64(p.bySize[size])/float64(p.chars)) + } + t.Logf(" chars by size:%s", line) + + // The two signals side by side. Which of them carries the document is not the + // same on the two manuals, which is the finding that says neither can be + // dropped: see the commit that added Font. + boldChars, boldRuns := 0, 0 + for _, s := range p.byFont { + if s.markedBold > 0 { + boldRuns += s.markedBold + boldChars += s.chars * s.markedBold / s.runs + } + } + t.Logf(" poppler marked bold: %d runs, about %d chars (%.1f%%)", + boldRuns, boldChars, 100*float64(boldChars)/float64(p.chars)) + + for _, w := range []doc.Weight{doc.WeightUnknown, doc.WeightLight, doc.WeightRegular, + doc.WeightMedium, doc.WeightSemibold, doc.WeightBold, doc.WeightHeavy} { + if p.byWeight[w] == 0 { + continue + } + t.Logf(" name says %-9s %8d chars %5.1f%%", w, + p.byWeight[w], 100*float64(p.byWeight[w])/float64(p.chars)) + } +} + +// markedBoldChars and markedBoldRuns count what poppler itself called bold. +func (p *fontProfile) markedBoldRuns() int { + n := 0 + for _, s := range p.byFont { + n += s.markedBold + } + return n +} + +// heavierThanRegularChars is what the family names claim above regular weight — +// the signal poppler's markup does not carry. +func (p *fontProfile) heavierThanRegularChars() int { + n := 0 + for w, chars := range p.byWeight { + if w >= doc.WeightMedium { + n += chars + } + } + return n +} + +// TestFontDistributionOfBothManuals records what the two real documents are set +// in, and asserts the parts a change must not break. +func TestFontDistributionOfBothManuals(t *testing.T) { + if !extern.Available(extern.PDFToHTML) { + t.Skip("pdftohtml is not installed") + } + + _, columnPath := columnFixture(t) + _, sequentialPath := loadFixture(t) + + for _, tc := range []struct { + name string + path string + }{ + {"parallel-columns manual", columnPath}, + {"sequential manual", sequentialPath}, + } { + pages, err := doc.ExtractRuns(context.Background(), tc.path) + if err != nil { + t.Fatalf("%s: ExtractRuns: %v", tc.name, err) + } + p := profileFonts(pages) + p.report(t, tc.name) + + // Every run of a real manual must resolve to a font. This is the assertion + // that fails if the fontspec table is scoped to a page: poppler declares each + // id once, on the page that first uses it, so 83% of the columns manual's + // runs and 95% of the sequential manual's refer back to an earlier page. + if p.unresolved != 0 { + t.Errorf("%s: %d of %d runs have no font — a fontspec id is declared once "+ + "per document, on the page that first uses it, so the table cannot be "+ + "scoped to a page", tc.name, p.unresolved, p.runs) + } + + // Both weight signals must keep contributing, for the same reason the column + // language test checks both of its sources: either one collapsing to zero is + // invisible in a total, and each catches emphasis the other cannot see. + if got := p.markedBoldRuns(); got == 0 { + t.Errorf("%s: poppler marked no run bold; it marks emphasis no family "+ + "name admits, such as the columns manual's FuturaBQ", tc.name) + } + if got := p.heavierThanRegularChars(); got*100/p.chars < 3 { + t.Errorf("%s: only %d of %d chars are in a face whose name claims more "+ + "than regular weight; Medium alone is 17.2%% of the columns manual and "+ + "poppler marks none of it", tc.name, got, p.chars) + } + } +} diff --git a/internal/doc/furniture.go b/internal/doc/furniture.go new file mode 100644 index 0000000..03cf461 --- /dev/null +++ b/internal/doc/furniture.go @@ -0,0 +1,700 @@ +package doc + +import ( + "fmt" + "math" + "sort" + "strconv" +) + +// Page furniture: the text a manual prints because of where a page is, not +// because of what the page says — a language tab in the corner, a folio in the +// footer, a running head at the top. +// +// [RegionBlocks] serves all of it as content and cannot do otherwise. The reason +// is recorded in blocks.go and in docs/design/conversion.md and it is worth +// repeating because it is the whole design of this file: NOTHING ON A SINGLE PAGE +// SEPARATES FURNITURE FROM CONTENT. The sequential manual genuinely titles +// sections "A", "B", "C" and "E" — 28 pages of it head a page with a bare "A" — +// so "a one-letter heading is a tab" is false on the document this is for. What +// identifies furniture is that it repeats in the same place page after page, +// which is a property of the document and not of a page, so it belongs here, +// beside [Convert], where the whole document is in view. +// +// # The rule, and the denominator that had to be got right first +// +// Furniture is text that repeats at the same height on the pages of ONE +// LANGUAGE'S section, and the second half is the part the first attempt got +// wrong. Repetition across the pages a HOUSEHOLD converted is not the signal: a +// household reading German, Russian and Japanese converts 59 pages of the +// sequential manual, and the German tab is on 16 of them — a 0.27 share that no +// threshold can separate from a genuinely repeated heading. Measured the same way +// on the column manual, German plus Ukrainian is 52 pages and the German tab is +// 19, a 0.37 share. Counted against the language's OWN pages both are 1.00. So +// the pass is per language, and the denominator is the pages that language's +// regions occupy. +// +// Three clauses, because the three kinds of furniture differ in what stays the +// same — the tab prints the same characters on every page of a section, the folio +// prints different ones on every page, and the running head prints the same +// characters on a RUN of pages and then changes — and one rule cannot have it +// three ways. +// +// 1. A TAB, OR ANY REPEATED LINE. The same text at the same height on at least +// [furnitureMinShare] of a language's pages, and on at least +// [furnitureMinPages] of them. Stated generally rather than as "a short +// token": if a manual prints the same running head in the same place on most +// pages of a section, that is furniture by the same evidence, and narrowing +// the rule to two-letter tabs would be fitting it to the two documents in +// hand. What the two documents in hand actually contain is measured at +// [furnitureMinShare]. +// +// 2. A FOLIO. A run whose text is exactly the page number that page prints, at a +// height where such a run occurs on at least [furnitureMinPages] pages of the +// language. It needs no share threshold, and it must not have one: the column +// manual prints its folio in the outer margin, so the folio falls inside +// whichever language holds the outer column of that page, and German gets it +// on 7 of its 26 pages — a 0.27 share, below any cut that clause 1 can use. +// What replaces the share is a second opinion. "The page number that page +// prints" is [Page.Folio], which `pdftotext` read from the same bytes through +// none of this code, so a run agreeing with it is not a coincidence being +// believed on repetition alone. +// +// 3. A RUNNING HEAD. The page's FIRST PRINTED LINE, when the page before it in +// the same language section printed an identical first line. The first page of +// each such run keeps its line and every page after it loses one, so a title +// is served exactly once, where its section starts. Like the folio it needs no +// share and no page floor, and unlike either of the others it cannot be stated +// over the whole section at once, because what makes it furniture is that the +// repetition is CONSECUTIVE. [runningHeads] has the rule and every number under +// it. +// +// # What this deliberately does not identify +// +// **The second line of a two-line running head.** Clause 3 claims one line per +// page and never more. The column manual's Polish head is two printed lines inside +// one grey banner — "Czyszczenie pojemnika" over "AQUA-Box", read off a 108 dpi +// render — so pages 42 and 44 keep "AQUA-Box" after losing the line above it. The +// measurement that stopped the clause at one line is at [runningHeads]; the cost +// is 2 pages of one language of one document, and it is an UNDER-removal, which is +// the only direction this clause can fail in. +// +// **A tab that only some pages of a section print.** The share is a share, so a +// section printing its tab on a third of its pages keeps it. Nothing in either +// manual does; a document that does would need this measured again rather than +// the threshold lowered, because 0.5 is where it is for a reason. + +// Bounds on what repetition is, measured over every language section of both +// fixtures — 5 of the 68-page parallel-columns manual and 34 of the 560-page +// sequential one, 39 sections in all. TestFurnitureThresholdsOnBothManuals +// prints the sweep these came from. +const ( + // furnitureYTolerance is how far apart two runs' tops may be and still count + // as the same height, in the 1.5-scaled space [ExtractRuns] reports. + // + // The same 2.0 [orderSlack] in internal/verify uses, and for the same reason: + // two runs a typesetter put on one line differ by rounding, and the tabs + // measured here differ by nothing at all — the sequential manual's tab is at + // y=58.0 on all 16 pages of its German section and the column manual's at + // y=16.0 on all 26. A tolerance is carried anyway because a document that + // composes its head per page rather than on a master would jitter, and 2.0 is + // an eighth of the tightest line pitch either manual sets (16), so it cannot + // merge two lines. + furnitureYTolerance = 2.0 + + // furnitureMinShare is how many of a language's pages must carry the same text + // at the same height before it is furniture. + // + // Measured over all 39 language sections, as (pages carrying it / the + // language's pages), the two populations do not touch: + // + // the printed tab 0.81 to 1.00 -- 37 of the 39 sections + // the widest anything else 0.29 + // + // The 37 are every section of the sequential manual at 1.00 — its tab is on + // every page of every section, 12 to 22 pages each — and German, Polish and + // Ukrainian on the column manual at 1.00, 0.81 and 0.85. The two sections with + // no tab bucket at all are the column manual's Russian and Kazakh, which print + // none in their columns. + // + // The column manual's 0.81 is not the tab being absent from five pages. It is + // [usableRuns] dropping it as sub-legible: the tab is set smaller than the + // [minRunHeightFraction] of the page's median run height on the pages whose + // median is a heading's, so on those pages it was never a block to claim. Which + // is why the numerator here is counted after that filter and not before — a + // share taken over runs the block builder never sees is a share of the wrong + // thing. + // + // The 0.29 is the ceiling of everything that is NOT furniture, and it is worth + // naming what is at it, because these are what a lower threshold would eat: + // the sequential manual's per-section running heads ("Плановое обслуживание" on + // 6 of Russian's 22 pages, 0.27; "Sicherheitshinweise" on 4 of German's 16, + // 0.25) and the column manual's chapter heads ("Waschsaugen" on 6 of 26, 0.23). + // Every one is a line a reader wants. + // + // 0.5 is the middle of 0.29 and 0.96 on a log scale as well as a linear one: + // 1.7x above everything real and 1.9x below every tab. It is not tuned to + // either document, and there is nothing between the two populations to tune it + // into. + furnitureMinShare = 0.5 + + // furnitureMinPages is how many pages must carry a thing before a share means + // anything, and it is not belt and braces: without it the rule is worthless on + // a short section. + // + // Measured. The column manual has a two-page spread of service addresses whose + // language no signal could name. At a share of 0.5 and no page floor, EVERY + // line printed on both pages is furniture — 400 buckets, the whole spread, + // because one page out of two is a half. The sequential manual's front matter + // does the same over 7 pages. + // + // 4 is above every accident measured and far below every real tab: the smallest + // tab bucket in either document is the sequential manual's Chinese section at + // 12 pages, and the smallest section of either document is 12 pages. It is also + // what the folio clause uses in place of a share, where it is the only guard, + // so it is stated once. + furnitureMinPages = 4 + + // maxFolioRunes bounds how long a run can be and still be compared against a + // printed page number. The same 4 [maxRunesInFolio] allows, and deliberately + // the same constant's value rather than a new one: a folio this does not + // recognise is one [pageFolio] never reported either. + maxFolioRunes = maxRunesInFolio +) + +// Furniture is which runs of a document are page furniture, and why. +// +// It is built once for a whole document by [FindFurniture] and consulted per +// region by [RegionBlocks]. A nil *Furniture is a normal argument and means +// "furniture was not looked for", which is what every caller that has one page +// and not the document passes — a page cannot answer the question. +type Furniture struct { + // notes is page -> the furniture on it -> why. Keyed on the height and the + // text rather than on an index into the page's runs, because the runs + // [RegionBlocks] asks about are copies twice removed: usableRuns and runsInBox + // each return a new slice. + notes map[int]map[furnitureKey]string + + // Tabs, Folios and Heads are how many distinct pieces of furniture each clause + // claimed, over the whole document: one per page per thing, so a page printing + // its tab twice at the same height counts once. Counted rather than derived so + // that a test and a report can hold the three clauses apart, which is how each + // rule was measured in the first place. + // + // Heads counts RUNS, not lines: a running head set as two runs on one baseline + // is one head. The other two count runs, because a tab and a folio are each one + // run by construction. + Tabs, Folios, Heads int +} + +type furnitureKey struct { + // line is the run's top, rounded to furnitureYTolerance. + line int + // text is the run's text, normalised the way [furnitureText] normalises it. + text string +} + +// furnitureText is how two runs' text is compared. +// +// [stripFormatting] first, for the reason it exists: a right-to-left page wraps +// its Latin furniture in bidi controls, so the Hebrew section's tab reading "HE" +// is really RLE LRE H E PDF PDF, and matching it against the same tab on the next +// page fails on the invisible characters. The sequential manual has a Hebrew and +// an Arabic section, and both were checked — 16 of 16 pages each. +func furnitureText(s string) string { return collapseSpaces(stripFormatting(s)) } + +func furnitureLine(y float64) int { return int(math.Round(y / furnitureYTolerance)) } + +func keyOf(r *TextRun) furnitureKey { + return furnitureKey{line: furnitureLine(r.Y), text: furnitureText(r.Text)} +} + +// Note says whether a run on a page is furniture, and in checkable terms why. +// +// The empty string means it is content. A nil receiver answers that for +// everything, which is what makes [RegionBlocks] work unchanged for a caller who +// has no document. +func (f *Furniture) Note(page int, r *TextRun) string { + if f == nil { + return "" + } + return f.notes[page][keyOf(r)] +} + +// Total is how many pieces of furniture were claimed, all three clauses together. +func (f *Furniture) Total() int { + if f == nil { + return 0 + } + return f.Tabs + f.Folios + f.Heads +} + +func (f *Furniture) mark(page int, k furnitureKey, note string) bool { + if f.notes[page] == nil { + f.notes[page] = make(map[furnitureKey]string, 4) + } + if _, seen := f.notes[page][k]; seen { + return false + } + f.notes[page][k] = note + return true +} + +// FindFurniture reads a whole document and says which of its runs are furniture. +// +// pages is the positioned text of every page, regions the language map, inScope +// the household's base languages — nil for every language, the same meaning +// [RegionsBlocks] gives it — and folios the page number each page prints, keyed +// on PDF page number, which is [Page.Folio] for the pages that print one. A nil +// or empty folios map costs the folio clause and nothing else: the tabs are found +// from the runs alone. +// +// The result is per document and is consulted per region, so it is computed once +// per conversion rather than once per page. Cost is one pass over the runs of the +// pages in scope, no tool spawned and nothing read twice. +func FindFurniture(pages []PageRuns, regions []Region, inScope map[string]bool, + folios map[int]int) *Furniture { + f := &Furniture{notes: make(map[int]map[furnitureKey]string, 16)} + + byPage := make(map[int]*PageRuns, len(pages)) + for i := range pages { + byPage[pages[i].No] = &pages[i] + } + + // Grouped by base language, the key ScopeFor, RegionChars and RegionsBlocks all + // use, so that a document printing CN, JA and ZH-HK counts one section and not + // three. A region whose language was never established is skipped outright: it + // has no section for a share to be a share of, and the two-page spread the + // column manual leaves unnamed is exactly the accident furnitureMinPages exists + // to refuse. + byLang := make(map[string][]int, 8) + for i := range regions { + base := BaseLanguage(regions[i].Lang) + if base == "" { + continue + } + if inScope != nil && !inScope[base] { + continue + } + byLang[base] = append(byLang[base], i) + } + + for _, langs := range sortedKeysOfSlices(byLang) { + f.findInSection(byPage, regions, byLang[langs], folios) + } + return f +} + +// findInSection applies both clauses to one language's section. +func (f *Furniture) findInSection(byPage map[int]*PageRuns, regions []Region, + idx []int, folios map[int]int) { + // Every usable run inside this language's regions, by page. A page is counted + // once however many regions of the language it holds, and a run once however + // many of them contain it — regions.md rule 3 stores a whole page of one + // language as one region, but nothing here may depend on that. + runsOn := make(map[int][]TextRun, len(idx)) + for _, i := range idx { + r := ®ions[i] + p := byPage[r.Page] + if p == nil { + continue + } + var dropped DroppedRuns + inside := runsInBox(usableRuns(p.Runs, p.Width, p.Height, &dropped), r.X0, r.X1) + seen := make(map[furnitureKey]bool, len(inside)) + for j := range runsOn[r.Page] { + seen[keyOf(&runsOn[r.Page][j])] = true + } + for j := range inside { + if k := keyOf(&inside[j]); !seen[k] { + seen[k] = true + runsOn[r.Page] = append(runsOn[r.Page], inside[j]) + } + } + } + total := len(runsOn) + if total < furnitureMinPages { + return + } + + // Clause 1: the same text at the same height. Counted in pages and not in runs, + // so that a page printing its tab twice is one page's worth of evidence. + repeats := make(map[furnitureKey]map[int]bool, 64) + // Clause 2: the heights at which a run agreeing with the page's printed folio + // was seen, and on which pages. + folioLines := make(map[int]map[int]bool, 4) + + for page, runs := range runsOn { + want, hasFolio := folios[page] + text := strconv.Itoa(want) + for i := range runs { + k := keyOf(&runs[i]) + if repeats[k] == nil { + repeats[k] = make(map[int]bool, total) + } + repeats[k][page] = true + if hasFolio && len([]rune(k.text)) <= maxFolioRunes && k.text == text { + if folioLines[k.line] == nil { + folioLines[k.line] = make(map[int]bool, total) + } + folioLines[k.line][page] = true + } + } + } + + need := int(math.Ceil(furnitureMinShare * float64(total))) + if need < furnitureMinPages { + need = furnitureMinPages + } + for k, pgs := range repeats { + if len(pgs) < need { + continue + } + note := fmt.Sprintf("page furniture: %q is printed at y=%.0f on %d of this "+ + "language's %d pages", k.text, float64(k.line)*furnitureYTolerance, len(pgs), total) + for page := range pgs { + if f.mark(page, k, note) { + f.Tabs++ + } + } + } + + for line, pgs := range folioLines { + if len(pgs) < furnitureMinPages { + continue + } + for page := range pgs { + k := furnitureKey{line: line, text: strconv.Itoa(folios[page])} + note := fmt.Sprintf("page furniture: the printed folio %q, at the y=%.0f where "+ + "this language prints one on %d of its %d pages", k.text, + float64(line)*furnitureYTolerance, len(pgs), total) + if f.mark(page, k, note) { + f.Folios++ + } + } + } + + f.runningHeads(runsOn) +} + +// runningHeads is clause 3, and it runs last because it reads what the first two +// wrote: the tab is above the head on some pages and below it on others, so "the +// page's first printed line" is only the head once the tab is out of the way. +// +// # The rule +// +// Take the pages of one language section in order. On each, take the first +// printed line — the runs sharing the topmost baseline, once the runs clauses 1 +// and 2 already claimed are removed. A page whose first line is identical to the +// first line of the PREVIOUS page of the section is printing a running head, and +// loses it. The first page of each such run keeps it. +// +// # Why consecutive, and why the first page of a run is kept +// +// This was blocked for one measured reason and the block is dissolved rather than +// argued away. The sequential manual's running head IS its section title, printed +// identically on the page where the sub-section starts and on every page after — +// Russian's "Меры предосторожности" sits at y=45-46, x=61-66, as the page's first +// line on pages 517, 518, 519 and 520 alike, and NOTHING ON THE PAGE distinguishes +// the first occurrence from the repeats. Every earlier attempt therefore had two +// choices, remove them all and lose the titles or keep them all and serve the +// defect, and picked the second. The sequence is the third choice: consecutive +// repetition has a first element even when no page does. +// +// Consecutive means consecutive in the SECTION's own page order, not in the PDF's. +// The column manual's German holds every even page, so its head runs 14-16-18-20-22 +// with the Polish pages between them, and a rule reading PDF adjacency would find +// no run at all. +// +// # The same place, expressed without a tolerance +// +// The text matching is not sufficient on its own, and the case that shows it is +// synthetic rather than hypothetical: a stock phrase that opens a note — "Hinweis:" +// does this on ten pages of the sequential manual's German section — is the page's +// first line whenever the note is what the page starts with, and it slides down the +// page as the note moves. TestFurnitureIsPositionalNotTextual is that document. +// +// So the two lines must also OVERLAP VERTICALLY: the band from the topmost run's +// top to the lowest run's bottom, on this page, must intersect the same band on the +// page before. That is a predicate and not a threshold, and it is scale-free — a +// head measures itself in its own type size, so nothing here has to be re-measured +// for a document set in a different one. +// +// Measured, it has all the room it needs and no more. Over both documents the +// matched head moves by 0 units on 141 page pairs, 1 on 35, and never more than 8, +// against a head 29 to 33 units tall in the sequential manual and 19 in the column +// manual — so every real head overlaps its predecessor with two thirds of its +// height to spare. The synthetic note moves 22.5 against a 17-unit line and misses +// entirely. +// +// # Why one line and not the matching prefix +// +// Measured over both documents, comparing each page of a section against the one +// before it and counting how far down the two agree: the prefix is 0 lines on 400 +// page pairs, 1 line on 207, and 2 lines on 38. Never 3, with the probe allowed to +// look 8 deep — so 2 is where the documents stop, not where a constant did. +// +// All 38 of those second lines are one thing: a troubleshooting table's column +// header, repeated where the table runs onto another page. "Problem Lösung", +// "Проблема Решение", "Ақау Шешім" — one or two per language section of the +// sequential manual, at y=88 to y=108 against a first line at y=46 to y=53. A +// reader on the continuation page wants those; they are the labels on the columns +// under them. So the clause stops at the first line. +// +// The two populations can be separated — by the gap between the head's bottom and +// the next line, 2 units against 13, or 0.11 of the head's own height against 0.45 +// — but that is a cut with one document on each side, which is exactly the shape +// of threshold that kept this clause unbuilt for so long. One line needs no +// threshold at all, and its cost is bounded in the safe direction: the worst it can +// do is leave a line it should have taken. +// +// # Why no share and no page floor +// +// Two consecutive pages leading with the identical line is the whole of the +// evidence, and it is enough because the claim it supports is small — one line off +// the second page, nothing off the first. Measured over both documents, the runs +// found are 2 to 6 pages long and every one of them is a chapter or section title +// verified against a 108 dpi render. The section still has to clear +// [furnitureMinPages] before any clause runs, which is where the short-section +// accidents were shown to live. +func (f *Furniture) runningHeads(runsOn map[int][]TextRun) { + order := make([]int, 0, len(runsOn)) + for page := range runsOn { + order = append(order, page) + } + sort.Ints(order) + + prevText := "" + var prevTop, prevBottom float64 + for _, page := range order { + cur := f.firstLine(page, runsOn[page]) + text := furnitureText(joinRuns(cur)) + top, bottom := vExtent(cur) + if text != "" && text == prevText && top < prevBottom && prevTop < bottom { + note := fmt.Sprintf("page furniture: the running head %q, printed as this page's "+ + "first line and as the first line of page %d before it", text, prevPage(order, page)) + marked := false + for i := range cur { + if f.mark(page, keyOf(&cur[i]), note) { + marked = true + } + } + if marked { + f.Heads++ + } + // The head this page printed is still what the next page must match: a run + // of five pages is five heads and not two. + } + prevText, prevTop, prevBottom = text, top, bottom + } +} + +// firstLine is the runs on a page's topmost printed baseline, once the runs the +// other clauses claimed are gone. +// +// A line and not a run, because a head can be set in pieces: the column manual +// puts its tab and its chapter name on one baseline, and with the tab claimed the +// name may still arrive as more than one run. +func (f *Furniture) firstLine(page int, runs []TextRun) []TextRun { + free := make([]TextRun, 0, len(runs)) + for i := range runs { + if f.Note(page, &runs[i]) == "" { + free = append(free, runs[i]) + } + } + if len(free) == 0 { + return nil + } + sort.SliceStable(free, func(i, j int) bool { + if free[i].Y != free[j].Y { + return free[i].Y < free[j].Y + } + return free[i].X < free[j].X + }) + tol := baselineToleranceFraction * medianHeight(free) + n := 1 + for n < len(free) && sameBaseline(free[0].Y, free[n].Y, tol) { + n++ + } + return free[:n] +} + +// vExtent is the vertical band a line of runs occupies. +func vExtent(runs []TextRun) (top, bottom float64) { + if len(runs) == 0 { + return 0, 0 + } + top, bottom = math.Inf(1), math.Inf(-1) + for i := range runs { + top = math.Min(top, runs[i].Y) + bottom = math.Max(bottom, runs[i].bottom()) + } + return top, bottom +} + +// prevPage is the page before p in an ordered slice that contains it, for a note +// that has to name it. +func prevPage(order []int, p int) int { + for i := range order { + if order[i] == p && i > 0 { + return order[i-1] + } + } + return 0 +} + +// splitFurniture divides a region's runs into content and furniture. +// +// Both slices keep the order they arrived in. The furniture runs are taken out +// BEFORE anything is measured or grouped, and that is the point of doing this at +// run level rather than on the finished blocks: the tab is not always a block of +// its own. The column manual sets it on the same baseline as the chapter head, so +// page 14 arrives as one heading reading "D Trockensaugen" and page 57 as +// "D Fehlerbehebung"; the sequential manual sets it under the running head close +// enough to join it, so pages 34 and 35 arrive as "Fehlersuche DE" with the tab +// at the END. Removing a whole block is wrong on all four, and taking two letters +// off the front of a block's text is wrong on two of them and unsafe on the +// others — the sequential manual heads 28 pages with a bare "A" and 22 with a +// bare "D", so a rule that eats a leading capital eats a real section title. Taken +// out as a run, the tab is simply not there when the line is assembled, and the +// heading that remains is the heading that was printed. +func splitFurniture(runs []TextRun, page int, f *Furniture) (content, furniture []TextRun) { + if f == nil { + return runs, nil + } + if _, onPage := f.notes[page]; !onPage { + return runs, nil + } + content = make([]TextRun, 0, len(runs)) + for i := range runs { + if f.Note(page, &runs[i]) != "" { + furniture = append(furniture, runs[i]) + continue + } + content = append(content, runs[i]) + } + return content, furniture +} + +// furnitureBlocks turns a region's furniture runs into blocks. +// +// One block per printed line, which is what furniture is: runs on one baseline +// carrying the same note are one thing the page prints. Two furniture items that +// share a baseline stay apart — the column manual would otherwise join a tab to a +// chapter head that is not furniture, and nothing here may reunite what +// [splitFurniture] separated — so the note is part of the grouping and not only +// of the result. +// +// Every block is a [BlockParagraph] whatever the type it is set in, and that is +// deliberate. A kind is a reading decision — "this line titles what follows" — +// and a line that is on the page because of where the page is titles nothing. The +// tab classified as a heading is the defect, not a fact worth carrying forward, +// and the classification is not even stable: measured over the sequential +// manual's German section, the same tab came back as a level-2 heading on 11 of +// its 16 pages, a level-1 heading on 3 and part of a paragraph on 2, because what +// it is compared against is whatever else that page happens to set. +// +// Note while you are here that conversion.md, blocks.go, figures.go and +// verify/order.go all say this tab is on "110 pages". It is not. Measured over +// the sequential manual: a tab-shaped run sits near the top of 556 of its 560 +// pages, 553 of them inside a language region this pass reads, and the 34 sections +// print one on every page they have. 110 is not a page count of anything here — +// even the x=27-41 band order.go names holds 263 of them, because the tab is set +// against a margin and its left edge moves with the width of the code. +func furnitureBlocks(runs []TextRun, r *Region, f *Furniture, from int) []Block { + if len(runs) == 0 { + return nil + } + ordered := make([]TextRun, len(runs)) + copy(ordered, runs) + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].Y != ordered[j].Y { + return ordered[i].Y < ordered[j].Y + } + return ordered[i].X < ordered[j].X + }) + + tol := baselineToleranceFraction * medianHeight(ordered) + var out []Block + var cur []TextRun + var curNote string + + flush := func() { + if len(cur) == 0 { + return + } + // Right to left is repaired here for the same reason it is repaired in + // [textLine.finish], and it became load-bearing when clause 3 arrived. While + // furniture was a two-letter tab and a number, joining the runs left to right + // could not be wrong: "HE" reads the same either way. A running head is a + // SENTENCE, and the sequential manual has a Hebrew and an Arabic section, so + // joining those left to right stores them backwards — internal/verify's + // `right-to-left-reversed` check went from 0 pages to 10 on the first version + // of this clause and named the words. Furniture reaches neither a reader nor + // the index, so nothing would have read them; that is not a reason to write + // them down wrong. + text := joinRuns(cur) + if lineIsRightToLeft(cur, IsRightToLeftLanguage(r.Lang)) { + text = joinRunsRightToLeft(cur) + } + b := Block{ + Page: r.Page, RegionX0: r.X0, Index: from + len(out), + Kind: BlockParagraph, Lang: r.Lang, Furniture: true, Note: curNote, + Text: collapseSpaces(text), Lines: 1, + X0: math.Inf(1), X1: math.Inf(-1), Y0: math.Inf(1), Y1: math.Inf(-1), + } + for i := range cur { + b.X0 = math.Min(b.X0, cur[i].X) + b.X1 = math.Max(b.X1, cur[i].right()) + b.Y0 = math.Min(b.Y0, cur[i].Y) + b.Y1 = math.Max(b.Y1, cur[i].bottom()) + } + b.Chars = len([]rune(b.Text)) + if b.Chars > 0 { + out = append(out, b) + } + cur, curNote = nil, "" + } + + for i := range ordered { + note := f.Note(r.Page, &ordered[i]) + if len(cur) > 0 && (note != curNote || !sameBaseline(cur[0].Y, ordered[i].Y, tol)) { + flush() + } + cur, curNote = append(cur, ordered[i]), note + } + flush() + return out +} + +// sortedKeysOfSlices returns a map's keys in order, so that a document converts +// identically twice. Same reason [sortedKeys] exists; a separate function because +// Go 1.25 will not let one body serve two map value types without generics that +// buy nothing here. +func sortedKeysOfSlices(m map[string][]int) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// FoliosOf collects the page numbers a document's pages print, in the shape +// [FindFurniture] wants them. +// +// Separate from [FindFurniture] so that a test can state the folios it means +// without building a [Result], and so that the one place that knows where they +// come from is here rather than in [Convert]. +func FoliosOf(pages []Page) map[int]int { + if len(pages) == 0 { + return nil + } + out := make(map[int]int, len(pages)) + for i := range pages { + if pages[i].Folio != nil { + out[pages[i].No] = *pages[i].Folio + } + } + return out +} diff --git a/internal/doc/furniture_fixture_test.go b/internal/doc/furniture_fixture_test.go new file mode 100644 index 0000000..5dcacc6 --- /dev/null +++ b/internal/doc/furniture_fixture_test.go @@ -0,0 +1,479 @@ +package doc_test + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + "unicode" + + "github.com/gordon2/manualbox/internal/doc" +) + +// The furniture pass against both real manuals. Everything asserted here was +// measured by running it; the counts are quoted at each assertion together with +// what the number was before the pass existed, so that a change to the rule shows +// up as a moved number and not as a silent improvement. + +// wholeDocumentFurniture probes a fixture and reads every region of it for every +// language, which is what the pass needs: a share is a share of a language's own +// pages, and reading one household's languages would hide the other sections. +func wholeDocumentFurniture(t *testing.T, name string) ([]doc.Block, *doc.Furniture) { + t.Helper() + var path string + if name == "thomas-drybox-amfibia" { + _, path = columnFixture(t) + } else { + _, path = loadFixture(t) + } + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + pages, err := doc.ExtractRuns(context.Background(), path) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + fur := doc.FindFurniture(pages, res.Regions, nil, doc.FoliosOf(res.Pages)) + return doc.RegionsBlocks(pages, res.Regions, nil, nil, fur), fur +} + +// TestFurnitureClaimsOnlyTabsFoliosAndHeadsOnBothManuals is the exhaustive +// false-positive check, and it is the assertion that matters most: over all 628 +// pages of both documents and all 39 language sections, EVERY block the rule +// claims is a printed language tab, a bare number, or a running head — and the +// tabs and the numbers are still named exhaustively. Not a sample: every one. +// +// Measured. The column manual: 172 blocks, being "D" on 26 of German's 26 pages, +// "PL" on 22 of Polish's 27, "UA" on 22 of Ukrainian's 26, 41 folios, and 61 +// running heads. The three tab shares are 1.00, 0.81 and 0.85, and the two below 1 +// are not the tab being absent — they are usableRuns dropping it as sub-legible on +// the pages whose median run is a heading's. The sequential manual: 1,289, being +// its 34 tabs on every page of every section (553 in all), 552 folios, and 184 +// running heads. +// +// Clause 3 gets no list of strings here, because the list would be 97 chapter and +// section titles in 39 languages and would assert only that they had been copied +// out of a previous run. What holds clause 3 is the invariant, in +// [TestFurnitureKeepsEveryTitleItClaims]. +func TestFurnitureClaimsOnlyTabsFoliosAndHeadsOnBothManuals(t *testing.T) { + for _, tc := range []struct { + name string + blocks int + tabs, folios, heads int + wantTabStrings []string + }{ + { + name: "thomas-drybox-amfibia", blocks: 172, tabs: 70, folios: 41, heads: 61, + wantTabStrings: []string{"D", "PL", "UA"}, + }, + { + name: "dreame-l40-ultra", blocks: 1289, tabs: 553, folios: 552, heads: 184, + // Every code the manual prints in its corner, including the two it prints + // non-canonically: CZ for Czech and UA for Ukrainian. + wantTabStrings: []string{"AR", "CZ", "DA", "DE", "EL", "EN", "ES", "FI", "FR", + "HE", "HU", "ID", "IT", "JA", "KK", "LT", "LV", "MS", "NL", "NO", "PL", + "PT", "RO", "RU", "SK", "SL", "SR", "SV", "TH", "TR", "UA", "UZ", "VI", + "ZH-HK"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + blocks, fur := wholeDocumentFurniture(t, tc.name) + if fur.Tabs != tc.tabs || fur.Folios != tc.folios || fur.Heads != tc.heads { + t.Errorf("claimed %d tab run(s), %d folio(s) and %d head(s), was %d, %d and %d", + fur.Tabs, fur.Folios, fur.Heads, tc.tabs, tc.folios, tc.heads) + } + + tabs := map[string]int{} + numeric, heads := 0, 0 + for i := range blocks { + b := &blocks[i] + if !b.Furniture { + continue + } + switch { + case isRunningHead(b): + heads++ + case allDigits(b.Text): + numeric++ + default: + tabs[b.Text]++ + } + } + total := numeric + heads + for _, n := range tabs { + total += n + } + if total != tc.blocks || heads != tc.heads { + t.Errorf("%d furniture block(s) of which %d head(s), was %d and %d", + total, heads, tc.blocks, tc.heads) + } + + // The whole point: apart from the heads, nothing but a tab and a number. + got := make([]string, 0, len(tabs)) + for s := range tabs { + got = append(got, s) + } + sort.Strings(got) + want := append([]string(nil), tc.wantTabStrings...) + sort.Strings(want) + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("the furniture that is neither a number nor a head is %v\nwant %v", got, want) + } + t.Logf("%s: %d furniture blocks — %d numeric, %d heads, %d tabs over %d distinct codes", + tc.name, total, numeric, heads, total-numeric-heads, len(tabs)) + }) + } +} + +// isRunningHead reports whether a furniture block was claimed by clause 3. The +// note is the only thing that records which clause claimed a block, which is what +// [doc.Furniture.Note] is for. +func isRunningHead(b *doc.Block) bool { + return b.Furniture && strings.Contains(b.Note, "the running head") +} + +// TestFurnitureKeepsEveryTitleItClaims is clause 3's false-positive check, and it +// is an invariant rather than a list of strings for the reason given above. +// +// The invariant is the whole promise of the clause. A running head is claimed only +// where the page BEFORE it in the same section printed the same line, so the first +// page of every run keeps it — which means every string clause 3 removes must +// still be served as content somewhere in the same language. If one is not, a title +// was deleted, and that is precisely the failure that kept this clause unbuilt. +// +// Measured: 61 claims on the column manual over 20 distinct titles, 184 on the +// sequential manual over 77, and 0 of the 245 has no surviving content copy. +// +// The comparison is exact, and it was not always able to be. Furniture blocks were +// joined left to right whatever the language, which is harmless for a tab reading +// "HE" and wrong for a Hebrew or Arabic sentence: the first version of clause 3 +// stored 10 of these heads backwards, so this test had to accept a reversed match +// and internal/verify's `right-to-left-reversed` check went from 0 pages to 10. +// [doc.FindFurniture] repairs the direction now and both are exact again. +func TestFurnitureKeepsEveryTitleItClaims(t *testing.T) { + for _, tc := range []struct { + name string + heads, titles int + }{ + {name: "thomas-drybox-amfibia", heads: 61, titles: 20}, + {name: "dreame-l40-ultra", heads: 184, titles: 77}, + } { + t.Run(tc.name, func(t *testing.T) { + blocks, _ := wholeDocumentFurniture(t, tc.name) + + content := map[string]map[string]bool{} + for i := range blocks { + b := &blocks[i] + if b.Furniture { + continue + } + if content[b.Lang] == nil { + content[b.Lang] = map[string]bool{} + } + content[b.Lang][b.Text] = true + } + + titles := map[string]bool{} + heads, lost := 0, 0 + for i := range blocks { + b := &blocks[i] + if !isRunningHead(b) { + continue + } + heads++ + titles[b.Text] = true + if content[b.Lang][b.Text] { + continue + } + lost++ + t.Errorf("page %d: %q was claimed as %s's running head and is served nowhere "+ + "in that language as content — a title was deleted", b.Page, b.Text, b.Lang) + } + if heads != tc.heads || len(titles) != tc.titles { + t.Errorf("%d head(s) over %d distinct title(s), was %d and %d", + heads, len(titles), tc.heads, tc.titles) + } + t.Logf("%s: %d head(s) over %d distinct title(s), %d with no surviving content copy", + tc.name, heads, len(titles), lost) + }) + } +} + +func allDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if !unicode.IsDigit(r) { + return false + } + } + return true +} + +// TestFurnitureKeepsTheSequentialManualsLetteredSections is the false positive +// the rule is arranged to avoid, held against the document that contains it. +// +// The manual labels the parts of its product overview "A" to "E", set in 15pt, one +// letter per section per page — measured: A on 31 pages of the document, B on 30, +// C on 30, D on 30, E on 30, which is once or twice in each of the 34 sections. +// Those are the pages that make "a one-letter line near the top is a tab" false, +// and D in particular sits at x=59 y=56 on the German section's page 29, two units +// from where that section prints its own "DE" tab. +func TestFurnitureKeepsTheSequentialManualsLetteredSections(t *testing.T) { + blocks, _ := wholeDocumentFurniture(t, "dreame-l40-ultra") + + for _, letter := range []string{"A", "B", "C", "D", "E"} { + content, furniture := 0, 0 + for i := range blocks { + if !hasBareWord(blocks[i].Text, letter) { + continue + } + if blocks[i].Furniture { + furniture++ + t.Errorf("page %d: the section letter %q was claimed as furniture in %q", + blocks[i].Page, letter, truncate(blocks[i].Text, 60)) + continue + } + content++ + } + if content < 20 { + t.Errorf("the section letter %q survives in %d content block(s); it is printed "+ + "in 30 of this manual's sections", letter, content) + } + t.Logf("%q: %d content block(s), %d claimed as furniture", letter, content, furniture) + } +} + +// hasBareWord reports whether s contains word as a standalone token. +func hasBareWord(s, word string) bool { + for _, f := range strings.FieldsFunc(s, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + if f == word { + return true + } + } + return false +} + +// TestFurnitureOnTheColumnManualsGluedPages is the case that put this pass beside +// [doc.Convert] rather than inside RegionBlocks. The column manual sets its "D" +// tab on the SAME baseline as the running head, so before the pass page 14's first +// block read "D Trockensaugen" and page 57's "D Fehlerbehebung" — one block each, +// folded from one printed line. The tab could not be removed as a block, and +// stripping it from the front of the text would be a rule that eats a real word. +// +// Both were read against a 108 dpi render while this was written. What remains on +// page 14 is the chapter head the paper prints — "Trockensaugen" in the grey +// banner, where the chapter starts — and clause 3 now takes it off pages 16, 18, +// 20 and 22, which are the pages that only continue it. +func TestFurnitureOnTheColumnManualsGluedPages(t *testing.T) { + conv := convertFixture(t, "thomas-drybox-amfibia", "de") + + // The funnel is unmoved: the pass reads no page the gate did not charge for. + if len(conv.Pages) != 26 { + t.Errorf("converted %d pages, the gate charges this household for 26", len(conv.Pages)) + } + + // 432 blocks before the pass; 427 content and 33 furniture after it. The content + // falls by 5 and not by 33 because 28 of the 33 were already blocks of their own + // and the other 5 were glued into a block that survives without them. + // + // 443 since the contents page came apart: its 17 printed entries were one + // run-together block of dot leaders, and each is now its own, which is +16 on the + // one page of this section that has a table of contents. + // + // 431 and 45 since clause 3: German's four chapter heads are printed on 16 of its + // 26 pages and the first page of each of the four runs keeps its own, so 12 move + // from content to furniture and the two totals move by 12 in opposite directions. + // + // 460 since a page with no second column is read as two strips. The furniture does + // not move at all — this is text that was always content and was welded into the + // wrong blocks. It is +29 on two pages: page 11's parts list arrived as two + // run-together blocks with the diagram's callouts spliced mid-sentence + // ("17 Staubbehälter für Grobschmutz und Feinstaub 7 18 Saugschlauch*…") and is + // now its own list, and page 57's troubleshooting table gets its own header rather + // than one welded from both columns'. + // + // Higher is better here and the failure message says so, because the number alone + // cannot: read the sequence, not the value. + content, furniture := len(conv.ContentBlocks()), len(conv.FurnitureBlocks()) + if content != 460 || furniture != 45 { + t.Errorf("%d content and %d furniture blocks, was 460 and 45 — and MORE content "+ + "here has meant better every time so far, since these are blocks that were "+ + "welded rather than missing (431 and 45 before two columns were read as two, "+ + "443 and 33 before the running-head clause, 427 before the contents page "+ + "came apart, 432 before the furniture pass)", content, furniture) + } + if conv.Furniture.Tabs != 26 || conv.Furniture.Folios != 7 || conv.Furniture.Heads != 12 { + t.Errorf("claimed %d tab(s), %d folio(s) and %d head(s) in German, was 26, 7 and 12", + conv.Furniture.Tabs, conv.Furniture.Folios, conv.Furniture.Heads) + } + + for _, tc := range []struct { + page int + want string + }{ + {page: 14, want: "Trockensaugen"}, + {page: 57, want: "Fehlerbehebung"}, + } { + first := "" + for _, b := range conv.ContentBlocks() { + if b.Page == tc.page { + first = b.Text + break + } + } + if first != tc.want { + t.Errorf("page %d's first content block is %q, want %q — the tab was %q", + tc.page, first, tc.want, "D "+tc.want) + } + } + + // Nothing anywhere in the German conversion still serves the tab as content. + for _, b := range conv.ContentBlocks() { + if b.Text == "D" || strings.HasPrefix(b.Text, "D ") && len(b.Text) < 30 { + t.Errorf("page %d block %d still reads %q", b.Page, b.Index, b.Text) + } + } +} + +// TestFurnitureOnTheSequentialManualsPage24 is the page conversion.md compares +// against a render bullet for bullet. It arrived as 15 blocks, the two extra being +// the documented furniture — the tab as a level-2 heading and the folio "18" as a +// paragraph. +// +// The reading it was pinned to, "one heading and 12 list items", was itself the +// defect and clause 3 is what showed it. Page 23 opens Sicherheitshinweise with an +// introduction and a sub-heading under the title; page 24 prints the same title at +// the same place and then 12 more bullets, and there is no new section on it. Both +// pages were re-read at 108 dpi. So the heading on page 24 is a running head, and +// what is printed on page 24 is 12 list items — the title being served here was a +// third piece of furniture that had gone unnoticed because it is a real word. +// +// Page 23 keeps its Sicherheitshinweise, and [TestFurnitureKeepsEveryTitleItClaims] +// is what holds that for every title in both documents. +func TestFurnitureOnTheSequentialManualsPage24(t *testing.T) { + conv := convertFixture(t, "dreame-l40-ultra", "de") + + if conv.Furniture.Tabs != 16 || conv.Furniture.Folios != 16 || conv.Furniture.Heads != 5 { + t.Errorf("claimed %d tab(s), %d folio(s) and %d head(s) over German's 16 pages, "+ + "was 16, 16 and 5", conv.Furniture.Tabs, conv.Furniture.Folios, conv.Furniture.Heads) + } + // 481 blocks before the pass; 453 content and 32 furniture after it, 448 and 37 + // since clause 3 moved German's five repeated section titles across, and 449 since + // a page with no second column is read as two strips — one block on one page, this + // section being mostly single-column prose where the columns manual is not. + if content, furniture := len(conv.ContentBlocks()), len(conv.FurnitureBlocks()); content != 449 || + furniture != 37 { + t.Errorf("%d content and %d furniture blocks, was 449 and 37 (448 before two "+ + "columns were read as two, 453 and 32 before the running-head clause, 481 "+ + "before the pass)", content, furniture) + } + + var kinds []string + var furniture []string + for i := range conv.Blocks { + b := &conv.Blocks[i] + if b.Page != 24 { + continue + } + if b.Furniture { + furniture = append(furniture, b.Text) + continue + } + kinds = append(kinds, fmt.Sprintf("%s%d", b.Kind, b.Level)) + } + var want []string + for i := 0; i < 12; i++ { + want = append(want, "list-item0") + } + if strings.Join(kinds, " ") != strings.Join(want, " ") { + t.Errorf("page 24's content is\n %v\nwant the 12 list items and nothing else\n %v", + kinds, want) + } + if strings.Join(furniture, "|") != "Sicherheitshinweise|DE|18" { + t.Errorf("page 24's furniture is %v, want the running head, the tab and the folio 18", + furniture) + } + + // The title is not lost: page 23 is where the section starts and keeps it. + first := "" + for _, b := range conv.ContentBlocks() { + if b.Page == 23 { + first = b.Text + break + } + } + if first != "Sicherheitshinweise" { + t.Errorf("page 23's first content block is %q, want the section title clause 3 "+ + "took off page 24", first) + } +} + +// TestFurnitureThresholdSweepOnBothManuals prints the measurement behind +// furnitureMinShare, over every language section of both documents: the share of a +// section's pages held by its most-repeated line, and the share held by the most +// repeated line that is NOT the tab. The two populations are what the constant +// sits between, and a document that closed the gap would show up here. +// +// It counts clause 1's claims only. A head is not a tab and shares the denominator +// without sharing the rule, and counting both put German on the column manual at +// 1.46 and its Kazakh at 0.46 — a share above 1 being the tell that two rules were +// being read as one. +func TestFurnitureThresholdSweepOnBothManuals(t *testing.T) { + for _, name := range []string{"thomas-drybox-amfibia", "dreame-l40-ultra"} { + blocks, _ := wholeDocumentFurniture(t, name) + // Pages per language, and the tab blocks per language, which is the numerator + // clause 1 used. + pages := map[string]map[int]bool{} + claimed := map[string]int{} + for i := range blocks { + b := &blocks[i] + if pages[b.Lang] == nil { + pages[b.Lang] = map[int]bool{} + } + pages[b.Lang][b.Page] = true + if b.Furniture && !allDigits(b.Text) && !isRunningHead(b) { + claimed[b.Lang]++ + } + } + langs := make([]string, 0, len(pages)) + for l := range pages { + langs = append(langs, l) + } + sort.Strings(langs) + low, high := 1.0, 0.0 + for _, l := range langs { + n := len(pages[l]) + if n == 0 || claimed[l] == 0 { + continue + } + share := float64(claimed[l]) / float64(n) + if share < low { + low = share + } + if share > high { + high = share + } + } + t.Logf("%s: the tab is on %.2f to %.2f of its language's pages over %d section(s) "+ + "that print one; the cut is %.2f", name, low, high, countClaimed(claimed), 0.5) + if low < 0.5 { + t.Errorf("%s: a claimed tab sits at %.2f, under the cut it had to pass", name, low) + } + } +} + +func countClaimed(m map[string]int) int { + n := 0 + for _, v := range m { + if v > 0 { + n++ + } + } + return n +} diff --git a/internal/doc/furniture_test.go b/internal/doc/furniture_test.go new file mode 100644 index 0000000..85c18d9 --- /dev/null +++ b/internal/doc/furniture_test.go @@ -0,0 +1,635 @@ +package doc_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for the furniture pass. No PDF and no poppler: the pages are +// built here, so each clause and each threshold is stated where it can be read +// against the reasoning in furniture.go. Every shape below is one the two real +// manuals actually print, and the fixture tests in furniture_fixture_test.go hold +// the same rules against them. +// +// The geometry is the sequential manual's, measured: a 918x620 page setting its +// tab at x=28 y=58 in 11pt, its running head at x=55 y=52 in 21pt, its body at +// x=55 from y=95 on a 22.5-unit pitch, and its folio at x=55 y=576 in 12pt. + +// furnitureSection builds one language's section: n pages, each carrying the +// lines build returns for it, and one whole-page region per page. +func furnitureSection(n int, lang string, build func(page, i int) []line) ([]doc.PageRuns, []doc.Region) { + pages := make([]doc.PageRuns, 0, n) + regions := make([]doc.Region, 0, n) + for i := 0; i < n; i++ { + no := 23 + i + pages = append(pages, *blockPage(no, build(no, i)...)) + regions = append(regions, doc.Region{ + Page: no, X0: 0, X1: testBlockPageWidth, Lang: lang, Source: doc.SourceRepertoire, + }) + } + return pages, regions +} + +// tabLine is the sequential manual's language tab: 11pt medium in the top-left +// margin, at the same y on every page of a section. +func tabLine(text string) line { + return line{y: 58, x: 28, w: 14, size: 11, weight: doc.WeightMedium, text: text} +} + +// headLine is a 21pt semibold heading across the top of the measure. +func headLine(text string) line { + return line{y: 52, x: 55, w: 202, size: 21, weight: doc.WeightSemibold, bold: true, text: text} +} + +// folioLine is the printed page number in the footer. +func folioLine(text string) line { + return line{y: 576, x: 55, w: 11, size: 12, text: text} +} + +func blockTextsOf(blocks []doc.Block) []string { + out := make([]string, len(blocks)) + for i := range blocks { + out[i] = blocks[i].Text + } + return out +} + +func furnitureOf(blocks []doc.Block) []string { + var out []string + for i := range blocks { + if blocks[i].Furniture { + out = append(out, blocks[i].Text) + } + } + return out +} + +func contentOf(blocks []doc.Block) []string { + var out []string + for i := range blocks { + if !blocks[i].Furniture { + out = append(out, blocks[i].Text) + } + } + return out +} + +// TestFurnitureFindsATabOnEveryPage is clause 1 at its plainest: the same two +// letters at the same height on all 16 pages of a section, which is what the +// sequential manual prints on all 34 of its sections. +func TestFurnitureFindsATabOnEveryPage(t *testing.T) { + pages, regions := furnitureSection(16, "de", func(page, i int) []line { + lines := []line{tabLine("DE"), headLine(fmt.Sprintf("Kapitel %d", i))} + return append(lines, bodyLines(95, 22.5, 6, fmt.Sprintf("Absatz auf Seite %d", page))...) + }) + + fur := doc.FindFurniture(pages, regions, nil, nil) + if fur.Tabs != 16 { + t.Errorf("claimed %d tab run(s) over 16 pages printing one each", fur.Tabs) + } + if fur.Folios != 0 { + t.Errorf("claimed %d folio(s) where no folios were supplied", fur.Folios) + } + + blocks := doc.RegionsBlocks(pages, regions, nil, nil, fur) + got := furnitureOf(blocks) + if len(got) != 16 { + t.Fatalf("%d furniture block(s), want 16: %v", len(got), got) + } + for _, s := range got { + if s != "DE" { + t.Errorf("furniture block reads %q, want the tab", s) + } + } + for i := range blocks { + if strings.Contains(blocks[i].Text, "DE") && !blocks[i].Furniture { + t.Errorf("the tab reached a content block: %q", blocks[i].Text) + } + } +} + +// TestFurnitureIsLastInItsRegionAndKeepsContentContiguous pins the ordering +// decision RegionBlocks records: content keeps 0..n-1 so that "paragraph 4 of the +// German region of page 62" means the fourth paragraph a reader sees. +func TestFurnitureIsLastInItsRegionAndKeepsContentContiguous(t *testing.T) { + pages, regions := furnitureSection(8, "de", func(page, i int) []line { + lines := []line{tabLine("DE"), headLine(fmt.Sprintf("Kapitel %d", i))} + lines = append(lines, bodyLines(95, 22.5, 3, fmt.Sprintf("Erster Absatz %d", page))...) + return append(lines, folioLine(fmt.Sprintf("%d", page-6))) + }) + folios := map[int]int{} + for i := 0; i < 8; i++ { + folios[23+i] = 23 + i - 6 + } + + fur := doc.FindFurniture(pages, regions, nil, folios) + blocks := doc.RegionsBlocks(pages, regions, nil, nil, fur) + + perPage := map[int][]doc.Block{} + for i := range blocks { + perPage[blocks[i].Page] = append(perPage[blocks[i].Page], blocks[i]) + } + for page, bs := range perPage { + seenFurniture := false + for i := range bs { + if bs[i].Index != i { + t.Errorf("page %d block %d carries index %d", page, i, bs[i].Index) + } + if bs[i].Furniture { + seenFurniture = true + continue + } + if seenFurniture { + t.Errorf("page %d: content block %q sits after furniture", page, bs[i].Text) + } + } + if !seenFurniture { + t.Errorf("page %d produced no furniture at all", page) + } + } +} + +// TestFurnitureUnGluesATabSetOnAHeadingsBaseline is the case that decides where +// this pass belongs. The column manual sets its tab on the SAME baseline as the +// running head, so the tab is not a block of its own: page 14 arrives as one +// heading reading "D Trockensaugen". Removing a block is wrong; taking two +// characters off the front of the text is unsafe. Removing the RUN is neither. +func TestFurnitureUnGluesATabSetOnAHeadingsBaseline(t *testing.T) { + heads := []string{"Trockensaugen", "Waschsaugen", "Wartung", "Fehlerbehebung", + "Trockensaugen", "Waschsaugen"} + pages, regions := furnitureSection(6, "de", func(page, i int) []line { + lines := []line{ + {y: 16, x: 340, w: 10, size: 11, weight: doc.WeightMedium, text: "D"}, + {y: 16, x: 380, w: 120, size: 17, weight: doc.WeightMedium, text: heads[i]}, + } + return append(lines, bodyLines(95, 16, 5, fmt.Sprintf("Absatz auf Seite %d", page))...) + }) + + fur := doc.FindFurniture(pages, regions, nil, nil) + if fur.Tabs != 6 { + t.Fatalf("claimed %d tab run(s) over 6 glued pages", fur.Tabs) + } + + blocks := doc.RegionsBlocks(pages, regions, nil, nil, fur) + for i := range blocks { + b := &blocks[i] + if b.Furniture { + if b.Text != "D" { + t.Errorf("page %d furniture reads %q, want the bare tab", b.Page, b.Text) + } + continue + } + if strings.HasPrefix(b.Text, "D ") { + t.Errorf("page %d still serves the tab glued to content: %q", b.Page, b.Text) + } + } + // And what remains on the first page is the head the printer set, not a + // substring of it. + first := contentOf(blocks) + if len(first) == 0 || first[0] != "Trockensaugen" { + t.Errorf("page 23's first content block is %q, want %q", first[0], "Trockensaugen") + } +} + +// TestFurnitureUnGluesATabThatJoinedTheBlockBelow is the same defect with the tab +// at the END. The sequential manual's pages 34 and 35 set the running head at +// y=52 and the tab at y=58, close enough for the paragraph rule to fold them into +// one block reading "Fehlersuche DE" — so a rule that strips a leading token +// would leave both of those untouched. +func TestFurnitureUnGluesATabThatJoinedTheBlockBelow(t *testing.T) { + pages, regions := furnitureSection(6, "de", func(page, i int) []line { + lines := []line{ + {y: 52, x: 28, w: 150, size: 17, weight: doc.WeightMedium, + text: fmt.Sprintf("Fehlersuche %d", i)}, + tabLine("DE"), + } + return append(lines, bodyLines(120, 22.5, 5, fmt.Sprintf("Absatz %d", page))...) + }) + + before := doc.RegionsBlocks(pages, regions, nil, nil, nil) + glued := 0 + for i := range before { + if strings.Contains(before[i].Text, " DE") { + glued++ + } + } + if glued != 6 { + t.Fatalf("the fixture does not reproduce the glued shape: %d of 6 pages, %v", + glued, blockTextsOf(before)[:3]) + } + + fur := doc.FindFurniture(pages, regions, nil, nil) + after := doc.RegionsBlocks(pages, regions, nil, nil, fur) + for i := range after { + b := &after[i] + if !b.Furniture && strings.Contains(b.Text, "DE") { + t.Errorf("page %d still serves the tab as content: %q", b.Page, b.Text) + } + if !b.Furniture && strings.HasPrefix(b.Text, "Fehlersuche") && + b.Text != fmt.Sprintf("Fehlersuche %d", b.Page-23) { + t.Errorf("page %d's running head came back as %q", b.Page, b.Text) + } + } + if got := len(furnitureOf(after)); got != 6 { + t.Errorf("%d furniture block(s) over 6 pages", got) + } +} + +// TestFurnitureClaimsARunningHeadThatNeverChanges pins that clause 1 is stated +// generally on purpose, and it is the one place a caller can be surprised: a +// section printing the SAME running head at the same height on most of its pages +// loses it, tab or not. Neither fixture does that — the column manual's head +// names the chapter and changes every few pages, and the sequential manual's +// names the section and repeats on at most 4 of 16 — so the behaviour is +// asserted here rather than measured there. +// +// It was found by accident, by a version of the test above that put one heading +// on all 16 pages, and it is the correct reading: a line the printer set on every +// page of a section because of where the page is IS furniture, and the fact that +// this one happens to be words rather than two letters changes nothing about the +// evidence. +func TestFurnitureClaimsARunningHeadThatNeverChanges(t *testing.T) { + pages, regions := furnitureSection(10, "de", func(page, i int) []line { + lines := []line{headLine("Sicherheitshinweise")} + return append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz %d", page))...) + }) + fur := doc.FindFurniture(pages, regions, nil, nil) + if fur.Tabs != 10 { + t.Errorf("claimed %d run(s) for a head printed identically on all 10 pages", fur.Tabs) + } + blocks := doc.RegionsBlocks(pages, regions, nil, nil, fur) + for _, s := range furnitureOf(blocks) { + if s != "Sicherheitshinweise" { + t.Errorf("furniture block reads %q", s) + } + } +} + +// TestFurnitureRestartsARunWhenTheHeadComesBack is the one property of clause 3 +// that NEITHER real manual can hold, and it is here because a mutation test said +// so: relaxing "the page before it" to "any earlier page of the section" changed +// nothing on 628 real pages and was caught by no test at all. Every repeated head +// in both fixtures is one unbroken run, so the two rules agree on both documents. +// +// A manual that returns to a chapter does not. This section heads pages 23 and 24 +// "Wartung", page 25 "Fehlersuche", then pages 26 and 27 "Wartung" again. Page 26 +// is the second run's FIRST page and must keep its title — a reader arriving there +// after a different chapter needs to be told which chapter they are back in — +// while pages 24 and 27 lose theirs. Under the relaxed rule page 26 would lose it +// to page 24 and the section would read as though maintenance never resumed. +// +// Re-run against that mutant afterwards: this test fails and the other nine — +// including both whole-document fixture tests and both verify runs — all pass. It +// is the only thing standing between "consecutive" and "ever". +func TestFurnitureRestartsARunWhenTheHeadComesBack(t *testing.T) { + titles := []string{"Wartung", "Wartung", "Fehlersuche", "Wartung", "Wartung", + "Zubehör", "Entsorgung", "Garantie", "Technische Daten"} + pages, regions := furnitureSection(len(titles), "de", func(page, i int) []line { + lines := []line{headLine(titles[i])} + return append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz auf Seite %d", page))...) + }) + + fur := doc.FindFurniture(pages, regions, nil, nil) + // Clause 1 must not reach any of this, and the section is nine pages rather than + // five to make sure of it: "Wartung" on 4 of 9 is a 0.44 share, under the 0.50 + // cut, where 4 of 5 would be 0.80 and clause 1 would take page 23's title too. + // Asserted rather than assumed, because that failure mode would leave this test + // passing on the wrong rule. + if fur.Tabs != 0 { + t.Fatalf("clause 1 claimed %d run(s); this section is arranged so that only "+ + "clause 3 can reach its heads", fur.Tabs) + } + if fur.Heads != 2 { + t.Errorf("claimed %d head(s), want 2 — page 24 repeating page 23 and page 27 "+ + "repeating page 26", fur.Heads) + } + + blocks := doc.RegionsBlocks(pages, regions, nil, nil, fur) + for _, tc := range []struct { + page int + furniture bool + why string + }{ + {page: 23, furniture: false, why: "the first page of the first Wartung run"}, + {page: 24, furniture: true, why: "a repeat of page 23"}, + {page: 25, furniture: false, why: "a different head, and a run of one"}, + {page: 26, furniture: false, why: "the first page of the SECOND Wartung run — " + + "page 25 broke the first one"}, + {page: 27, furniture: true, why: "a repeat of page 26"}, + } { + var got, found bool + for i := range blocks { + b := &blocks[i] + if b.Page == tc.page && b.Text == titles[tc.page-23] { + got, found = b.Furniture, true + break + } + } + if !found { + t.Errorf("page %d serves no block reading %q at all", tc.page, titles[tc.page-23]) + continue + } + if got != tc.furniture { + t.Errorf("page %d's %q: furniture=%v, want %v — %s", + tc.page, titles[tc.page-23], got, tc.furniture, tc.why) + } + } +} + +// TestFurnitureKeepsASectionGenuinelyTitledA is the false positive the whole +// design is arranged around. The sequential manual titles sections "A", "B", "C" +// and "E" — 28 of its pages head a page with a bare "A" — so a one-letter line at +// the top of a page is not evidence of a tab, and only repetition across a +// section's own pages is. +func TestFurnitureKeepsASectionGenuinelyTitledA(t *testing.T) { + titles := []string{"A", "B", "C", "A", "D", "E", "B", "C"} + pages, regions := furnitureSection(8, "de", func(page, i int) []line { + lines := []line{headLine(titles[i])} + return append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz auf Seite %d", page))...) + }) + + fur := doc.FindFurniture(pages, regions, nil, nil) + if fur.Total() != 0 { + t.Errorf("claimed %d run(s) of furniture on a section whose titles are single "+ + "letters printed 2 or 3 times each", fur.Total()) + } + blocks := doc.RegionsBlocks(pages, regions, nil, nil, fur) + seen := map[string]bool{} + for i := range blocks { + if blocks[i].Furniture { + t.Errorf("page %d: %q was called furniture", blocks[i].Page, blocks[i].Text) + } + seen[blocks[i].Text] = true + } + for _, want := range []string{"A", "B", "C", "D", "E"} { + if !seen[want] { + t.Errorf("the section titled %q was lost", want) + } + } +} + +// TestFurnitureShareIsWhereTheMeasurementPutIt walks the share threshold. The +// numbers are the ones furnitureMinShare records: the widest thing in either +// manual that is NOT furniture repeats on 0.29 of its language's pages, and the +// narrowest tab on 0.96. +// +// It asks the question of [doc.Furniture.Tabs] and not of Total, and that is the +// point rather than a detail. The pages carrying the tab here are the FIRST `on` of +// the section — consecutive — so below the cut clause 3 reads them as a running +// head and claims all but the first, which is correct and is asserted just below. +// Reading Total would let clause 3's answer stand in for clause 1's and the share +// could be moved anywhere without this failing. +func TestFurnitureShareIsWhereTheMeasurementPutIt(t *testing.T) { + const n = 20 + for _, tc := range []struct { + on int + want bool + }{ + {on: 5, want: false}, // 0.25 -- a running head repeated over five pages + {on: 6, want: false}, // 0.30 -- the ceiling of everything real, measured + {on: 9, want: false}, // 0.45 + {on: 10, want: true}, // 0.50 -- the cut + {on: 19, want: true}, // 0.95 + {on: 20, want: true}, // 1.00 -- every page, which is what a tab does + } { + pages, regions := furnitureSection(n, "de", func(page, i int) []line { + lines := []line{} + if i < tc.on { + lines = append(lines, tabLine("DE")) + } + return append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz %d", page))...) + }) + fur := doc.FindFurniture(pages, regions, nil, nil) + if got := fur.Tabs > 0; got != tc.want { + t.Errorf("a tab on %d of %d pages (%.2f): clause 1 claimed it = %v, want %v", + tc.on, n, float64(tc.on)/n, got, tc.want) + } + // Whichever clause owns it, a line printed on `on` consecutive pages leaves + // exactly one of them: clause 1 takes all `on` and clause 3 takes `on`-1. + wantHeads := tc.on - 1 + if tc.want { + wantHeads = 0 + } + if fur.Heads != wantHeads { + t.Errorf("a tab on %d of %d pages: clause 3 claimed %d head(s), want %d", + tc.on, n, fur.Heads, wantHeads) + } + } +} + +// TestFurnitureNeedsFourPagesWhateverTheShare is the other guard, and it is not +// belt and braces. The column manual has a two-page spread of service addresses +// whose language no signal could name; at a share of 0.5 and no page floor, every +// line printed on both of them is furniture, because one page out of two is a +// half. +func TestFurnitureNeedsFourPagesWhateverTheShare(t *testing.T) { + for _, n := range []int{2, 3, 4} { + pages, regions := furnitureSection(n, "de", func(page, i int) []line { + lines := []line{tabLine("DE")} + return append(lines, bodyLines(95, 22.5, 4, "Kundendienststellen")...) + }) + fur := doc.FindFurniture(pages, regions, nil, nil) + if got, want := fur.Total() > 0, n >= 4; got != want { + t.Errorf("a tab on all %d pages of a %d-page section: furniture=%v, want %v", + n, n, got, want) + } + } +} + +// TestFurnitureIsPositionalNotTextual: the same words at a different height on +// each page are not furniture, however often they repeat. This is what stops a +// stock phrase — "Hinweis:" opens a note on ten pages of the sequential manual's +// German section — from being taken for a running head. +func TestFurnitureIsPositionalNotTextual(t *testing.T) { + pages, regions := furnitureSection(10, "de", func(page, i int) []line { + lines := []line{{y: 95 + float64(i)*22.5, x: 55, w: 80, size: 17, + weight: doc.WeightMedium, text: "Hinweis:"}} + return append(lines, bodyLines(300, 22.5, 5, fmt.Sprintf("Absatz %d", page))...) + }) + fur := doc.FindFurniture(pages, regions, nil, nil) + if fur.Total() != 0 { + t.Errorf("claimed %d run(s) for a phrase that repeats at a different height "+ + "on every page", fur.Total()) + } +} + +// TestFurnitureFindsTheFolioByAgreeingWithPdftotext is clause 2. The values +// differ page to page, so clause 1 cannot see them; what identifies them is that +// the second extraction of the same bytes read the same string as this page's +// printed page number. +func TestFurnitureFindsTheFolioByAgreeingWithPdftotext(t *testing.T) { + const n = 8 + folios := map[int]int{} + pages, regions := furnitureSection(n, "de", func(page, i int) []line { + folios[page] = page - 6 + lines := []line{folioLine(fmt.Sprintf("%d", page-6))} + return append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz %d", page))...) + }) + + fur := doc.FindFurniture(pages, regions, nil, folios) + if fur.Folios != n { + t.Errorf("claimed %d folio(s) over %d pages each printing one", fur.Folios, n) + } + if fur.Tabs != 0 { + t.Errorf("clause 1 claimed %d run(s); every folio prints a different number", fur.Tabs) + } + + // Without the folios there is nothing to agree with, and the numbers stay. + if bare := doc.FindFurniture(pages, regions, nil, nil); bare.Total() != 0 { + t.Errorf("claimed %d run(s) with no printed folios supplied", bare.Total()) + } + + blocks := doc.RegionsBlocks(pages, regions, nil, nil, fur) + for _, s := range furnitureOf(blocks) { + if len(s) > 2 { + t.Errorf("furniture block %q is not a folio", s) + } + } +} + +// TestFurnitureLeavesANumberThatIsNotTheFolio: a table cell or a callout that +// happens to be a number at a repeated height is content, because it does not +// agree with what the page prints as its own page number. +func TestFurnitureLeavesANumberThatIsNotTheFolio(t *testing.T) { + const n = 8 + folios := map[int]int{} + pages, regions := furnitureSection(n, "de", func(page, i int) []line { + folios[page] = page - 6 + // A callout numbered 1..8 at a fixed height beside a diagram, and NOT this + // page's folio except by accident on one page. + lines := []line{{y: 300, x: 55, w: 11, size: 12, text: fmt.Sprintf("%d", i+1)}} + return append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz %d", page))...) + }) + + fur := doc.FindFurniture(pages, regions, nil, folios) + if fur.Folios != 0 { + t.Errorf("claimed %d folio(s) among callout numbers at a fixed height", fur.Folios) + } +} + +// TestFurnitureCountsAgainstTheLanguagesOwnPages is the denominator the first +// attempt got wrong. Two languages of 8 pages each, converted together as one +// household of 16: each tab is on 8 of 16 pages of the conversion and on 8 of 8 +// pages of its own section. +func TestFurnitureCountsAgainstTheLanguagesOwnPages(t *testing.T) { + var pages []doc.PageRuns + var regions []doc.Region + for i := 0; i < 16; i++ { + no := 23 + i + lang, tab := "de", "DE" + if i >= 8 { + lang, tab = "ru", "RU" + } + lines := []line{tabLine(tab)} + lines = append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz %d", no))...) + pages = append(pages, *blockPage(no, lines...)) + regions = append(regions, doc.Region{Page: no, X0: 0, X1: testBlockPageWidth, + Lang: lang, Source: doc.SourceRepertoire}) + } + + inScope := map[string]bool{"de": true, "ru": true} + fur := doc.FindFurniture(pages, regions, inScope, nil) + if fur.Tabs != 16 { + t.Errorf("claimed %d tab run(s); each of two 8-page sections prints one on "+ + "every page, and 8 of 16 converted pages is 0.5 only by accident", fur.Tabs) + } + + // And the same section read alone reaches the same conclusion, which is what + // makes the pass independent of who is reading. + alone := doc.FindFurniture(pages[:8], regions[:8], map[string]bool{"de": true}, nil) + if alone.Tabs != 8 { + t.Errorf("claimed %d tab run(s) reading German alone", alone.Tabs) + } +} + +// TestFurnitureSkipsARegionWithNoLanguage: a region no signal could name has no +// section for a share to be a share of, so it is left entirely alone. This is +// what keeps the column manual's unnamed two-page spread of service addresses out +// of the pass, which is the accident the page floor also guards. +func TestFurnitureSkipsARegionWithNoLanguage(t *testing.T) { + pages, regions := furnitureSection(8, "", func(page, i int) []line { + lines := []line{tabLine("DE")} + return append(lines, bodyLines(95, 22.5, 5, "Kundendienststellen")...) + }) + if fur := doc.FindFurniture(pages, regions, nil, nil); fur.Total() != 0 { + t.Errorf("claimed %d run(s) inside regions with no language", fur.Total()) + } +} + +// TestRegionBlocksWithNilFurnitureIsUnchanged: nil is a normal argument and it +// must produce exactly the reading that shipped before this existed, furniture +// and all. Without this the pass could be "fixing" a defect it introduced. +func TestRegionBlocksWithNilFurnitureIsUnchanged(t *testing.T) { + pages, regions := furnitureSection(8, "de", func(page, i int) []line { + lines := []line{tabLine("DE"), headLine(fmt.Sprintf("Kapitel %d", i))} + return append(lines, bodyLines(95, 22.5, 4, fmt.Sprintf("Absatz %d", page))...) + }) + + bare := doc.RegionsBlocks(pages, regions, nil, nil, nil) + if len(furnitureOf(bare)) != 0 { + t.Fatalf("nil furniture produced %d flagged block(s)", len(furnitureOf(bare))) + } + tabs := 0 + for i := range bare { + if bare[i].Text == "DE" { + tabs++ + } + } + if tabs != 8 { + t.Errorf("the tab came back as a block on %d of 8 pages with no pass run", tabs) + } +} + +// TestFurnitureIsDeterministic: the pass walks maps, and a conversion whose block +// indices depend on map order is a conversion that inserts a parallel set of rows +// the second time a job runs. +func TestFurnitureIsDeterministic(t *testing.T) { + folios := map[int]int{} + pages, regions := furnitureSection(12, "de", func(page, i int) []line { + folios[page] = page - 6 + lines := []line{tabLine("DE"), headLine(fmt.Sprintf("Kapitel %d", i)), + folioLine(fmt.Sprintf("%d", page-6))} + return append(lines, bodyLines(95, 22.5, 5, fmt.Sprintf("Absatz %d", page))...) + }) + + first := doc.RegionsBlocks(pages, regions, + nil, nil, doc.FindFurniture(pages, regions, nil, folios)) + for run := 0; run < 5; run++ { + again := doc.RegionsBlocks(pages, regions, + nil, nil, doc.FindFurniture(pages, regions, nil, folios)) + if len(again) != len(first) { + t.Fatalf("run %d produced %d blocks against %d", run, len(again), len(first)) + } + for i := range first { + if first[i].Text != again[i].Text || first[i].Index != again[i].Index || + first[i].Furniture != again[i].Furniture { + t.Fatalf("run %d block %d diverged: %+v against %+v", run, i, again[i], first[i]) + } + } + } +} + +// TestFurnitureNoteNamesItsEvidence: a note that cannot be held against the page +// is not evidence, which is the stance every other note in this package takes. +func TestFurnitureNoteNamesItsEvidence(t *testing.T) { + pages, regions := furnitureSection(10, "de", func(page, i int) []line { + lines := []line{tabLine("DE")} + return append(lines, bodyLines(95, 22.5, 4, fmt.Sprintf("Absatz %d", page))...) + }) + blocks := doc.RegionsBlocks(pages, regions, nil, nil, + doc.FindFurniture(pages, regions, nil, nil)) + for i := range blocks { + if !blocks[i].Furniture { + continue + } + note := blocks[i].Note + for _, want := range []string{"page furniture", `"DE"`, "y=58", "10 of this language's 10"} { + if !strings.Contains(note, want) { + t.Errorf("note %q does not say %q", note, want) + } + } + } +} diff --git a/internal/doc/lang.go b/internal/doc/lang.go new file mode 100644 index 0000000..bb54b5d --- /dev/null +++ b/internal/doc/lang.go @@ -0,0 +1,202 @@ +package doc + +import ( + "strings" + + "golang.org/x/text/language" +) + +// codeAliases maps the language labels manuals actually print onto BCP-47. +// +// Two distinct problems live here. The first is that some manuals use a code +// that is simply wrong: the measured fixture prints UA for Ukrainian (the tag is +// uk; UA is the country) and CZ for Czech (the tag is cs). The second is that +// many manuals label a section by the *country* they sell it in rather than by +// the language it is written in — DK for Danish, SE for Swedish, JP for Japanese. +// +// Both are aliases rather than errors to reject. A manual that says CZ is telling +// us something true in a non-standard way, and dropping the section because the +// label is not a valid tag would lose real information. +var codeAliases = map[string]string{ + // Wrong tag for the right language, seen in real manuals. + "UA": "uk", // Ukraine (country) used for Ukrainian + "CZ": "cs", // Czechia (country) used for Czech + "GR": "el", // Greece used for Greek + "RS": "sr", // Serbia used for Serbian + "SI": "sl", // Slovenia used for Slovenian + "EE": "et", // Estonia used for Estonian + "DK": "da", // Denmark used for Danish + "SE": "sv", // Sweden used for Swedish + "JP": "ja", // Japan used for Japanese + "CN": "zh", // China used for Chinese + "KR": "ko", // Korea used for Korean + "IL": "he", // Israel used for Hebrew + "IR": "fa", // Iran used for Persian + "BR": "pt-BR", + "TW": "zh-TW", + "HK": "zh-HK", + + // Three-letter codes, as manuals actually print them. Mostly ISO 639-2/B, + // which is the vernacular form — GER rather than DEU, and both of the + // measured manual's Cyrillic codes are of this kind. + "ENG": "en", "GER": "de", "DEU": "de", "FRA": "fr", "FRE": "fr", + "ITA": "it", "SPA": "es", "ESP": "es", "POR": "pt", "NLD": "nl", + "DUT": "nl", "POL": "pl", "CZE": "cs", "CES": "cs", "SVK": "sk", + "SLO": "sk", "SLV": "sl", "HUN": "hu", "ROM": "ro", "RON": "ro", + "BUL": "bg", "RUS": "ru", "UKR": "uk", "BLR": "be", "SRP": "sr", + "SRB": "sr", "HRV": "hr", "BOS": "bs", "MKD": "mk", "LIT": "lt", + "LAV": "lv", "EST": "et", "FIN": "fi", "SWE": "sv", "NOR": "no", + "DAN": "da", "ISL": "is", "GRE": "el", "ELL": "el", "TUR": "tr", + "KAZ": "kk", "UZB": "uz", "ARA": "ar", "HEB": "he", "THA": "th", + "VIE": "vi", "IND": "id", "MSA": "ms", "CHN": "zh", "ZHO": "zh", + "JPN": "ja", "KOR": "ko", + + // Single letters. Real and common on European manuals, and the most + // ambiguous token a page can carry — "D" is also a list marker and a + // diagram label — so these are believed only with corroboration. See + // singleLetterNeedsSupport. + "D": "de", "F": "fr", "I": "it", "E": "es", "P": "pt", + "N": "no", "S": "sv", "H": "hu", +} + +// PlausibleCodeToken reports whether a token could be a printed language label. +// +// Shape is not enough, and the gap is wider than it looks. golang.org/x/text +// accepts "one", "two", "the", "and", "for" and "abc" as languages — they are +// real ISO 639-3 codes for languages no appliance manual is printed in — so +// letting a three-letter token fall through to the parser turns the first word of +// any page into a language tag. That happened: "one" was read as a code. +// +// So the rule is by length. Two letters, with an optional region, go to the +// parser, which knows the small closed set of ISO 639-1. One and three letters +// must appear in codeAliases, which lists what manuals actually print. +func PlausibleCodeToken(s string) bool { + base, region, hasRegion := strings.Cut(strings.TrimSpace(s), "-") + if hasRegion && len(region) != 2 { + return false + } + switch len(base) { + case 2: + _, ok := NormalizeCode(s) + return ok + case 1, 3: + _, ok := codeAliases[strings.ToUpper(base)] + return ok + default: + return false + } +} + +// NormalizeCode turns a printed language label into a BCP-47 tag. +// +// The raw code is always preserved by the caller alongside the result, so a label +// that cannot be normalised is still reportable rather than discarded. That is +// the point of storing both `code` and `lang` on a language run. +func NormalizeCode(raw string) (string, bool) { + code := strings.TrimSpace(raw) + if code == "" { + return "", false + } + upper := strings.ToUpper(code) + + if alias, ok := codeAliases[upper]; ok { + code = alias + } + + tag, err := language.Parse(code) + if err != nil { + return "", false + } + // Canonical form, e.g. "zh-hk" becomes "zh-HK" and "EN" becomes "en". + return tag.String(), true +} + +// BaseLanguage reduces a BCP-47 tag to its base subtag: "zh-HK" becomes "zh", +// "pt-BR" becomes "pt". Used when comparing against the household's configured +// languages, where a reader of pt reads pt-BR. +func BaseLanguage(tag string) string { + parsed, err := language.Parse(tag) + if err != nil { + return "" + } + base, _ := parsed.Base() + return base.String() +} + +// SameLanguage reports whether two tags name the same language, ignoring region +// and script. It is what decides whether a section is one the household reads. +func SameLanguage(a, b string) bool { + ba, bb := BaseLanguage(a), BaseLanguage(b) + return ba != "" && ba == bb +} + +// MatchesAny reports whether tag is one of the household's languages, and which +// one it matched. +func MatchesAny(tag string, household []string) (string, bool) { + for _, h := range household { + if SameLanguage(tag, h) { + return h, true + } + } + return "", false +} + +// DisplayName returns a human-readable language name in English, falling back to +// the tag itself when it cannot be named. Used in the pre-flight gate, where +// "Ukrainian" is far more useful than "uk". +func DisplayName(tag string) string { + parsed, err := language.Parse(tag) + if err != nil { + return tag + } + if name := languageNames[BaseLanguage(parsed.String())]; name != "" { + return name + } + return parsed.String() +} + +// KnownLanguage reports whether a tag names a language manualbox recognises. +// +// A tag can parse cleanly and name nothing at all: BCP-47 constrains the shape of +// a subtag, not its meaning, so language.Parse accepts FAX as "fax", TEL as "te" +// and NDE as "nd". That is measured rather than hypothetical. The column manual +// prints FAX on its page of service addresses, the printed-index parser reads that +// page as a contents table and offers FAX as an index entry, and reconciliation +// then labelled two of the document's pages "fax" — overriding two columns that +// correctly read as German and Polish. +// +// [languageNames] is the set of languages that actually appear in appliance +// manuals, so membership is the available definition of "a language a household +// could read". This is deliberately not a filter on what may be stored: an +// unrecognised code is still kept and still reported, because a manual printing an +// unknown code is information. It is a filter on what may outrank other evidence. +func KnownLanguage(tag string) bool { + if tag == "" { + return false + } + parsed, err := language.Parse(tag) + if err != nil { + return false + } + return languageNames[BaseLanguage(parsed.String())] != "" +} + +// languageNames covers the languages that actually turn up in appliance manuals. +// x/text can produce display names only with the full display package and its +// tables, which is a large dependency for a label; this is the subset that +// matters. Anything absent falls back to the tag. +var languageNames = map[string]string{ + "ar": "Arabic", "be": "Belarusian", "bg": "Bulgarian", "bs": "Bosnian", + "ca": "Catalan", "cs": "Czech", "da": "Danish", "de": "German", + "el": "Greek", "en": "English", "es": "Spanish", "et": "Estonian", + "fa": "Persian", "fi": "Finnish", "fr": "French", "he": "Hebrew", + "hi": "Hindi", "hr": "Croatian", "hu": "Hungarian", "hy": "Armenian", + "id": "Indonesian", "is": "Icelandic", "it": "Italian", "ja": "Japanese", + "ka": "Georgian", "kk": "Kazakh", "ko": "Korean", "lt": "Lithuanian", + "lv": "Latvian", "mk": "Macedonian", "ms": "Malay", "nb": "Norwegian Bokmål", + "ne": "Nepali", "nl": "Dutch", "nn": "Norwegian Nynorsk", "no": "Norwegian", + "pl": "Polish", "pt": "Portuguese", "ro": "Romanian", "ru": "Russian", + "sk": "Slovak", "sl": "Slovenian", "sq": "Albanian", "sr": "Serbian", + "sv": "Swedish", "th": "Thai", "tr": "Turkish", "uk": "Ukrainian", + "ur": "Urdu", "uz": "Uzbek", "vi": "Vietnamese", "zh": "Chinese", +} diff --git a/internal/doc/neutral.go b/internal/doc/neutral.go new file mode 100644 index 0000000..33d8cca --- /dev/null +++ b/internal/doc/neutral.go @@ -0,0 +1,195 @@ +package doc + +import ( + "context" + "errors" + "fmt" + + "github.com/gordon2/manualbox/internal/extern" +) + +// A NEUTRAL PAGE IS A PAGE NO LANGUAGE OWNS, AND IT IS NOT THE SAME QUESTION AS +// AN UNLABELLED ONE. +// +// [CountUnlabelled] and the gate's own unlabelled count both exclude front matter +// and the back cover deliberately: those pages carry text and belong to no section +// legitimately, so counting them would report a fault on every document. That is +// the right answer to "how much would a statistical detector add?". +// +// It is the wrong answer to "what can a reader not reach?", because front matter +// is exactly where the unreachable content is. The sequential manual's exploded +// parts diagram is on PDF page 5, its four sub-drawings are found, 31 places in its +// content pages say "see A-1", and no language section contains page 5 — so no +// conversion has ever served it. Filtering front matter out here would filter out +// the whole finding. +// +// So this pass asks the other question, over every page of the document with no +// content range applied, and the two counts are deliberately different numbers on +// the same document: the sequential manual is 4 unlabelled and 7 neutral. +// +// WHY THE PICTURE COUNT IS PART OF IT AND NOT AN EXTRA. Measured on both fixtures, +// the character count on its own inverts the truth: +// +// neutral pages chars figures +// sequential manual 7 1,656 61 +// parallel-columns manual 2 11,256 0 +// +// (Those are pdftotext's page counts. The gate reports the regions' — 1,639 and +// 10,782 — because that is the measurement the rest of that screen uses. The +// ordering, which is the point, is the same either way.) +// +// The columns manual's two pages are a print code and a page of service addresses +// in twelve languages; they hold seven times the text and not one picture. The +// sequential manual's are a cover, three contents pages, TWO DIAGRAM PLATES +// carrying 59 of those 61 figures, and a colophon. A gate that offered "7 pages, +// 1,656 characters" against "2 pages, 11,256 characters" would invite the user to +// decline the one worth taking and accept the one that is furniture. The picture +// count is the discriminator, so it is measured rather than left out. + +// maxNeutralInkPages bounds the ink pass. Reading a page's drawings is one +// pdftocairo spawn — 42.3 s over the sequential manual's 560 pages, which +// pageRegionsWithTables measured and which is why that pass is lazy too — and this +// runs inside the free pre-flight. +// +// The bound is not a performance hedge with a round number behind it; it is what +// separates the case this feature is for from a different failure. A handful of +// pages no language owns is front matter, which is normal and is where the +// diagrams are: 7 and 2 on the fixtures. Hundreds of them means language detection +// did not work on this document, and then the honest answer is the count and not a +// picture census costing a spawn per page. Over the bound the counts stay nil, +// which reads as "not counted" and never as "none" — see [Page.Figures]. +const maxNeutralInkPages = 32 + +// NeutralPages are the pages of a document that carry content and that no named +// language region claims, ascending. +// +// Named region, not region: an unnamed region is a reportable state and never a +// language, which is the stance `attribute` and `figureLang` already take. A page +// holding one unnamed whole-page region is as unowned as a page holding none. +// +// Empty when the document has no regions at all. That is not the claim that every +// page is owned — it is that the question cannot be asked at this resolution. A +// document probed without pdftohtml has a complete per-page language map and no +// coordinates, and the neutral set is defined against the region map; offering +// pages from the per-page map instead would offer a different set under the same +// name. The gate says so rather than guessing. +func (r *Result) NeutralPages() []int { + if len(r.Regions) == 0 { + return nil + } + named := make(map[int]bool, len(r.Regions)) + for i := range r.Regions { + reg := &r.Regions[i] + if reg.Lang != "" || reg.Code != "" { + named[reg.Page] = true + } + } + + out := make([]int, 0, 8) + for i := range r.Pages { + p := &r.Pages[i] + if named[p.No] || !CarriesContent(p.Chars, p.Figures) { + continue + } + out = append(out, p.No) + } + return out +} + +// CarriesContent reports that there is something on a page worth converting. +// +// Either enough text to clear [MinTextChars] — the same floor every other size +// question in this package applies, so a folio alone does not qualify — or at least +// one picture. The second arm is why the ink pass runs before this is asked: a +// full-page diagram plate with no labels at all carries 0 characters and is the +// single most valuable page in the set. +// +// A nil figures means the ink was never read, and then only the text can answer. +// That is a deliberate under-count rather than an assumption: it can only omit a +// page, never offer one that holds nothing. +// +// EXPORTED, AND TAKING TWO SCALARS RATHER THAN A Page, for the reason [MinTextChars] +// is exported: the pre-flight gate asks this question of stored rows and this +// package asks it of a probe result, and the two must not be able to disagree about +// the same page. A page the gate offers and the conversion then skips is a promise +// the funnel broke — and a mutation that changed one copy of a duplicated rule was +// caught by only one of the two tests, which is how this came to be one function. +func CarriesContent(chars int, figures *int) bool { + if chars >= MinTextChars { + return true + } + return figures != nil && *figures > 0 +} + +// countNeutralInk fills in [Page.Figures] for the pages no named region claims. +// +// It runs inside [Analyze], after the regions, and it is lazy in exactly the way +// pageRegionsWithTables is lazy: a spawn per page is affordable for seven pages of +// front matter and is not affordable for a document. Measured on the fixtures, the +// whole pass is 7 spawns on the 560-page manual and 2 on the 68-page one. +// +// A missing or failing pdftocairo leaves every count nil and is not an error. The +// document has been probed by this point; losing this costs the gate its picture +// count and nothing else, and nil already means "not counted". +func countNeutralInk(ctx context.Context, path string, res *Result, runs []PageRuns) string { + // Candidates are chosen before any content test, because the text floor cannot + // judge a plate whose labels are all inside its drawings. + named := make(map[int]bool, len(res.Regions)) + for i := range res.Regions { + reg := &res.Regions[i] + if reg.Lang != "" || reg.Code != "" { + named[reg.Page] = true + } + } + + byNo := make(map[int]*PageRuns, len(runs)) + for i := range runs { + byNo[runs[i].No] = &runs[i] + } + + candidates := make([]int, 0, 8) + for i := range res.Pages { + if !named[res.Pages[i].No] { + candidates = append(candidates, res.Pages[i].No) + } + } + if len(candidates) == 0 { + return "" + } + if len(candidates) > maxNeutralInkPages { + return fmt.Sprintf("the pictures on the %d pages no language claims were not "+ + "counted: that is more than %d pages, which is a language map that did not "+ + "work rather than the front matter this counts", + len(candidates), maxNeutralInkPages) + } + + pageByNo := make(map[int]*Page, len(res.Pages)) + for i := range res.Pages { + pageByNo[res.Pages[i].No] = &res.Pages[i] + } + + for _, no := range candidates { + if err := ctx.Err(); err != nil { + return "the pictures on the pages no language claims were not counted: " + err.Error() + } + pr := byNo[no] + if pr == nil { + // No positioned text for this page, so the text guard has nothing to judge a + // background wash against and FindFigures would answer one page-sized + // figure. Left uncounted rather than answered wrongly. + continue + } + ink, err := ExtractInk(ctx, path, no) + if err != nil { + if errors.Is(err, extern.ErrNotFound) { + return "the pictures on the pages no language claims were not counted: " + err.Error() + } + // One page failing is not the whole pass failing, and nil says so for that + // page on its own. + continue + } + n := len(FindFigures(ink, pr)) + pageByNo[no].Figures = &n + } + return "" +} diff --git a/internal/doc/neutral_fixture_test.go b/internal/doc/neutral_fixture_test.go new file mode 100644 index 0000000..32f4624 --- /dev/null +++ b/internal/doc/neutral_fixture_test.go @@ -0,0 +1,172 @@ +package doc_test + +import ( + "context" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// neutralFixture is the manual, its probe result and the pages no language owns. +func neutralFixture(t *testing.T, name string) (res *doc.Result, path string, neutral []int) { + t.Helper() + if name == "thomas-drybox-amfibia" { + _, path = columnFixture(t) + } else { + _, path = loadFixture(t) + } + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + neutral = res.NeutralPages() + return res, path, neutral +} + +// TestNeutralPagesOfBothManuals pins the set this feature exists to offer, and +// pins it as CONTENTS and not only as a count. +// +// The brief that asked for this said to stop and report if the set turned out to be +// mostly covers and blank leaves rather than diagrams. It is one of each, which is +// why both documents are pinned here rather than only the one that motivated it: +// +// - The sequential manual's 7 pages are a cover, three contents pages, TWO DIAGRAM +// PLATES and a colophon. Pages 5 and 6 carry 59 of the set's 61 pictures, and +// page 5 is the A-1 exploded parts diagram that 31 places in the content pages +// point at. Worth offering. +// +// THIS IS THE DETECTOR'S COUNT AND NOT THE SERVED ONE, which is a distinction the band +// created and this test is where it shows. doc.ServedFigures drops a crop that lies +// wholly inside another, and one of these plate crops does, so a conversion of these +// pages renders 60. The gate says 61 because it is answering "what is drawn on these +// pages", which is the question a user deciding whether to opt in is asking, and +// because the probe runs long before any crop exists. The one-picture difference is +// recorded rather than reconciled: making the gate render every crop to count it would +// be paying the conversion's cost to answer the question that precedes it. +// - The columns manual's 2 pages are a print code and a page of service addresses +// in twelve languages. They hold seven times the text of the other set and NOT +// ONE PICTURE. Correctly offered and expected to be declined. +// +// The second row is the one that earns the picture count. A gate reporting only +// characters would rank these two sets in exactly the wrong order. +func TestNeutralPagesOfBothManuals(t *testing.T) { + for _, tc := range []struct { + name string + pages []int + figures int + // withFigures is how many of those pages carry at least one picture. + withFigures int + }{ + {"dreame-l40-ultra", []int{1, 2, 3, 4, 5, 6, 560}, 61, 3}, + {"thomas-drybox-amfibia", []int{67, 68}, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + res, _, neutral := neutralFixture(t, tc.name) + + if len(neutral) != len(tc.pages) { + t.Fatalf("neutral pages = %v, want %v", neutral, tc.pages) + } + for i := range neutral { + if neutral[i] != tc.pages[i] { + t.Fatalf("neutral pages = %v, want %v", neutral, tc.pages) + } + } + + // The census is over exactly those pages and nowhere else. A non-nil count on + // a page a language owns would mean the pass had stopped being lazy, which is + // the property that keeps it affordable. + offered := make(map[int]bool, len(neutral)) + for _, p := range neutral { + offered[p] = true + } + total, with, counted := 0, 0, 0 + for i := range res.Pages { + p := &res.Pages[i] + if p.Figures == nil { + continue + } + if !offered[p.No] { + // Candidates are chosen before the content test, so a page with no + // region and too little text to offer may legitimately be counted. + if res.Regions != nil && p.Chars >= doc.MinTextChars { + t.Errorf("page %d is owned by a language and was still counted", p.No) + } + continue + } + counted++ + total += *p.Figures + if *p.Figures > 0 { + with++ + } + } + if counted != len(neutral) { + t.Errorf("%d of %d offered pages were counted; the gate cannot sum a "+ + "partial census", counted, len(neutral)) + } + if total != tc.figures { + t.Errorf("figures on the neutral pages = %d, want %d", total, tc.figures) + } + if with != tc.withFigures { + t.Errorf("figure-bearing neutral pages = %d, want %d", with, tc.withFigures) + } + if res.NeutralNote != "" { + t.Errorf("NeutralNote = %q; both fixtures are well under the census bound", + res.NeutralNote) + } + }) + } +} + +// TestTheDiagramPlateIsInTheNeutralSet is the finding itself, asserted rather than +// described: the page the manual's own cross-references point at is a page the set +// on offer contains, and it is the pictures on it that make it worth offering. +func TestTheDiagramPlateIsInTheNeutralSet(t *testing.T) { + res, _, neutral := neutralFixture(t, "dreame-l40-ultra") + + const plate = 5 + found := false + for _, p := range neutral { + if p == plate { + found = true + } + } + if !found { + t.Fatalf("PDF page %d is the A-1 exploded parts diagram and is not on offer: %v", + plate, neutral) + } + + var figures *int + for i := range res.Pages { + if res.Pages[i].No == plate { + figures = res.Pages[i].Figures + } + } + if figures == nil { + t.Fatal("the diagram plate's pictures were not counted, so the gate cannot say " + + "what is on the page it is offering") + } + // 31 on the measured document. Pinned as a floor rather than an equality because + // the figure pass's merge and growth rules are still moving and this test is about + // the page being reachable, not about the clustering; the count itself is pinned + // exactly in TestNeutralPagesOfBothManuals. + if *figures < 20 { + t.Errorf("the diagram plate holds %d pictures; it is a plate of drawings and a "+ + "count this low means the census is not seeing them", *figures) + } +} + +// TestNeutralPagesNeedRegions pins the one case where the answer is "the question +// cannot be asked" rather than a set: a document probed without positioned text has +// no region map, and the neutral set is defined against the region map. +// +// Answering it from the per-page runs instead would offer a different set under the +// same name, which is the failure the gate's own region fallback is careful about. +func TestNeutralPagesNeedRegions(t *testing.T) { + res := &doc.Result{ + Info: doc.Info{Pages: 3}, + Pages: []doc.Page{{No: 1, Chars: 900}, {No: 2, Chars: 900}, {No: 3, Chars: 900}}, + } + if got := res.NeutralPages(); len(got) != 0 { + t.Errorf("NeutralPages = %v with no regions stored; want none offered", got) + } +} diff --git a/internal/doc/neutral_pdf_test.go b/internal/doc/neutral_pdf_test.go new file mode 100644 index 0000000..8b76ea1 --- /dev/null +++ b/internal/doc/neutral_pdf_test.go @@ -0,0 +1,128 @@ +package doc_test + +import ( + "context" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// unownedPage is a page of ordinary prose carrying no language code at all, dense +// enough that the region pass gives it a whole-page region. +// +// Twelve lines rather than one, and that is a fact about the region pass worth +// keeping: a page holding a single text run gets no region, so a generated document +// of one-line pages produces an empty region map and quietly skips every test built +// on it. The first version of this test did exactly that and passed by skipping. +func unownedPage() testpdf.Page { + lines := make([]string, 0, 12) + for i := range 12 { + lines = append(lines, + strings.Repeat("Maintenance information for this appliance. ", 2)+string(rune('a'+i))) + } + return testpdf.Page{ + Lines: lines, + // A drawing on every page, so a census that DID run would find something. + // Without it this could pass by finding nothing. + Drawings: []testpdf.Drawing{{X: 100, Y: 120, W: 200, H: 150, Strokes: 40}}, + } +} + +// TestTheCensusIsBoundedAndSaysSoWhenItStops pins the cost guard on the picture +// count, which is the one thing this feature made the free probe do more of. +// +// Counting a page's drawings is a pdftocairo spawn — 42.3 s over the sequential +// manual's 560 pages, which is why doc.pageRegionsWithTables is lazy for exactly the +// same reason. Seven pages of front matter is affordable and is the case this exists +// for. Hundreds of unowned pages is not front matter at all, it is a language map +// that did not work, and answering that with a spawn per page would make the free +// pre-flight ten times slower on the document that is already going badly. +// +// So over the bound the counts stay nil and a note says why. nil reads as "not +// counted" everywhere and never as "no pictures", which is what lets the gate +// withhold a total rather than show an understated one. +func TestTheCensusIsBoundedAndSaysSoWhenItStops(t *testing.T) { + const pages = 40 // over the bound of 32 + generated := make([]testpdf.Page, 0, pages) + for range pages { + generated = append(generated, unownedPage()) + } + path := figurePDF(t, testpdf.Doc{Pages: generated}) + + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + + // Asserted rather than skipped past: if these pages stopped being unowned this + // test would silently stop testing the bound. + unowned := 0 + for i := range res.Regions { + r := &res.Regions[i] + if r.Lang == "" && r.Code == "" { + unowned++ + } + } + if unowned != pages { + t.Fatalf("%d of %d pages came back unowned; this test needs all of them to be, "+ + "or it is not exercising the bound", unowned, pages) + } + + if res.NeutralNote == "" { + t.Error("40 unowned pages is over the bound and nothing says the pictures were " + + "not counted") + } + for i := range res.Pages { + if res.Pages[i].Figures != nil { + t.Fatalf("page %d was counted although the census should have stopped", + res.Pages[i].No) + } + } + + // The pages are still offered. They are still outside every language, which is the + // thing the user needs told; only the picture count is missing. + if got := res.NeutralPages(); len(got) != pages { + t.Errorf("offered %d pages, want all %d — the bound withholds the count, not the "+ + "offer", len(got), pages) + } +} + +// TestASmallSetIsCensusedRatherThanRefused is the other side of the bound: the case +// this feature is actually for must not be caught by the guard meant for the case it +// is not. +func TestASmallSetIsCensusedRatherThanRefused(t *testing.T) { + // Two unowned pages in front of three tagged sections, which is the shape of front + // matter and is well under the bound. + generated := []testpdf.Page{unownedPage(), unownedPage()} + tagged := testpdf.TaggedSections([]string{"EN", "DE", "FR"}, 3, false) + generated = append(generated, tagged.Pages...) + path := figurePDF(t, testpdf.Doc{Pages: generated}) + + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if res.NeutralNote != "" { + t.Fatalf("NeutralNote = %q; two pages is far under the bound", res.NeutralNote) + } + + got := res.NeutralPages() + if len(got) != 2 || got[0] != 1 || got[1] != 2 { + t.Fatalf("offered %v, want pages 1 and 2", got) + } + for _, no := range got { + var figures *int + for i := range res.Pages { + if res.Pages[i].No == no { + figures = res.Pages[i].Figures + } + } + if figures == nil { + t.Errorf("page %d was offered and its pictures were not counted", no) + } else if *figures == 0 { + t.Errorf("page %d carries a drawing and the census found none", no) + } + } +} diff --git a/internal/doc/neutral_test.go b/internal/doc/neutral_test.go new file mode 100644 index 0000000..31f5b54 --- /dev/null +++ b/internal/doc/neutral_test.go @@ -0,0 +1,98 @@ +package doc_test + +import ( + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +func pageWith(no, chars int, figures *int) doc.Page { + return doc.Page{No: no, Chars: chars, Figures: figures} +} + +func figs(n int) *int { return &n } + +// TestNeutralPagesExcludeEveryPageALanguageOwns is the funnel's boundary on the +// second scope: the offer is what is left over, never the document. +// +// It exists because a mutation that deleted the ownership test survived the whole +// suite. Nothing else asked the question — the gate builds its own map of which +// pages are named, and the end-to-end tests assert that the plate IS served without +// asserting that the other 553 pages are not. Dropping this test makes converting a +// 560-page manual mean rendering all 560. +func TestNeutralPagesExcludeEveryPageALanguageOwns(t *testing.T) { + res := &doc.Result{ + Info: doc.Info{Pages: 5}, + Pages: []doc.Page{ + pageWith(1, 900, nil), // no region names it + pageWith(2, 900, nil), // German + pageWith(3, 900, nil), // named by code alone, no BCP-47 tag + pageWith(4, 900, nil), // an unnamed region, which is not a language + pageWith(5, 10, nil), // unnamed but too small to be worth anything + }, + Regions: []doc.Region{ + {Page: 2, Lang: "de", Code: "DE"}, + // A printed code nothing could normalise still names the page. Real manuals + // print D, RUS, UA and KAZ, and a page carrying one is not unowned. + {Page: 3, Code: "KAZ"}, + {Page: 4, Note: "no language established for this page"}, + {Page: 5}, + }, + } + + got := res.NeutralPages() + want := []int{1, 4} + if len(got) != len(want) { + t.Fatalf("NeutralPages = %v, want %v", got, want) + } + for i := range got { + if got[i] != want[i] { + t.Fatalf("NeutralPages = %v, want %v", got, want) + } + } +} + +// TestAPictureIsEnoughToOfferAPageWithNoText is the arm the character floor cannot +// reach, and it is the one the whole feature turns on: a plate of drawings whose +// labels are all inside the drawings carries no extractable text at all. +func TestAPictureIsEnoughToOfferAPageWithNoText(t *testing.T) { + res := &doc.Result{ + Info: doc.Info{Pages: 3}, + Pages: []doc.Page{ + pageWith(1, 4, figs(31)), // the plate: 4 characters, 31 drawings + pageWith(2, 4, figs(0)), // counted, and genuinely empty + pageWith(3, 4, nil), // never counted, so only the text can answer + }, + Regions: []doc.Region{{Page: 1}, {Page: 2}, {Page: 3}}, + } + + got := res.NeutralPages() + if len(got) != 1 || got[0] != 1 { + t.Fatalf("NeutralPages = %v, want just the plate on page 1", got) + } +} + +// TestCarriesContentSeparatesNotCountedFromNone pins the distinction the whole +// nullable column exists for. Nothing may treat "nobody looked" as "there is +// nothing there". +func TestCarriesContentSeparatesNotCountedFromNone(t *testing.T) { + for _, tc := range []struct { + name string + chars int + figures *int + want bool + }{ + {"enough text on its own", doc.MinTextChars, nil, true}, + {"one rune under the floor with nothing else", doc.MinTextChars - 1, nil, false}, + {"no text but a picture", 0, figs(1), true}, + {"no text and a real zero", 0, figs(0), false}, + {"no text and nobody counted", 0, nil, false}, + {"plenty of text and a real zero is still content", 900, figs(0), true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := doc.CarriesContent(tc.chars, tc.figures); got != tc.want { + t.Errorf("CarriesContent(%d, %v) = %t, want %t", tc.chars, tc.figures, got, tc.want) + } + }) + } +} diff --git a/internal/doc/neutralconvert_fixture_test.go b/internal/doc/neutralconvert_fixture_test.go new file mode 100644 index 0000000..3662684 --- /dev/null +++ b/internal/doc/neutralconvert_fixture_test.go @@ -0,0 +1,222 @@ +package doc_test + +import ( + "context" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// convertBothWays converts one household twice off ONE probe: without the pages no +// language owns, and with them. One Analyze rather than two because it is the +// expensive half and because the comparison must hold the probe constant — a +// difference between the two conversions has to be the option and nothing else. +func convertBothWays(t *testing.T, name string, langs ...string) (without, with *doc.Conversion) { + t.Helper() + var path string + if name == "thomas-drybox-amfibia" { + _, path = columnFixture(t) + } else { + _, path = loadFixture(t) + } + ctx := context.Background() + res, err := doc.Analyze(ctx, path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if res.RegionNote != "" { + t.Skipf("no regions were produced: %s", res.RegionNote) + } + + without, err = doc.Convert(ctx, path, res, langs, doc.ConvertOptions{}) + if err != nil { + t.Fatalf("Convert without the neutral pages: %v", err) + } + with, err = doc.Convert(ctx, path, res, langs, doc.ConvertOptions{IncludeNeutralPages: true}) + if err != nil { + t.Fatalf("Convert with the neutral pages: %v", err) + } + t.Logf("without: %s", without.Summary()) + t.Logf("with: %s", with.Summary()) + return without, with +} + +// TestOptingOutIsTodaysConversionExactly is the constraint the whole feature is +// held to: a household that does not ask for these pages must get byte-identical +// output, because this shipped behind a gate people have already used. +// +// Asserted over the blocks AND the figures rather than over their counts. Two sets +// of the same size can differ, and a check on the totals would pass while a page +// swapped for another page. +func TestOptingOutIsTodaysConversionExactly(t *testing.T) { + // The sequential manual for Russian: 22 pages, and the neutral set is 7 pages + // carrying 61 figures sitting immediately in front of it, which is the largest + // change opting in could make and therefore the sharpest test that opting out + // makes none. + without, with := convertBothWays(t, "dreame-l40-ultra", "ru") + + if len(without.NeutralPages) != 0 { + t.Errorf("NeutralPages = %v without the option; want none", without.NeutralPages) + } + for _, p := range without.Pages { + if p < 7 { + t.Errorf("page %d is front matter and was converted without the option", p) + } + } + + // MEASURED HERE, NOT COPIED FROM PROSE. CLAUDE.md and conversion.md both say the + // sequential manual's Russian is "431 content blocks and 65 figures". The figure + // count is right and THE BLOCK COUNT IS STALE: 431 was the COLUMN manual's German + // before a page with no second column was read as two strips, and Russian measures + // 449 content blocks and 58 furniture on this commit. Pinning 431 here would have + // asserted another document's superseded number, which is the trap conversion.md + // records as "a total needs its sequence beside it". + // + // Read the sequence, not the value: a move in either number is a change in the + // funnel and not in this feature, because opting out cannot reach them. + // + // 449 IS NOW ALSO SUPERSEDED, which is the third entry in this number's sequence and + // the reason the paragraph above is kept rather than tidied away. Content is 409 + // because the 89 runs a figure's leaders point at left the block flow to be drawn + // beside the picture instead — see doc.Callouts. Nothing was lost: they are the 89 + // callout blocks, they are still in Conversion.Blocks, and verify.checkCoverage still + // counts every one of their characters. 409 + 89 = 498 against the 449 that used to + // be content, and the 49 extra blocks are labels that used to arrive glued INSIDE a + // paragraph and are now their own line. + // AND 409 IS SUPERSEDED IN TURN, by the wrapped-label tail: 404 content and 95 + // callouts. Six more runs left the flow and only five blocks went with them, + // because one of the six was already inside a bigger paragraph. Read the sequence + // 431 -> 449 -> 409 -> 404 rather than any one value. + // + // THESE TWO WERE ALREADY WRONG HERE when the band work started: the tail fix moved + // them at 3532d4b and this test was not re-run, because fixture tests need + // MANUALBOX_TEST_FIXTURES=1 and do not run in the default suite. Verified by + // stashing the band work and re-running this test against 3532d4b, where it fails + // with exactly these numbers. The band moves neither: it changes the crop and + // nothing about which runs leave the prose. + const ( + baselineBlocks = 404 + baselineFurniture = 58 + baselineCallouts = 95 + baselineFigures = 63 + ) + if got := len(without.ContentBlocks()); got != baselineBlocks { + t.Errorf("content blocks without the option = %d, want %d", got, baselineBlocks) + } + if got := len(without.FurnitureBlocks()); got != baselineFurniture { + t.Errorf("furniture blocks without the option = %d, want %d", got, baselineFurniture) + } + if got := len(without.CalloutBlocks()); got != baselineCallouts { + t.Errorf("callout blocks without the option = %d, want %d", got, baselineCallouts) + } + if got := len(without.Figures); got != baselineFigures { + t.Errorf("figures without the option = %d, want %d", got, baselineFigures) + } + // Every callout block's text is a label some figure carries, which is the invariant + // that says the two halves cannot drift: a run may only leave the flow because a + // picture prints it. + // Compared on collapsed whitespace, because a block's text goes through + // collapseSpaces and a label's through TrimSpace — the same characters, spaced + // differently. + norm := func(s string) string { return strings.Join(strings.Fields(s), " ") } + labels := map[string]bool{} + for i := range without.Figures { + for _, l := range without.Figures[i].Labels { + labels[norm(l)] = true + } + } + for _, b := range without.CalloutBlocks() { + if !labels[norm(b.Text)] { + t.Errorf("the callout block %q is not carried by any figure; it left the "+ + "block flow and reaches a reader nowhere", b.Text) + } + } + + // And every block the opted-in conversion holds for this language is one the + // opted-out conversion already held, in the same order. The neutral pages + // contribute no text — their regions are unnamed, so RegionsBlocks filters them + // out — and this is what says no language was invented for them anywhere. + if len(with.Blocks) != len(without.Blocks) { + t.Fatalf("blocks with the option = %d, without = %d; the neutral pages must "+ + "contribute pictures and no text", len(with.Blocks), len(without.Blocks)) + } + for i := range with.Blocks { + a, b := &without.Blocks[i], &with.Blocks[i] + if a.Page != b.Page || a.Index != b.Index || a.Lang != b.Lang || a.Text != b.Text { + t.Fatalf("block %d differs: page %d/%d idx %d/%d lang %q/%q", + i, a.Page, b.Page, a.Index, b.Index, a.Lang, b.Lang) + } + } +} + +// TestOptingInReachesTheDiagram is the point of the exercise. A Russian reader of +// the sequential manual gets the A-1 exploded parts plate, which 31 places in the +// document's own Russian and other content pages tell them to look at. +func TestOptingInReachesTheDiagram(t *testing.T) { + without, with := convertBothWays(t, "dreame-l40-ultra", "ru") + + // Nothing served page 5 before. That is the defect, asserted rather than recalled. + for i := range without.Figures { + if without.Figures[i].Page <= 6 { + t.Fatalf("page %d was already served; the premise of this feature is wrong", + without.Figures[i].Page) + } + } + + if len(with.NeutralPages) == 0 { + t.Fatal("the option converted no extra pages") + } + + plates := map[int]int{} + for i := range with.Figures { + f := &with.Figures[i] + if f.Page > 6 { + continue + } + plates[f.Page]++ + // EVERY ONE OF THEM IS NEUTRAL, and that is the funnel's promise holding rather + // than a detail: a picture on a page no language owns belongs to every language + // in scope. If one of these had acquired "ru" it would be invisible to a German + // household converted from the same document, which is the failure the brief + // forbade. + if !f.Neutral { + t.Errorf("figure on page %d claims languages %v; a page no language owns "+ + "must yield neutral pictures", f.Page, f.Langs) + } + } + if plates[5] == 0 { + t.Error("PDF page 5 is the A-1 diagram and none of its drawings arrived") + } + t.Logf("front-matter figures now served: page 5 = %d, page 6 = %d", + plates[5], plates[6]) + + // The set grew by the pictures and by nothing else. + if got := len(with.Figures) - len(without.Figures); got <= 0 { + t.Errorf("figures grew by %d; the plates carry 59 between them", got) + } +} + +// TestOptingInOnTheColumnManualAddsNothingWorthHaving is the other half of the +// measurement, kept as a test because it is the finding that nearly stopped this +// feature: on that document the pages no language owns are a print code and a page +// of service addresses, and they hold no pictures at all. +// +// It is not a failure. The gate offers the set and says what is on it, and the +// honest outcome for this document is a user who declines. What would be a failure +// is the option quietly adding blocks or figures here, so that is what is asserted. +func TestOptingInOnTheColumnManualAddsNothingWorthHaving(t *testing.T) { + without, with := convertBothWays(t, "thomas-drybox-amfibia", "de") + + if len(with.NeutralPages) != 2 { + t.Fatalf("NeutralPages = %v, want the two pages 67 and 68", with.NeutralPages) + } + if len(with.Figures) != len(without.Figures) { + t.Errorf("figures %d -> %d; those two pages hold none", + len(without.Figures), len(with.Figures)) + } + if len(with.Blocks) != len(without.Blocks) { + t.Errorf("blocks %d -> %d; the neutral pages contribute no text", + len(without.Blocks), len(with.Blocks)) + } +} diff --git a/internal/doc/pdf.go b/internal/doc/pdf.go new file mode 100644 index 0000000..d987b95 --- /dev/null +++ b/internal/doc/pdf.go @@ -0,0 +1,441 @@ +package doc + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" + + "github.com/gordon2/manualbox/internal/extern" +) + +// pageSeparator is what pdftotext writes between pages: a form feed. Splitting +// on it is what makes one invocation over the whole document equivalent to one +// invocation per page, at a fraction of the cost. +const pageSeparator = "\f" + +// Bounds on the poppler subprocesses. +// +// Neither existed at first, and the job context alone is not a bound: it is +// cancelled only at shutdown, while the worker renews its lease for as long as +// the handler runs. A pdftotext that never terminates therefore held a worker for +// ever, and the default pool is two workers — so two such documents stopped all +// ingest permanently. +// +// The limits are set from measurement with generous headroom. A 560-page, 15 MB +// manual takes 0.06 s for pdfinfo and 1.8 s for pdftotext, and yields 1.4 MB of +// text. +const ( + infoTimeout = 30 * time.Second + extractTimeout = 5 * time.Minute + // maxExtractedBytes caps the text held in memory. A PDF with heavily + // compressed content streams is a small upload that expands enormously, and + // the whole extraction is buffered before it is split into pages. + maxExtractedBytes = 64 << 20 +) + +// errOutputTooLarge is returned when a tool produces more output than the cap. +var errOutputTooLarge = errors.New("doc: extracted text exceeds the size limit") + +// limitedBuffer collects output up to a cap and then refuses more, so a runaway +// tool fails its job instead of exhausting the server's memory. +type limitedBuffer struct { + buf bytes.Buffer + limit int +} + +func (l *limitedBuffer) Write(p []byte) (int, error) { + if l.buf.Len()+len(p) > l.limit { + return 0, errOutputTooLarge + } + return l.buf.Write(p) +} + +// redact replaces a filesystem path with its final component. +// +// Blob paths sit under the data directory, which normally lives in a home +// directory and so carries an operating-system username. These messages reach +// `documents.last_error`, the API, and the log — and a user pasting that into a +// public issue is threat 2 in docs/design/privacy.md. The base name is the +// content digest, which identifies the document precisely and reveals nothing. +func redact(msg, path string) string { + if path == "" { + return msg + } + msg = strings.ReplaceAll(msg, path, filepath.Base(path)) + if dir := filepath.Dir(path); dir != "" && dir != "." && dir != string(filepath.Separator) { + msg = strings.ReplaceAll(msg, dir, "…") + } + return msg +} + +// Info is what stage 0 discovers: the free, instant facts about a document. +// +// This is deliberately the cheapest possible question. It decides whether the +// document is processable at all and whether it is large enough to need the +// user's permission before anything is spent on it. +type Info struct { + // Pages is the page count. + Pages int + // Encrypted reports whether the PDF is password-protected. An encrypted file + // cannot be extracted from and is stored as-is. + Encrypted bool + // Tagged reports whether the PDF carries structure tags. Tagged PDFs have + // usable reading order; untagged ones need geometry to recover it. + Tagged bool + // Producer and Creator identify the authoring tool, which is a useful hint + // about layout conventions. + Producer string + Creator string + // WidthPts and HeightPts are the first page's dimensions. + WidthPts, HeightPts float64 +} + +// ProbeInfo runs pdfinfo. Measured at 0.06 s on a 560-page, 15 MB document, so +// it is safe to call on upload rather than in a job. +func ProbeInfo(ctx context.Context, path string) (Info, error) { + bin, err := extern.Require(extern.PDFInfo) + if err != nil { + return Info{}, err + } + + ctx, cancel := context.WithTimeout(ctx, infoTimeout) + defer cancel() + + // #nosec G204 -- bin is resolved by extern from its own tool table; path is a + // blob-store path derived from a validated SHA-256 digest. + cmd := exec.CommandContext(ctx, bin, path) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + if err := cmd.Run(); err != nil { + return Info{}, fmt.Errorf("doc: pdfinfo failed: %w: %s", + err, redact(strings.TrimSpace(errOut.String()), path)) + } + + info := Info{} + for line := range strings.Lines(out.String()) { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + switch key { + case "Pages": + info.Pages, _ = strconv.Atoi(value) + case "Encrypted": + // pdfinfo prints "no" or a description of the encryption in use. + info.Encrypted = value != "no" + case "Tagged": + info.Tagged = value == "yes" + case "Producer": + info.Producer = value + case "Creator": + info.Creator = value + case "Page size": + info.WidthPts, info.HeightPts = parsePageSize(value) + } + } + if info.Pages <= 0 { + // The digest, never the directory: this message reaches the API and the log. + return Info{}, fmt.Errorf("doc: pdfinfo reported no pages for %s", filepath.Base(path)) + } + return info, nil +} + +// parsePageSize reads pdfinfo's "612.283 x 413.858 pts" form. +func parsePageSize(v string) (w, h float64) { + fields := strings.Fields(v) + if len(fields) < 3 { + return 0, 0 + } + w, _ = strconv.ParseFloat(fields[0], 64) + h, _ = strconv.ParseFloat(fields[2], 64) + return w, h +} + +// ErrNoTextLayer is returned when a document yields no extractable text at all, +// which means the OCR or vision path is required. +var ErrNoTextLayer = errors.New("doc: no text layer") + +// ExtractText runs pdftotext once over the whole document and splits the result +// into pages. +// +// One invocation, not one per page: measured at 1.76 s for 560 pages against +// roughly 0.1 s of process startup per page had it been called 560 times. The +// cost of extracting every page is low enough that there is no reason to sample. +func ExtractText(ctx context.Context, path string, pageCount int) ([]Page, error) { + bin, err := extern.Require(extern.PDFToText) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, extractTimeout) + defer cancel() + + // -enc UTF-8 is explicit rather than relying on the build default, because + // every downstream signal counts runes and a Latin-1 fallback would silently + // corrupt every non-Latin section. + // #nosec G204 -- see ProbeInfo. + cmd := exec.CommandContext(ctx, bin, "-enc", "UTF-8", path, "-") + out := &limitedBuffer{limit: maxExtractedBytes} + var errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = out, &errOut + if err := cmd.Run(); err != nil { + if errors.Is(err, errOutputTooLarge) { + return nil, fmt.Errorf("%w (limit %d bytes)", errOutputTooLarge, maxExtractedBytes) + } + return nil, fmt.Errorf("doc: pdftotext failed: %w: %s", + err, redact(strings.TrimSpace(errOut.String()), path)) + } + + chunks := strings.Split(out.buf.String(), pageSeparator) + // pdftotext emits a trailing separator after the final page, so the split + // leaves an empty tail. Drop it rather than reporting a phantom blank page. + if n := len(chunks); n > 0 && strings.TrimSpace(chunks[n-1]) == "" && n > pageCount { + chunks = chunks[:n-1] + } + + pages := make([]Page, 0, len(chunks)) + for i, body := range chunks { + pages = append(pages, newPage(i+1, body)) + } + return pages, nil +} + +// Page is one page of a document and everything the free signals could tell +// about it. +type Page struct { + // No is the 1-based page number in the original PDF. + No int + // Text is the extracted text, trimmed. + Text string + // Chars is the rune count. Deliberately runes and not bytes: a page of + // Cyrillic or CJK has roughly twice the bytes for the same amount of writing, + // so a byte-based threshold would judge scripts differently from each other. + Chars int + // Script is the dominant Unicode script, empty when there is no text. + Script string + // Tag is the language code the page prints on itself, empty when absent. + // Unvalidated at this stage: it is a candidate, not a conclusion. + Tag string + // TagCandidates holds every standalone code-shaped token on the page, in + // reading order. Needed because a right-to-left page does not put its tab + // first: pdftotext emits the section heading ahead of it, and on many pages + // it falls outside any small window from the top. Widening the window + // indiscriminately is unsafe — NO, IT, IS, AS, BE and MY are all valid + // language codes and all ordinary English words — so candidates are narrowed + // by cross-checking against the codes the printed index knows about. See + // [EffectiveTags]. + TagCandidates []string + // Folio is the page number printed in the page's own footer, which differs + // from No by the length of the front matter. Nil when the page prints none. + Folio *int + // Lang is the resolved language, filled in by reconciliation. + Lang string + // LangSource records which signal resolved Lang. + LangSource string + // Figures is how many pictures this page holds, and nil when nobody counted. + // + // A POINTER BECAUSE nil AND 0 ARE DIFFERENT ANSWERS. Only the pages no named + // language region claims are counted — see countNeutralInk, which explains why + // counting every page costs 42 s on a 560-page manual — so on a normal document + // almost every entry here is nil. Storing 0 for "nobody looked" would tell a gate + // that a diagram plate holds no pictures, which is the exact wrong answer, and it + // is the same absent-is-not-zero trap registry.FolioOffset records. + Figures *int +} + +// newPage derives the free per-page facts from extracted text. +func newPage(no int, body string) Page { + text := strings.TrimSpace(body) + p := Page{No: no, Text: text, Chars: len([]rune(text))} + if text == "" { + return p + } + p.Script = DominantScript(text) + p.Tag = pageTag(text) + p.TagCandidates = pageTagCandidates(text) + p.Folio = pageFolio(text) + return p +} + +// HasText reports whether the page yielded any extractable text. +func (p Page) HasText() bool { return p.Chars > 0 } + +// nonBlankLines returns up to limit trimmed, non-empty lines from the start of +// the text. +func nonBlankLines(text string, limit int) []string { + out := make([]string, 0, limit) + for line := range strings.Lines(text) { + line = strings.TrimSpace(line) + if line == "" { + continue + } + out = append(out, line) + if len(out) == limit { + break + } + } + return out +} + +// stripFormatting removes Unicode formatting characters, which carry no content +// but do break naive matching. +// +// This is not hygiene, it is required for correctness on right-to-left documents. +// A Hebrew or Arabic page wraps its Latin-script furniture in bidirectional +// embedding marks, so the language tab that reads "HE" is actually the five-rune +// sequence RLE LRE H E PDF PDF. Matching two ASCII letters against that fails, +// and the entire Hebrew and Arabic sections of a manual go unlabelled — which is +// exactly what happened before this existed. +// +// Category Cf covers the bidi controls (U+202A-U+202E, U+2066-U+2069), the +// directional marks (U+200E, U+200F), the byte-order mark and the soft hyphen. +// Zero-width space is category Zs and is stripped explicitly. +func stripFormatting(s string) string { + return strings.Map(func(r rune) rune { + if unicode.Is(unicode.Cf, r) || r == '​' { + return -1 + } + return r + }, s) +} + +// maxRunesInFolio bounds how long a line can be and still be a page number. +const maxRunesInFolio = 4 + +// pageFolio reads the page number printed on the page itself. +// +// It is the last purely numeric line, because a page number sits in the footer +// and pdftotext emits text in reading order. Folios are what let a printed +// index's claimed page be resolved to a real PDF page without assuming a global +// offset: on the measured fixture the offset is a constant +6, but that is a +// property of that document's front matter, not a constant of the format. +func pageFolio(text string) *int { + lines := strings.Split(text, "\n") + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(stripFormatting(lines[i])) + if line == "" || len([]rune(line)) > maxRunesInFolio { + continue + } + if n, err := strconv.Atoi(line); err == nil && n > 0 { + return &n + } + } + return nil +} + +// maxRunesInCodeLine bounds how long a line can be and still be considered a +// bare language tag. "ZH-HK" is the longest real example. +const maxRunesInCodeLine = 6 + +// pageTag finds a language code printed alone on its own line near the top of +// the page. +// +// Many manuals print each page's language in a corner tab, and pdftotext puts it +// first in reading order. It is the cheapest accurate language signal available — +// but it is a candidate only. Contents pages list language codes the same way and +// produce false positives, and a bare two-letter token like ON or TV is not a +// language at all. Both are filtered later, by run length and by script +// agreement. See docs/design/language-detection.md. +func pageTag(text string) string { + for _, line := range nonBlankLines(text, 3) { + line = strings.TrimSpace(stripFormatting(line)) + if line == "" || len([]rune(line)) > maxRunesInCodeLine { + continue + } + if !looksLikeLanguageCode(line) || !PlausibleCodeToken(line) { + continue + } + // A single letter cannot be trusted from position alone. On the measured + // 34-language manual, page 511 of the Cantonese section opens with a + // figure label "F" and carries its real ZH-HK tag on the next line; + // reading the F as French split that section in two. Keep looking — the + // real tag is often the line below — and let EffectiveTags adopt a single + // letter only where the document's own contents table lists it. + if singleLetterNeedsSupport(line) { + continue + } + return strings.ToUpper(line) + } + return "" +} + +// pageTagCandidates returns every standalone code-shaped token on the page, in +// reading order and deduplicated. +// +// Unlike [pageTag] this searches the whole page, because a right-to-left page's +// language tab is not near the start of the extracted text. The result is +// therefore permissive and must be narrowed before use — see [EffectiveTags]. +func pageTagCandidates(text string) []string { + var out []string + seen := make(map[string]bool, 4) + for line := range strings.Lines(text) { + line = strings.TrimSpace(stripFormatting(line)) + if line == "" || len([]rune(line)) > maxRunesInCodeLine { + continue + } + if !looksLikeLanguageCode(line) || !PlausibleCodeToken(line) { + continue + } + upper := strings.ToUpper(line) + if seen[upper] { + continue + } + seen[upper] = true + out = append(out, upper) + } + return out +} + +// looksLikeLanguageCode reports whether s has the shape of a printed language +// code. +// +// Manuals do not agree on the shape. The measured pair uses two-letter codes +// (EN, DE, ZH-HK) and one-and-three-letter ones (D, PL, RUS, UA, KAZ) — the +// second manual marks five languages and an exactly-two-letter matcher reads two +// of them. +// +// So one to three ASCII letters, optionally with a region. That is deliberately +// permissive and cannot be the whole test: a lone "D" is also a list marker, a +// size and a diagram label. Narrowing is [EffectiveTags]'s job, using the +// document's own contents-table vocabulary, and for a single letter that +// corroboration is required rather than preferred — see [singleLetterNeedsSupport]. +func looksLikeLanguageCode(s string) bool { + base, region, hasRegion := strings.Cut(s, "-") + if !isASCIILetters(base, 1, 3) { + return false + } + if hasRegion && !isASCIILetters(region, 2, 2) { + return false + } + return true +} + +// singleLetterNeedsSupport reports whether a code is too short to stand alone. +// +// "D" for German is real and common, but a single letter is the most ambiguous +// token on a page. It is believed only where something else agrees: the printed +// index listing it, or the column's own alphabet. +func singleLetterNeedsSupport(code string) bool { + base, _, _ := strings.Cut(code, "-") + return len(base) == 1 +} + +func isASCIILetters(s string, minLen, maxLen int) bool { + if len(s) < minLen || len(s) > maxLen { + return false + } + for i := range s { + if s[i] > unicode.MaxASCII || !unicode.IsLetter(rune(s[i])) { + return false + } + } + return true +} diff --git a/internal/doc/pdf_internal_test.go b/internal/doc/pdf_internal_test.go new file mode 100644 index 0000000..4d6f6ca --- /dev/null +++ b/internal/doc/pdf_internal_test.go @@ -0,0 +1,123 @@ +package doc + +import "testing" + +// These tests cover the text-shape parsing that the free language signals depend +// on. They are hermetic: no PDF, no poppler, no network. The real-document +// assertions live in fixture_test.go and skip by default. + +func TestPageTagReadsBidiWrappedCode(t *testing.T) { + // A right-to-left page wraps Latin-script furniture in bidirectional + // embedding marks, so the tab that reads "HE" is really RLE LRE H E PDF PDF. + // Missing this left the Hebrew and Arabic sections of a real manual + // unlabelled, so it is the regression most worth pinning down. + // \u202b RLE, \u202a LRE, \u202c PDF (pop directional formatting). + const rtlPage = "\u202bמידע בטיחותי\u202c\n\u202b\u202aHE\u202c\u202c\n\u202bיש לקרוא\u202c\n" + + if got := pageTag(rtlPage); got != "HE" { + t.Errorf("pageTag on a right-to-left page = %q, want %q", got, "HE") + } +} + +func TestPageTagVariants(t *testing.T) { + tests := []struct { + name, text, want string + }{ + {"plain first line", "EN\n21. Charging Contacts\n", "EN"}, + {"lowercase is normalised", "en\nSomething\n", "EN"}, + {"region subtag", "ZH-HK\n用戶手冊\n", "ZH-HK"}, + {"after a heading", "Safety Information\nAR\nbody text\n", "AR"}, + {"blank lines skipped", "\n\n\nDE\nBenutzerhandbuch\n", "DE"}, + {"no tag at all", "Just prose with no code on its own line.\n", ""}, + {"too far down the page", "aaa\nbbb\nccc\nEN\n", ""}, + {"a single letter needs corroboration", "F\nZH-HK\nbody\n", "ZH-HK"}, + {"word is not a code", "Contents\nOverview\n", ""}, + {"a real three-letter code is accepted", "ENG\nbody\n", "ENG"}, + {"a three-letter word is not", "ONE\nbody\n", ""}, + {"an ordinary word is not", "Fig\nbody\n", ""}, + {"digits are not a code", "01\nbody\n", ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := pageTag(tc.text); got != tc.want { + t.Errorf("pageTag(%q) = %q, want %q", tc.text, got, tc.want) + } + }) + } +} + +func TestPageTagCandidatesSearchesWholePage(t *testing.T) { + // Candidates are permissive by design: a right-to-left page can carry its tab + // well below the fold. Narrowing happens in EffectiveTags, against the codes + // the printed index declares. + text := "heading\nbody\nmore body\nyet more\nAR\ntrailing\n" + + if got := pageTag(text); got != "" { + t.Errorf("pageTag should not reach past the top of the page, got %q", got) + } + got := pageTagCandidates(text) + if len(got) != 1 || got[0] != "AR" { + t.Errorf("pageTagCandidates = %v, want [AR]", got) + } +} + +func TestPageFolio(t *testing.T) { + tests := []struct { + name, text string + want int // 0 means "expect none" + }{ + {"trailing number", "body text\nmore\n6\n", 6}, + {"bidi wrapped", "body\n\u202b\u202a194\u202c\u202c\n", 194}, + {"four digits allowed", "body\n1024\n", 1024}, + {"five digits rejected", "body\n10240\n", 0}, + {"no number", "body text only\n", 0}, + {"zero rejected", "body\n0\n", 0}, + {"last number wins", "12\nbody\n77\n", 77}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := pageFolio(tc.text) + switch { + case tc.want == 0 && got != nil: + t.Errorf("pageFolio(%q) = %d, want none", tc.text, *got) + case tc.want != 0 && got == nil: + t.Errorf("pageFolio(%q) = none, want %d", tc.text, tc.want) + case tc.want != 0 && *got != tc.want: + t.Errorf("pageFolio(%q) = %d, want %d", tc.text, *got, tc.want) + } + }) + } +} + +func TestStripFormattingKeepsContent(t *testing.T) { + // Stripping must remove only formatting. A Hebrew line stripped of bidi marks + // must still be the same Hebrew. + const withMarks = "\u202bמידע\u202c" + const wantText = "מידע" + if got := stripFormatting(withMarks); got != wantText { + t.Errorf("stripFormatting = %q, want %q", got, wantText) + } + if got := stripFormatting("plain ascii"); got != "plain ascii" { + t.Errorf("stripFormatting altered plain text: %q", got) + } +} + +func TestNewPageCountsRunesNotBytes(t *testing.T) { + // A byte count would judge Cyrillic and CJK pages as larger than equivalent + // Latin ones, and the text-layer threshold would then behave differently per + // script. Chars must be runes. + p := newPage(1, "Руководство") + if got, want := p.Chars, 11; got != want { + t.Errorf("Chars = %d, want %d runes (not bytes)", got, want) + } +} + +func TestParsePageSize(t *testing.T) { + w, h := parsePageSize("612.283 x 413.858 pts") + if w != 612.283 || h != 413.858 { + t.Errorf("parsePageSize = %v x %v, want 612.283 x 413.858", w, h) + } + if w, h := parsePageSize("nonsense"); w != 0 || h != 0 { + t.Errorf("parsePageSize on nonsense = %v x %v, want 0 x 0", w, h) + } +} diff --git a/internal/doc/reconcile.go b/internal/doc/reconcile.go new file mode 100644 index 0000000..110cff3 --- /dev/null +++ b/internal/doc/reconcile.go @@ -0,0 +1,372 @@ +package doc + +import ( + "fmt" + "strings" +) + +// signalPriority is the order in which signals are believed when they disagree +// about a page, most trusted first. +// +// The ordering is evidential, not arbitrary: +// +// - The printed page tag is the document asserting its own language, per page. +// It gives a label and a boundary simultaneously, which no other signal does. +// - The index gives a real language label, including for languages no detector +// supports, but its page claims are known to be 1-2 pages off. +// - Script is certain about what it can see and silent about the rest; it +// resolves the non-Latin sections and cannot separate the Latin ones. +// - A statistical detector would sit last, as the fallback for pages the free +// signals could not name. None is wired up; see +// docs/design/language-detection.md for why that decision is still open. +var signalPriority = []Source{SourcePageTag, SourceIndex, SourceScript, SourceDetector} + +// Reconcile combines the signals into the language map manualbox believes. +// +// The rule from docs/design/ingest.md, generalised past two signals: prefer the +// cheapest signal present, corroborate with the next, and where they conflict +// record the conflict rather than resolving it silently. A page nobody could name +// stays unnamed — that is a reportable state, not something to guess at. +func Reconcile(pages []Page, bySource map[Source][]Run) []Run { + if len(pages) == 0 { + return []Run{} + } + + // Index each signal's runs by page for O(1) lookup during resolution. + type claim struct { + code, lang string + confidence float64 + note string + } + claims := make(map[Source]map[int]claim, len(bySource)) + for source, runs := range bySource { + byPage := make(map[int]claim, len(pages)) + for _, r := range runs { + // A run that named no language contributes nothing to resolution, + // though it is still stored and reportable on its own terms. + if r.Lang == "" { + continue + } + for p := r.Start; p <= r.End; p++ { + byPage[p] = claim{r.Code, r.Lang, r.Confidence, r.Note} + } + } + claims[source] = byPage + } + + // Resolve each page independently, then group. Resolving per page and grouping + // afterwards is what lets a boundary fall wherever the evidence puts it, + // rather than inheriting a boundary from whichever signal was consulted first. + type resolution struct { + code, lang string + source Source + confidence float64 + note string + disputedBy []string + } + resolved := make(map[int]resolution, len(pages)) + + for i := range pages { + p := &pages[i] + if p.Chars < MinTextChars { + continue + } + var winner *resolution + for _, source := range signalPriority { + c, ok := claims[source][p.No] + if !ok { + continue + } + // A claim that contradicts the page's own script is not evidence. This + // is what stops the printed index's final entry — which claims every + // page to the end of the document — from labelling a Latin-script back + // cover as Japanese. + if !ScriptCompatible(p.Script, c.lang) { + continue + } + if winner == nil { + winner = &resolution{ + code: c.code, lang: c.lang, source: source, + confidence: c.confidence, note: c.note, + } + continue + } + // A lower-priority signal that names a different language is a + // disagreement worth recording, even though it does not win. + if !SameLanguage(winner.lang, c.lang) { + winner.disputedBy = append(winner.disputedBy, + fmt.Sprintf("%s says %s", source, c.code)) + } + } + if winner != nil { + resolved[p.No] = *winner + } + } + + // Group consecutive pages that resolved to the same language. + titles := indexTitles(bySource[SourceIndex]) + var runs []Run + // lastSource records which signal resolved the most recent page of each run, + // which is what decides whether a differently-specific label continues it. + var lastSource []Source + // disputes records, per run index, which pages were disputed and by what. + disputes := make(map[int]map[int][]string) + + for i := range pages { + p := &pages[i] + r, ok := resolved[p.No] + if !ok { + continue + } + n := len(runs) + if n > 0 && runs[n-1].End == p.No-1 && + continuesRun(runs[n-1].Lang, lastSource[n-1], r.lang, r.source) { + runs[n-1].End = p.No + if len(r.lang) > len(runs[n-1].Lang) { + runs[n-1].Lang, runs[n-1].Code = r.lang, r.code + } + lastSource[n-1] = r.source + } else { + runs = append(runs, Run{ + Source: SourceReconciled, Code: r.code, Lang: r.lang, + Start: p.No, End: p.No, Confidence: r.confidence, + Title: titles[r.code], + Note: fmt.Sprintf("%s: %s", r.source, r.note), + }) + lastSource = append(lastSource, r.source) + } + if len(r.disputedBy) > 0 { + idx := len(runs) - 1 + if disputes[idx] == nil { + disputes[idx] = make(map[int][]string, 2) + } + disputes[idx][p.No] = r.disputedBy + } + } + + // Flag disputes against the runs as the evidence produced them, BEFORE any + // bridging. Bridging extends a run past a page that used to be its edge, and + // an edge dispute — the ordinary one-page-off index claim — would then look + // interior. Identical evidence must not produce a different verdict because a + // photograph happened to sit inside the section. + flagDisputes(runs, disputes) + runs = bridgeLowTextGaps(runs, pages) + annotateIndexDisagreements(runs, bySource[SourceIndex]) + + if runs == nil { + return []Run{} + } + return runs +} + +// continuesRun decides whether a page extends the current run or starts a new one. +// +// Identical languages always continue. Two labels sharing a base language — +// "zh" and "zh-HK", "pt" and "pt-BR" — are the interesting case, and the answer +// depends on *which signal* was less specific: +// +// - The script signal cannot express a region at all; the most it can say for +// Han glyphs is "zh". When it fills a gap inside a section the page tag called +// ZH-HK, that is one section and the specific label is right for all of it. +// - A page tag or a printed index naming CN on one page and ZH-HK on another is +// the document distinguishing two sections. Merging them loses a real +// boundary and then relabels half the pages with a variant their own printed +// tag contradicts — so a household reading one variant is scoped onto both +// and pays to translate the wrong one. +func continuesRun(prevLang string, prevSource Source, curLang string, curSource Source) bool { + if prevLang == curLang { + return true + } + if !SameLanguage(prevLang, curLang) { + return false + } + // Same language, different specificity: allowed only when the vaguer of the + // two came from a signal incapable of being more precise. + vaguerSource := curSource + if len(curLang) > len(prevLang) { + vaguerSource = prevSource + } + return vaguerSource == SourceScript +} + +// bridgeLowTextGaps joins runs of the same language separated only by pages with +// too little text to classify. +// +// A page carrying nothing but a photograph sits between two pages of Japanese; it +// is Japanese, and leaving it out splits one section into two. On the measured +// fixture exactly this happened, and because the two fragments' page counts still +// summed correctly it was invisible to a test that checked only totals. +func bridgeLowTextGaps(runs []Run, pages []Page) []Run { + if len(runs) < 2 { + return runs + } + byNo := make(map[int]Page, len(pages)) + for i := range pages { + byNo[pages[i].No] = pages[i] + } + + out := []Run{runs[0]} + for i := 1; i < len(runs); i++ { + prev := &out[len(out)-1] + cur := runs[i] + + // Two conditions, both learned the hard way. + // + // There must be an actual gap: `cur.Start > prev.End` is satisfied by + // merely adjacent runs, so bridging silently re-joined two sections that + // grouping had just decided to keep apart. + // + // And the languages must match exactly, not merely share a base. Grouping + // separates a Portuguese section from a Brazilian Portuguese one on + // purpose; bridging must not put them back together. + bridgeable := prev.Lang == cur.Lang && cur.Start > prev.End+1 + for p := prev.End + 1; bridgeable && p < cur.Start; p++ { + page, known := byNo[p] + if !known || page.Chars >= MinTextChars { + bridgeable = false + } + } + if bridgeable { + prev.End = cur.End + if len(cur.Lang) > len(prev.Lang) { + prev.Lang, prev.Code = cur.Lang, cur.Code + } + // A conflict already established on either fragment survives the merge. + if cur.Conflict && !prev.Conflict { + prev.Conflict, prev.Note = true, cur.Note + } + continue + } + out = append(out, cur) + } + return out +} + +// conflictNote renders a note that keeps both the winning evidence and what +// disagreed with it, because a conflict the user cannot see is a conflict that +// was resolved silently. +func conflictNote(base string, disputedBy []string) string { + seen := make(map[string]bool, len(disputedBy)) + unique := make([]string, 0, len(disputedBy)) + for _, d := range disputedBy { + if seen[d] { + continue + } + seen[d] = true + unique = append(unique, d) + } + return fmt.Sprintf("%s; disagreed with by %s", base, strings.Join(unique, ", ")) +} + +// indexTitles maps a language code to the section title the printed index gave +// it. Titles are the one thing only the index can supply. +func indexTitles(indexRuns []Run) map[string]string { + titles := make(map[string]string, len(indexRuns)) + for i := range indexRuns { + if indexRuns[i].Title != "" { + titles[indexRuns[i].Code] = indexRuns[i].Title + } + } + return titles +} + +// flagDisputes marks a run as conflicting when the disagreement is about the run +// itself rather than about where its edge falls. +// +// Two cases count. A disagreement strictly inside a run is a real conflict: the +// signals name different languages for a page the run claims. A disagreement +// covering *every* page of a run is the same thing — and it was previously +// missed entirely, because a run of one or two pages has no strict interior, so +// a flat contradiction about a whole short section was silently dropped. Short +// sections exist in real manuals. +// +// What is deliberately not flagged is a single dispute at an edge of a longer +// run. A printed index whose claimed start is one page off disagrees about +// exactly one page — the last of the preceding section — and that is a boundary +// claim being slightly wrong, reported once per section by +// annotateIndexDisagreements. Flagging it again per run turned 8 of 35 runs on +// the measured fixture into "conflicts" and made the flag worthless. +func flagDisputes(runs []Run, disputes map[int]map[int][]string) { + for i := range runs { + byPage := disputes[i] + if len(byPage) == 0 { + continue + } + + var reasons []string + for page, by := range byPage { + if page > runs[i].Start && page < runs[i].End { + reasons = append(reasons, by...) + } + } + // Every page disputed: the signals disagree about the whole section, at + // any length. + if len(reasons) == 0 && len(byPage) == runs[i].Pages() { + for _, by := range byPage { + reasons = append(reasons, by...) + } + } + if len(reasons) == 0 { + continue + } + runs[i].Conflict = true + runs[i].Note = conflictNote(runs[i].Note, reasons) + } +} + +// indexStartTolerance is how far a printed index's claimed start may sit from the +// reconciled start before it is reported as a disagreement. +// +// Zero would be noise: the claim resolves through a printed folio, and a section +// beginning on a spread's verso legitimately shifts a page. Two pages is what the +// measured drift reached on a document whose index was otherwise correct, so +// beyond that the claim is wrong about something real. +const indexStartTolerance = 2 + +// annotateIndexDisagreements records where the printed index's claimed start +// disagrees with where the section actually begins. +// +// This is deliberately a note rather than a correction. The index being wrong is +// information about the document — it is exactly the failure the design says must +// be surfaced instead of resolved — and it is also how a user recognises that a +// manual's contents table cannot be trusted for navigation. +func annotateIndexDisagreements(runs, indexRuns []Run) { + claimed := make(map[string]Run, len(indexRuns)) + for _, r := range indexRuns { + if r.Start > 0 { + claimed[r.Code] = r + } + } + + firstSeen := make(map[string]int, len(runs)) + for i, run := range runs { + if _, ok := firstSeen[run.Code]; !ok { + firstSeen[run.Code] = i + } + } + + for code, i := range firstSeen { + claim, ok := claimed[code] + if !ok { + continue + } + delta := runs[i].Start - claim.Start + if delta < 0 { + delta = -delta + } + if delta <= indexStartTolerance { + continue + } + runs[i].Conflict = true + // Quote the page the index actually printed. claim.Start is the + // folio-resolved PDF page, which is a number the index never showed and + // which the reader cannot check against their copy. + printed := claim.Start + if claim.PrintedPage != nil { + printed = *claim.PrintedPage + } + runs[i].Note = fmt.Sprintf( + "%s; the printed index lists %s as starting on page %d, but it begins on PDF page %d", + runs[i].Note, code, printed, runs[i].Start) + } +} diff --git a/internal/doc/reconcile_test.go b/internal/doc/reconcile_test.go new file mode 100644 index 0000000..c8abb6f --- /dev/null +++ b/internal/doc/reconcile_test.go @@ -0,0 +1,181 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Regression tests for reconciliation defects found in review. Each was +// reproduced before being fixed. + +func TestTwoRegionalVariantsStayTwoSections(t *testing.T) { + // Grouping continued a run whenever two labels shared a base language, then + // kept whichever tag string was longer. That could not tell "a vaguer signal + // filled a gap in one section" from "the document named two different + // variants", so a Portuguese section and a Brazilian Portuguese section merged + // into one — and half its pages were then relabelled with a variant their own + // printed tag contradicts. A household reading one variant would be scoped + // onto both and pay to translate the wrong sixteen pages. + pages := []doc.Page{ + page(1, "PT", doc.ScriptLatin, 1, "texto em idioma iberico"), + page(2, "PT", doc.ScriptLatin, 2, "texto em idioma iberico"), + page(3, "BR", doc.ScriptLatin, 3, "texto em idioma brasileiro"), + page(4, "BR", doc.ScriptLatin, 4, "texto em idioma brasileiro"), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: { + {Source: doc.SourcePageTag, Code: "PT", Lang: "pt", Start: 1, End: 2, Confidence: 1}, + {Source: doc.SourcePageTag, Code: "BR", Lang: "pt-BR", Start: 3, End: 4, Confidence: 1}, + }, + } + + runs := doc.Reconcile(pages, bySource) + if len(runs) != 2 { + t.Fatalf("expected 2 runs, got %d — the document names two variants: %+v", len(runs), runs) + } + if runs[0].Lang != "pt" || runs[0].End != 2 { + t.Errorf("first run = %s %d-%d, want pt 1-2", runs[0].Lang, runs[0].Start, runs[0].End) + } + if runs[1].Lang != "pt-BR" || runs[1].Start != 3 { + t.Errorf("second run = %s %d-%d, want pt-BR 3-4", runs[1].Lang, runs[1].Start, runs[1].End) + } +} + +func TestAVaguerScriptSignalStillContinuesASection(t *testing.T) { + // The converse must keep working: the script signal cannot express a region, + // so when it fills a page inside a ZH-HK section that is still one section. + pages := []doc.Page{ + page(1, "ZH-HK", doc.ScriptHan, 1, "用戶手冊"), + page(2, "", doc.ScriptHan, 2, "用戶手冊"), + page(3, "ZH-HK", doc.ScriptHan, 3, "用戶手冊"), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: { + {Source: doc.SourcePageTag, Code: "ZH-HK", Lang: "zh-HK", Start: 1, End: 1, Confidence: 1}, + {Source: doc.SourcePageTag, Code: "ZH-HK", Lang: "zh-HK", Start: 3, End: 3, Confidence: 1}, + }, + doc.SourceScript: { + {Source: doc.SourceScript, Code: "ZH", Lang: "zh", Start: 1, End: 3, Confidence: 0.7}, + }, + } + + runs := doc.Reconcile(pages, bySource) + if len(runs) != 1 { + t.Fatalf("expected 1 run, got %d: %+v", len(runs), runs) + } + if runs[0].Lang != "zh-HK" || runs[0].Pages() != 3 { + t.Errorf("run = %s covering %d pages, want zh-HK covering 3", runs[0].Lang, runs[0].Pages()) + } +} + +func TestAWhollyDisputedShortRunIsFlagged(t *testing.T) { + // Interior-only flagging used strict inequality, so a run of one or two pages + // had no interior and a flat contradiction about the entire section was + // dropped — the silent resolution the design forbids. Short sections are real. + pages := []doc.Page{ + page(1, "DA", doc.ScriptLatin, 1, "dansk"), + page(2, "DA", doc.ScriptLatin, 2, "dansk"), + } + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "DA", Lang: "da", Start: 1, End: 2, Confidence: 1}}, + doc.SourceIndex: {{Source: doc.SourceIndex, Code: "FI", Lang: "fi", Start: 1, End: 2, Confidence: 0.6}}, + } + + runs := doc.Reconcile(pages, bySource) + if len(runs) != 1 { + t.Fatalf("expected 1 run, got %d", len(runs)) + } + if !runs[0].Conflict { + t.Errorf("a two-page run contradicted on every page was not flagged: %q", runs[0].Note) + } + if !strings.Contains(runs[0].Note, "FI") { + t.Errorf("the note should name the disagreeing claim, got %q", runs[0].Note) + } +} + +func TestBridgingDoesNotInventAConflict(t *testing.T) { + // Bridging extends a run past a page that used to be its edge, and interior + // flagging then saw that page as interior. Identical evidence produced + // opposite verdicts depending on whether a photograph happened to sit inside + // the section, which is exactly the false positive the flag exists to avoid. + bySource := func() map[doc.Source][]doc.Run { + return map[doc.Source][]doc.Run{ + doc.SourcePageTag: { + {Source: doc.SourcePageTag, Code: "IT", Lang: "it", Start: 1, End: 3, Confidence: 1}, + {Source: doc.SourcePageTag, Code: "IT", Lang: "it", Start: 5, End: 6, Confidence: 1}, + }, + // The ordinary one-page-early index claim, landing on page 3. + doc.SourceIndex: {{Source: doc.SourceIndex, Code: "ES", Lang: "es", Start: 3, End: 3, Confidence: 0.6}}, + } + } + + // Control: the same evidence with no gap to bridge. + control := []doc.Page{ + page(1, "IT", doc.ScriptLatin, 1, "italiano"), + page(2, "IT", doc.ScriptLatin, 2, "italiano"), + page(3, "IT", doc.ScriptLatin, 3, "italiano"), + } + controlRuns := doc.Reconcile(control, map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "IT", Lang: "it", Start: 1, End: 3, Confidence: 1}}, + doc.SourceIndex: {{Source: doc.SourceIndex, Code: "ES", Lang: "es", Start: 3, End: 3, Confidence: 0.6}}, + }) + if len(controlRuns) != 1 || controlRuns[0].Conflict { + t.Fatalf("control: an edge dispute should not flag a conflict, got %+v", controlRuns) + } + + // Same evidence, plus an illustration page inside the section. + withGap := []doc.Page{ + page(1, "IT", doc.ScriptLatin, 1, "italiano"), + page(2, "IT", doc.ScriptLatin, 2, "italiano"), + page(3, "IT", doc.ScriptLatin, 3, "italiano"), + thinPage(4), + page(5, "IT", doc.ScriptLatin, 5, "italiano"), + page(6, "IT", doc.ScriptLatin, 6, "italiano"), + } + runs := doc.Reconcile(withGap, bySource()) + if len(runs) != 1 { + t.Fatalf("expected the illustration page to be bridged, got %d runs", len(runs)) + } + if runs[0].Conflict { + t.Errorf("bridging turned an edge dispute into a conflict: %q", runs[0].Note) + } +} + +func TestIndexDisagreementNoteQuotesThePrintedPage(t *testing.T) { + // The note said "the printed index places X at page N" but passed the + // folio-resolved PDF page — a number the index never showed, which the reader + // cannot check against their own copy. + pages := []doc.Page{ + page(1, "", doc.ScriptLatin, 0, "cover"), + page(2, "", doc.ScriptLatin, 0, "cover"), + page(3, "NL", doc.ScriptLatin, 1, "nederlands"), + page(4, "NL", doc.ScriptLatin, 2, "nederlands"), + page(5, "NL", doc.ScriptLatin, 3, "nederlands"), + page(6, "NL", doc.ScriptLatin, 4, "nederlands"), + page(7, "NL", doc.ScriptLatin, 5, "nederlands"), + page(8, "NL", doc.ScriptLatin, 6, "nederlands"), + } + printed := 1 + bySource := map[doc.Source][]doc.Run{ + doc.SourcePageTag: {{Source: doc.SourcePageTag, Code: "NL", Lang: "nl", Start: 3, End: 8, Confidence: 1}}, + // The index claims printed page 1, which resolves to PDF page 3 — but it is + // recorded here as starting far away, so the tolerance is exceeded. + doc.SourceIndex: {{ + Source: doc.SourceIndex, Code: "NL", Lang: "nl", + Start: 7, End: 8, PrintedPage: &printed, Confidence: 0.6, + }}, + } + + for _, r := range doc.Reconcile(pages, bySource) { + if r.Code != "NL" || !r.Conflict { + continue + } + if !strings.Contains(r.Note, "page 1") { + t.Errorf("note should quote the printed page 1, got %q", r.Note) + } + return + } + t.Fatal("expected the NL run to be flagged as disagreeing with the index") +} diff --git a/internal/doc/regions.go b/internal/doc/regions.go new file mode 100644 index 0000000..7170fcf --- /dev/null +++ b/internal/doc/regions.go @@ -0,0 +1,513 @@ +package doc + +import ( + "fmt" + "math" + "strings" +) + +// A region is a language's territory on a page: the whole page where a manual +// runs its languages in sequence, a box where it runs them in parallel columns. +// See docs/design/regions.md for the contract this implements and what it +// deliberately leaves unsolved. +// +// The rule for when a page splits is the one decision here that could not be +// taken from the contract, because it needed both fixtures to settle: a page +// splits into boxes only when its columns name MORE THAN ONE language. Column +// count is not language count, in both directions, and both were measured: +// +// - The column manual sets two columns of one language on pages 6 to 10, and +// three of one language on 52 to 56. Its manifest calls this out precisely +// because column count identifies nothing on its own. +// - The sectioned manual reads as three columns on 199 of its 560 pages and +// four on 71, and every one of those is a side-by-side troubleshooting table. +// Pages 20 and 100 were rendered at 108 dpi and checked: the regions are the +// table's cells, correctly located. A table cell is not a language boundary — +// regions.md records that geometry cannot tell them apart and that the call +// belongs above this layer. This is where that call is made. +// +// Splitting on geometry alone would therefore store four regions for a page in +// one language, on hundreds of pages of a manual that has no parallel columns at +// all, and would make the sectioned document's storage depend on its table +// layout. Splitting on language keeps a whole-page region for every one of them, +// which is the compatibility stance regions.md asks for. + +// Region is one language's territory on a page. +type Region struct { + // Page is the 1-based page number in the original PDF. + Page int + // X0 and X1 bound the region horizontally, in the coordinate space + // [ExtractRuns] reports — 1.5 times the PDF's own points. + // + // A region covering the whole page runs from 0 to the page width. That is the + // compatibility stance rather than a convenience: a caller clipping text to + // the box gets the whole page, so a page-at-a-time reader needs no special + // case and no null check for "this one has no box". + X0, X1 float64 + // Code is the label as the document expresses it, which need not be a valid + // tag: real manuals print D, RUS, UA and KAZ. + Code string + // Lang is Code normalised to BCP-47, empty when nothing was established. + Lang string + // Source is the signal that named it, empty when none could. Empty is a real + // state and not a defect: a page of service addresses in six languages is + // genuinely unnameable, and saying so beats guessing. + Source Source + // Chars is the rune count of the text inside the box — the unit of size that + // replaces pages, since a page holding three languages cannot be a unit of + // anything. Runes, not bytes: half a real manual is Cyrillic, Greek or CJK, + // where the same amount of writing runs a third more bytes. + Chars int + // Runs is how many text runs the region holds. It is the density evidence, the + // same as [Column.Runs]: a region of five runs is page furniture. + Runs int + // Conflict marks a region whose printed tag and whose alphabet disagreed. + // Recorded, never resolved silently. + Conflict bool + // Note says in checkable terms how the region was read. + Note string +} + +// Width is the region's horizontal extent. +func (r *Region) Width() float64 { return r.X1 - r.X0 } + +// PageResolution is what the per-page pass already concluded about a page. The +// region reader defers to it, and the order of precedence is the whole design: +// see [PageRegions]. +type PageResolution struct { + // Code, Lang and Source are the reconciled per-page answer, empty when the + // per-page signals could not name the page. + Code string + Lang string + Source Source + // Contents marks a page the printed-index parser recognises as a contents + // table. Such a page is furniture: it lists other sections' languages and its + // own letters are a poor guide to anything, so a column's guess about it must + // not become the page's language. Measured — the sectioned manual's pages 2 to + // 5 are contents pages whose alphabet reads as Swedish and Turkish, and the + // column manual's page 68 of service addresses reads as Turkish. All three are + // wrong and all three are suppressed by this. + Contents bool +} + +// PageRegions divides one page into language regions. +// +// knownCodes is the vocabulary the document's own contents table declares, passed +// through to [ColumnLanguages]. resolved is what the per-page pass concluded. +// tables are the page's ruled tables, which the region reader needs for the reason +// [mergeCellColumns] gives; nil is a normal argument and means "the ruled lines +// were not read", not "this page has none". +// +// The precedence, in order, and every branch of it is measured against both +// fixtures: +// +// 1. A page the per-page signals named is one whole-page region in that language. +// Those signals are reconciled from the printed tab, the printed index and the +// page's script, and on the sectioned manual the tab alone is right on all 553 +// content pages. A column's alphabet reading must not overturn that: doing so +// split 31 of its pages and contradicted the tab on 46 regions, every one of +// them a short table cell — "de" read as Finnish, Spanish, Portuguese. Where +// the columns disagree the region records a conflict, because the +// disagreement is real information; it is not allowed to change the answer. +// 2. Otherwise, if the columns name more than one language, the page divides into +// one region per column. This is the parallel-columns manual, where the +// per-page signals name nothing at all on any of its eight verified pages — +// measured, and the reason the columns are trusted here rather than there. +// 3. Otherwise, one whole-page region taking the columns' single language, unless +// the page is a contents table, whose letters name nothing trustworthy. +// 4. Otherwise, one whole-page region with no language, which is a reportable +// state and not a failure. +// +// What this deliberately cannot do is divide a page that carries BOTH a whole-page +// printed tab and parallel columns of different languages: rule 1 would call it one +// language and record a conflict. Neither measured manual is that document — one +// prints per-page tabs and sets one language per page, the other prints per-column +// tabs and names no page — so the mechanism for it would be invented rather than +// designed. If a third manual does it, that is the stop condition, in the sense +// docs/design/regions.md uses the term. +func PageRegions(p *PageRuns, knownCodes map[string]bool, resolved PageResolution, tables []RuledTable) []Region { + // Rule 1 defers to the per-page answer because it is stronger evidence — but + // only where it actually names a language. A page-level answer of "fax", which + // this document really did produce for two of its pages, is a broken index + // parse wearing the shape of a language tag, and it must not outrank two columns + // that read correctly as German and Polish. See [KnownLanguage] for how that + // arises. The junk is still stored as what it is, an index run, where it stays + // inspectable; it simply stops being evidence about the page. + if !KnownLanguage(resolved.Lang) { + resolved.Code, resolved.Lang, resolved.Source = "", "", "" + } + + layout := DetectColumns(p.Runs, p.Width, p.Height) + cols := ColumnLanguages(p.Runs, mergeCellColumns(layout.Columns, tables, p.Width), knownCodes) + + // Size is counted over the runs the detector considers text, not every run in + // the file. The column manual's text layer carries 522 sub-legible InDesign + // slugs and parks 218 runs of a superseded address list above the top edge of + // one page; counting those overstates that page's size by half, measured. The + // filter is the same one DetectColumns applies internally, so a region's + // characters and its columns are drawn from the same set of runs. + // + // Naming deliberately still reads every run inside the box, which is what + // ColumnLanguages was measured against — a language is read from a sample of + // text, while size is a measurement of it, and the right set differs. A boxed + // region cannot come out with no characters despite a language, because a + // column exists only where at least minColumnRuns kept runs do. + var dropped DroppedRuns + kept := usableRuns(p.Runs, p.Width, p.Height, &dropped) + + // Rule 2, and only where rule 1 does not apply: the columns divide the page + // solely when the per-page pass had no answer of its own. + if named := distinctLanguages(cols); len(named) > 1 && resolved.Lang == "" { + out := make([]Region, 0, len(cols)) + for i := range cols { + out = append(out, boxedRegion(p, &cols[i], kept)) + } + return out + } + + region, ok := wholePageRegion(p, cols, kept, resolved) + if !ok { + return nil + } + return []Region{region} +} + +// mergeCellColumns folds the columns a table's own cell dividers created back into +// one column for the whole table, so that a table's area cannot decide where a +// page divides on language. +// +// This is where a table's area is kept out of region derivation, and it is the +// root of the one language error this document has been carrying. Measured on the +// column manual's page 57, whose four stored regions and whose two tables' cell +// columns are the same four boundaries within about five units: +// +// stored region table cell column +// 36-178, read as Finnish 29.7-173.3 table 1's question cells +// 179-424, German 173.3-428.1 table 1's answer cells +// 457-589, German 450.2-593.9 table 2's question cells +// 601-846, German 593.9-848.7 table 2's answer cells +// +// So that page has no language columns at all. It has two tables, and the column +// detector found their cell dividers. Reading the narrow question-cell column of +// the left table on its own is what produced Finnish: 289 runes of short German +// labels carrying ä and ö and no ü or ß. Read as one table the same text is +// plainly German. That is the second cause of this document's one language error, +// compounding the one already recorded — the printed D in the page's corner is +// rejected for want of an index vocabulary — and neither alone explains it. +// +// Why merging rather than subtracting the table's area: a [Region] is one x-range, +// so there is no shape in which a page's regions can have a table-sized hole in +// them. What can be excluded is the table's interior boundaries from the set of +// candidate region boundaries, which is exactly this. It also runs BEFORE any +// region exists, which is what docs/design/conversion.md requires — joining a +// table to a region afterwards would join it to boundaries it created itself. +// +// Two guards keep this from merging columns a table did not create, because the +// hazard runs the other way too: a small table printed across two genuine +// language columns must not weld them together. +// +// - Only columns lying inside the table's own box are candidates. +// - Every gutter merged away must have a cell divider in it. The tolerance is +// [minGutterFraction] of the page, 8.9 units on this manual's 892-unit page, +// against the 5.2 units by which page 57's widest coincidence misses. +// +// With no tables — the tool absent, or the page drawing none — this returns its +// input untouched, which is the compatibility property that matters most here: +// region derivation without ruled lines is bit-identical to what shipped. +func mergeCellColumns(cols []Column, tables []RuledTable, pageWidth float64) []Column { + if len(tables) == 0 || len(cols) < 2 { + return cols + } + + tol := minGutterFraction * pageWidth + out := make([]Column, 0, len(cols)) + for i := 0; i < len(cols); { + t := tableAround(&cols[i], tables) + if t == nil { + out = append(out, cols[i]) + i++ + continue + } + edges := cellDividers(t) + merged, j := cols[i], i + for j+1 < len(cols) && tableCovers(t, &cols[j+1]) && + nearAny(edges, (cols[j].Max+cols[j+1].Min)/2, tol) { + j++ + merged.Max = cols[j].Max + merged.Runs += cols[j].Runs + } + if j == i { + out = append(out, cols[i]) + i++ + continue + } + merged.Note = fmt.Sprintf("%d strips whose boundaries are the cell dividers of a "+ + "%d by %d ruled table, so they are one column and not one language each", + j-i+1, t.Rows, t.Cols) + out = append(out, merged) + i = j + 1 + } + return out +} + +// tableAround returns the first table whose box holds this column, or nil. +func tableAround(col *Column, tables []RuledTable) *RuledTable { + for i := range tables { + if tableCovers(&tables[i], col) { + return &tables[i] + } + } + return nil +} + +// tableCovers reports that a table's box spans a column horizontally. The +// tolerance is [runsInBox]'s own, since a column's extent comes from the runs +// inside it and a cell lets its text overhang the rule that bounds it. +func tableCovers(t *RuledTable, col *Column) bool { + return col.Min >= t.Box.X0-cellTextMargin && col.Max <= t.Box.X1+cellTextMargin +} + +// cellDividers returns the x positions at which a table's cells begin and end. +func cellDividers(t *RuledTable) []float64 { + out := make([]float64, 0, 2*len(t.Cells)) + for i := range t.Cells { + out = append(out, t.Cells[i].Rect.X0, t.Cells[i].Rect.X1) + } + return out +} + +func nearAny(at []float64, v, tol float64) bool { + for _, x := range at { + if math.Abs(x-v) <= tol { + return true + } + } + return false +} + +// namedByMinority reports that more of a page's columns declined to name a language +// than named one. +// +// A page named on the strength of one column out of three, where the other two read +// nothing, is being named on weak evidence. The measured case is the column manual's +// back page: three columns of service addresses in six languages, two declining and +// one reading as Turkish, which would then label the page Turkish. Its three columns +// are recorded in the fixture as establishing nothing, checked by eye. +// +// Measured across both documents before adopting: this describes exactly one page, +// that one, and no page of the sectioned manual. So it is a rule about weak +// evidence rather than a threshold tuned to a document. +// +// It is a separate guard from the contents-page one beside it, and both are needed: +// the contents guard is what stops the sectioned manual's pages 2 to 5 being named +// from their own letters, and this one is what stops an address page being named +// from a third of its columns. Neither document exercises both. +func namedByMinority(cols []ColumnLanguage) bool { + named, declined := 0, 0 + for i := range cols { + if cols[i].Lang == "" { + declined++ + continue + } + named++ + } + return named > 0 && declined > named +} + +// distinctLanguages returns the base languages the columns named, deduplicated. +// +// Base languages, not labels: a page whose columns are tagged ZH-HK and zh is one +// language in two notations, and splitting it would invent a boundary. A column +// that named nothing is not evidence of a second language and is not counted. +func distinctLanguages(cols []ColumnLanguage) []string { + seen := make(map[string]bool, len(cols)) + out := make([]string, 0, len(cols)) + for i := range cols { + if cols[i].Lang == "" { + continue + } + key := BaseLanguage(cols[i].Lang) + if key == "" || seen[key] { + continue + } + seen[key] = true + out = append(out, key) + } + return out +} + +// boxedRegion records one column of a page that holds several languages. +func boxedRegion(p *PageRuns, col *ColumnLanguage, kept []TextRun) Region { + inside := runsInBox(kept, col.Column.Min, col.Column.Max) + return Region{ + Page: p.No, + X0: col.Column.Min, + X1: col.Column.Max, + Code: col.Code, + Lang: col.Lang, + Source: col.Source, + Chars: countRunes(inside), + Runs: len(inside), + Conflict: col.Conflict, + Note: col.Note, + } +} + +// wholePageRegion records a page that is one region: rules 1, 3 and 4 of +// [PageRegions]. ok is false for a page with neither text nor a language, which is +// nothing to record. +func wholePageRegion(p *PageRuns, cols []ColumnLanguage, kept []TextRun, resolved PageResolution) (region Region, ok bool) { + if len(kept) == 0 && resolved.Lang == "" { + return Region{}, false + } + + region = Region{ + Page: p.No, + X0: 0, + X1: p.Width, + Code: resolved.Code, + Lang: resolved.Lang, + Source: resolved.Source, + Chars: countRunes(kept), + Runs: len(kept), + } + + columnLangs := distinctLanguages(cols) + + switch { + case region.Lang != "": + // Rule 1. The per-page answer stands. A column naming a different language + // is recorded as a conflict and changes nothing — see [PageRegions] for the + // measurement that settled this direction. + var disputes []string + for _, lang := range columnLangs { + if !SameLanguage(lang, region.Lang) { + disputes = append(disputes, DisplayName(lang)) + } + } + if len(disputes) > 0 { + region.Conflict = true + region.Note = fmt.Sprintf("the page reads as %s, but %d of its %d columns read as %s", + DisplayName(region.Lang), len(disputes), len(cols), strings.Join(disputes, " and ")) + } + + case len(columnLangs) == 1 && !resolved.Contents && !namedByMinority(cols): + // Rule 3. The columns agree on one language the page-level pass could not + // find, which on the column manual is how a single-column page and a page of + // two same-language columns are named at all. + for i := range cols { + if cols[i].Lang == "" { + continue + } + region.Code, region.Lang, region.Source = cols[i].Code, cols[i].Lang, cols[i].Source + region.Conflict = cols[i].Conflict + region.Note = cols[i].Note + break + } + } + + if region.Note == "" { + region.Note = wholePageNote(len(cols), region.Lang, columnLangs, resolved.Contents) + } + return region, true +} + +func wholePageNote(columns int, lang string, columnLangs []string, contents bool) string { + switch { + case lang == "" && contents && len(columnLangs) > 0: + // Say that something was read and refused, or this looks like a page nothing + // could be made of. + return fmt.Sprintf("a contents page, whose letters read as %s; too weak a "+ + "guide to name the page by", strings.Join(columnLangs, " and ")) + case lang == "" && len(columnLangs) > 0: + // The other refusal: something was read, by too few of the page's columns. + return fmt.Sprintf("read as %s, but by a minority of this page's %d columns; "+ + "left unnamed rather than named on that", strings.Join(columnLangs, " and "), columns) + case lang == "": + return "no language established for this page" + case columns > 1: + // Worth saying, because it is the case that looks like a mistake and is not: + // several columns, one language, so the page is not divided. + return fmt.Sprintf("%d columns, all of them %s, so the whole page is one region", + columns, DisplayName(lang)) + default: + return fmt.Sprintf("the whole page is %s", DisplayName(lang)) + } +} + +// runsInBox returns the runs lying within a horizontal range. +// +// A run belongs to the box its left edge sits in, and a run crossing the boundary +// spans boxes and belongs to neither. The tolerance absorbs the rounding between +// a column's reported extent and the runs that produced it. +// +// This is the single definition of that membership, shared with the column +// language reader: a region's characters must be counted over the same runs its +// language was read from, or the two would describe different text. +func runsInBox(runs []TextRun, x0, x1 float64) []TextRun { + var inside []TextRun + for i := range runs { + r := &runs[i] + if r.X >= x0-1 && r.right() <= x1+1 { + inside = append(inside, *r) + } + } + return inside +} + +func countRunes(runs []TextRun) int { + n := 0 + for i := range runs { + n += len([]rune(runs[i].Text)) + } + return n +} + +// RegionChars totals the characters of the regions in the given languages. +// +// Keyed on base language for the reason ScopeFor was: a summary carries one label +// per language while the regions each carry their own, so keying on the label +// counted a document printing CN, JA and ZH-HK as three languages in pages and one +// in characters. +func RegionChars(regions []Region, inScope map[string]bool) int { + chars := 0 + for i := range regions { + if inScope[BaseLanguage(regions[i].Lang)] { + chars += regions[i].Chars + } + } + return chars +} + +// RegionSummary describes the regions of a document in one line, for logs and for +// a test that wants the shape rather than every row. +func RegionSummary(regions []Region) string { + if len(regions) == 0 { + return "no regions" + } + chars := 0 + perPage := make(map[int]int, len(regions)) + langs := make(map[string]bool, 8) + for i := range regions { + r := ®ions[i] + perPage[r.Page]++ + chars += r.Chars + if r.Lang != "" { + langs[BaseLanguage(r.Lang)] = true + } + } + // A region is boxed when its page carries more than one, which is robust where + // testing x0 against zero is not: a leftmost column may legitimately begin at + // the page's left edge. + boxed := 0 + for _, n := range perPage { + if n > 1 { + boxed += n + } + } + pages := perPage + var b strings.Builder + fmt.Fprintf(&b, "%d regions over %d pages, %d of them boxed, %d languages, %d chars", + len(regions), len(pages), boxed, len(langs), chars) + return b.String() +} diff --git a/internal/doc/regions_fixture_test.go b/internal/doc/regions_fixture_test.go new file mode 100644 index 0000000..8c79de8 --- /dev/null +++ b/internal/doc/regions_fixture_test.go @@ -0,0 +1,227 @@ +package doc_test + +import ( + "context" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/fixture" +) + +// The acceptance tests docs/design/regions.md asks for, in its own words: the +// column manual's five languages must read back across its parallel columns with +// the eight human-verified pages matching column for column, and the sectioned +// manual's 34 sequential sections must be unchanged in what they report. A change +// that improves the second by altering the first has broken something, so both are +// asserted here rather than one. + +func analyzeColumnFixture(t *testing.T) (manifest *fixture.Manifest, result *doc.Result) { + t.Helper() + m, path := columnFixture(t) + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if res.RegionNote != "" { + t.Fatalf("no regions were produced: %s", res.RegionNote) + } + return m, res +} + +// TestRegionsSplitTheColumnManualByLanguage is the first half of acceptance: a +// page holding three languages must yield three regions, and one holding three +// columns of a single language must yield one. +func TestRegionsSplitTheColumnManualByLanguage(t *testing.T) { + m, res := analyzeColumnFixture(t) + + byPage := make(map[int][]doc.Region, m.Pages) + for i := range res.Regions { + r := &res.Regions[i] + byPage[r.Page] = append(byPage[r.Page], *r) + } + t.Logf("%s", doc.RegionSummary(res.Regions)) + + for _, want := range m.VerifiedPages() { + got := byPage[want.Page] + + // How many languages the render shows on this page, from the ground truth. + langs := make(map[string]bool, len(want.Cols)) + for _, c := range want.Cols { + if c.Lang != "" { + langs[c.Lang] = true + } + } + + switch { + case len(langs) > 1: + // A genuinely multi-language page: one region per column, in order, each + // boxed at the column the human confirmed. + if len(got) != len(want.Cols) { + t.Errorf("page %d holds %d languages in %d columns but produced %d regions", + want.Page, len(langs), len(want.Cols), len(got)) + continue + } + for i, wantCol := range want.Cols { + if int(got[i].X0) != wantCol.X0 || int(got[i].X1) != wantCol.X1 { + t.Errorf("page %d region %d: x=%.0f-%.0f, the render shows the column at %d-%d", + want.Page, i, got[i].X0, got[i].X1, wantCol.X0, wantCol.X1) + } + if wantCol.Lang != "" && !doc.SameLanguage(got[i].Lang, wantCol.Lang) { + t.Errorf("page %d region %d: %s, the manifest says %s — %s", + want.Page, i, got[i].Lang, wantCol.Lang, got[i].Note) + } + if got[i].Chars == 0 { + t.Errorf("page %d region %d holds no characters, but a column needs "+ + "runs to exist at all", want.Page, i) + } + } + + default: + // One language, however many columns it is set in. Pages 6 and 12 of this + // manual are the case: two columns of German, and one column beside a + // full-height image. Both are one region covering the page. + if len(got) != 1 { + t.Errorf("page %d holds %d language across %d columns but produced %d regions; "+ + "column count is not language count", want.Page, len(langs), len(want.Cols), len(got)) + continue + } + if got[0].X0 != 0 || got[0].X1 == 0 { + t.Errorf("page %d: a whole-page region must span 0 to the page width, got %.0f-%.0f", + want.Page, got[0].X0, got[0].X1) + } + } + } +} + +// TestRegionsFindEveryLanguageOfTheColumnManual asserts the document-level claim: +// all five languages are present in the regions. Before regions, a page could +// carry only one language, so at most one of the three on page 2 could be stored. +func TestRegionsFindEveryLanguageOfTheColumnManual(t *testing.T) { + m, res := analyzeColumnFixture(t) + + found := make(map[string]int, 8) + for i := range res.Regions { + if lang := doc.BaseLanguage(res.Regions[i].Lang); lang != "" { + found[lang]++ + } + } + + for _, want := range m.Languages { + if found[want] == 0 { + t.Errorf("%s is in the manifest but no region records it", want) + } + } + for lang, n := range found { + t.Logf(" %-3s %d regions", lang, n) + } + + // Every language of a parallel-columns manual must reach a boxed region + // somewhere, or the columns were never really separated. + boxed := 0 + for i := range res.Regions { + if res.Regions[i].X0 != 0 { + boxed++ + } + } + if boxed == 0 { + t.Error("no region is boxed; the whole point of this manual is that a page holds several") + } +} + +// TestRegionsLeaveTheSectionedManualUnchanged is the other half of acceptance, and +// the one that fails if the column work was bought at the sectioned manual's +// expense: every page holds one language, so every page must be one whole-page +// region carrying exactly the language the per-page map already believed. +func TestRegionsLeaveTheSectionedManualUnchanged(t *testing.T) { + m, path := loadFixture(t) + res, err := doc.Analyze(context.Background(), path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if res.RegionNote != "" { + t.Skipf("no regions were produced: %s", res.RegionNote) + } + t.Logf("%s", doc.RegionSummary(res.Regions)) + + // The language map itself must be untouched. This repeats what + // TestLanguageMapMatchesManifest asserts, deliberately: that test would still + // pass if regions quietly disagreed with it, and the two must not diverge. + summaries := res.Languages() + if len(summaries) != len(m.Sections) { + t.Errorf("%d languages after regions landed, manifest records %d", + len(summaries), len(m.Sections)) + } + + perPage := make(map[int]int, m.Pages) + for i := range res.Regions { + perPage[res.Regions[i].Page]++ + } + + // Not one region per page in general — a page with no text at all is nothing to + // record — but never more than one, because no page of this manual holds two + // languages. 199 of its pages read as three columns, and if those became three + // regions each this is where it would show. + var split []int + for page, n := range perPage { + if n > 1 { + split = append(split, page) + } + } + if len(split) > 0 { + show := split + if len(show) > 10 { + show = show[:10] + } + t.Errorf("%d pages produced more than one region, e.g. %v; every page of this "+ + "manual is a single language, and its multi-column pages are tables", + len(split), show) + } + + // Each region's language must be the one the per-page map resolved, or regions + // have become a second opinion rather than a finer-grained record of the same one. + disagreed := 0 + for i := range res.Regions { + r := &res.Regions[i] + want, _ := res.PageLang(r.Page) + if want != r.Lang { + if disagreed < 5 { + t.Errorf("page %d: region says %q, the page map says %q", r.Page, r.Lang, want) + } + disagreed++ + } + } + if disagreed > 0 { + t.Errorf("%d regions disagree with the reconciled page language", disagreed) + } +} + +// TestScopeCharsCountOnlyTheColumnsInScope is what the whole change is for. A +// household reading one of a page's three languages should be charged for its +// column, not for the page. +func TestScopeCharsCountOnlyTheColumnsInScope(t *testing.T) { + _, res := analyzeColumnFixture(t) + + german := res.ScopeFor([]string{"de"}) + all := res.ScopeFor([]string{"de", "pl", "ru", "uk", "kk"}) + + if german.Chars == 0 || all.Chars == 0 { + t.Fatalf("no characters counted: de=%d, all=%d", german.Chars, all.Chars) + } + t.Logf("German alone: %d chars; all five languages: %d chars (%.0f%%)", + german.Chars, all.Chars, 100*float64(german.Chars)/float64(all.Chars)) + + // The strict inequality is the assertion. Before regions both numbers were the + // same, because a page in scope contributed all of its characters however many + // languages shared it. + if german.Chars >= all.Chars { + t.Errorf("one language of five counts %d characters and all five count %d; "+ + "a single column cannot be the whole page", german.Chars, all.Chars) + } + + // Sanity on the magnitude rather than a tuned figure: five languages sharing a + // document, so one of them should be a minority of the text by some real margin. + if fraction := float64(german.Chars) / float64(all.Chars); fraction > 0.6 { + t.Errorf("German is %.0f%% of the document's in-scope characters, which is too "+ + "much of a five-language manual to be one language's columns", 100*fraction) + } +} diff --git a/internal/doc/regions_test.go b/internal/doc/regions_test.go new file mode 100644 index 0000000..e31198d --- /dev/null +++ b/internal/doc/regions_test.go @@ -0,0 +1,398 @@ +package doc_test + +import ( + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for the four rules in doc.PageRegions. No PDF and no poppler: +// runs are built here, so the rules are stated where they can be read. The +// real-document acceptance lives in regions_fixture_test.go, and every case below +// is drawn from something one of those two manuals actually does. + +const testRegionPageWidth = 892 + +// regionPage builds a page whose columns each hold the given lines, spaced so the +// projection finds a real gutter between them and each column clears the minimum +// run count. +func regionPage(no int, columns ...[]string) *doc.PageRuns { + p := &doc.PageRuns{No: no, Width: testRegionPageWidth, Height: 850} + for i, lines := range columns { + x := 30 + float64(i)*290 + for j, line := range lines { + p.Runs = append(p.Runs, doc.TextRun{ + X: x, Y: float64(20 + j*18), Width: 250, Height: 14, Text: line, + }) + } + } + return p +} + +// fill repeats a line enough times to make a column, since eight runs is the floor. +func fill(line string, n int) []string { + out := make([]string, n) + for i := range out { + out[i] = line + } + return out +} + +func onlyRegion(t *testing.T, regions []doc.Region) doc.Region { + t.Helper() + if len(regions) != 1 { + t.Fatalf("got %d regions, want 1", len(regions)) + } + return regions[0] +} + +// TestPageRegionsPageLevelAnswerWins is rule 1. This is the sectioned manual's +// every page: the printed tab names the whole page, and a short table cell whose +// alphabet reads as something else must not overturn it. +func TestPageRegionsPageLevelAnswerWins(t *testing.T) { + page := regionPage(7, fill(german, 10), fill(polish, 10)) + got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{ + Code: "DE", Lang: "de", Source: doc.SourceReconciled, + }, nil)) + + if got.Lang != "de" { + t.Errorf("region language = %q, want de from the page-level answer", got.Lang) + } + if got.X0 != 0 || got.X1 != testRegionPageWidth { + t.Errorf("region spans %.0f-%.0f, want the whole page 0-%d", + got.X0, got.X1, testRegionPageWidth) + } + // The disagreement must survive as a conflict. Overriding a column silently is + // what the design forbids; overriding it and saying so is the decision. + if !got.Conflict { + t.Error("a column read as another language and no conflict was recorded") + } + if got.Note == "" { + t.Error("the conflict has no note saying what disagreed") + } +} + +// TestPageRegionsIgnoreAPageLevelAnswerThatNamesNoLanguage is the FAX case, +// measured on the column manual: its service-address page is read as a contents +// table, FAX becomes an index entry, "fax" parses as a language tag, and two pages +// were labelled with it over columns that read correctly. +func TestPageRegionsIgnoreAPageLevelAnswerThatNamesNoLanguage(t *testing.T) { + page := regionPage(46, fill(german, 10), fill(polish, 10)) + got := doc.PageRegions(page, nil, doc.PageResolution{ + Code: "FAX", Lang: "fax", Source: doc.SourceReconciled, + }, nil) + + if len(got) != 2 { + t.Fatalf("got %d regions, want the two columns to decide the page", len(got)) + } + for i, want := range []string{"de", "pl"} { + if got[i].Lang != want { + t.Errorf("region %d = %q, want %q", i, got[i].Lang, want) + } + } +} + +// TestPageRegionsSplitOnLanguage is rule 2: the column manual's page 2, three +// languages side by side. +func TestPageRegionsSplitOnLanguage(t *testing.T) { + page := regionPage(2, fill(german, 10), fill(polish, 10), fill(rus, 10)) + got := doc.PageRegions(page, nil, doc.PageResolution{}, nil) + + if len(got) != 3 { + t.Fatalf("got %d regions, want 3", len(got)) + } + for i, want := range []string{"de", "pl", "ru"} { + if got[i].Lang != want { + t.Errorf("region %d = %q, want %q", i, got[i].Lang, want) + } + if got[i].Page != 2 { + t.Errorf("region %d is on page %d, want 2", i, got[i].Page) + } + if got[i].Chars == 0 { + t.Errorf("region %d holds no characters", i) + } + if got[i].Runs != 10 { + t.Errorf("region %d holds %d runs, want the 10 written into it", i, got[i].Runs) + } + } + + // Boxes must be ordered and disjoint, because the natural key is the page and + // the left edge: two regions sharing an x0 would collide in storage. + for i := 1; i < len(got); i++ { + if got[i].X0 <= got[i-1].X0 { + t.Errorf("region %d starts at %.0f, not to the right of region %d at %.0f", + i, got[i].X0, i-1, got[i-1].X0) + } + if got[i].X0 < got[i-1].X1 { + t.Errorf("regions %d and %d overlap: %.0f-%.0f and %.0f-%.0f", + i-1, i, got[i-1].X0, got[i-1].X1, got[i].X0, got[i].X1) + } + } +} + +// TestPageRegionsDoNotSplitColumnsOfOneLanguage is the other half of rule 2, and +// the reason the rule is about language and not geometry: the column manual sets +// two columns of German on pages 6 to 10 and three of Polish on 53, and the +// sectioned manual sets hundreds of pages as side-by-side tables. +func TestPageRegionsDoNotSplitColumnsOfOneLanguage(t *testing.T) { + page := regionPage(6, fill(german, 10), fill(german, 10)) + got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, nil)) + + if got.Lang != "de" { + t.Errorf("region language = %q, want de", got.Lang) + } + if got.X0 != 0 || got.X1 != testRegionPageWidth { + t.Errorf("two columns of one language gave a boxed region %.0f-%.0f, want the whole page", + got.X0, got.X1) + } + // Both columns' text must be counted, or the page's size is understated by + // however many columns it is set in. + if got.Runs != 20 { + t.Errorf("region holds %d runs, want all 20 across both columns", got.Runs) + } +} + +// TestPageRegionsNameASingleColumnPage is rule 3: the column manual's page 12 is +// one column beside a full-height image, and nothing per-page names it. +func TestPageRegionsNameASingleColumnPage(t *testing.T) { + page := regionPage(12, fill(ukr, 10)) + got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, nil)) + + if got.Lang != "uk" { + t.Errorf("region language = %q, want uk from the column", got.Lang) + } + if got.X1 != testRegionPageWidth { + t.Errorf("region ends at %.0f, want the page width", got.X1) + } +} + +// TestPageRegionsRefuseToNameAContentsPage is the exception to rule 3. Measured: +// the sectioned manual's contents pages 2 to 5 read as Swedish and Turkish, and the +// column manual's page of service addresses reads as Turkish. All are wrong. +func TestPageRegionsRefuseToNameAContentsPage(t *testing.T) { + page := regionPage(2, fill(german, 10)) + got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{Contents: true}, nil)) + + if got.Lang != "" { + t.Errorf("a contents page was named %q from its own letters", got.Lang) + } + // It must still say what it read and refused, or this is indistinguishable from + // a page nothing could be made of. + if got.Note == "" { + t.Error("no note explains why the page was left unnamed") + } + // And its characters still count: the text is there whether or not it was named. + if got.Chars == 0 { + t.Error("a contents page's characters were not counted") + } +} + +// TestPageRegionsLeaveAnUnnameablePageUnnamed is rule 4. The column manual's back +// page of service addresses in six languages is genuinely unnameable, and saying so +// is the honest outcome. +func TestPageRegionsLeaveAnUnnameablePageUnnamed(t *testing.T) { + page := regionPage(68, fill("Service 1234 5678 90", 10)) + got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, nil)) + + if got.Lang != "" { + t.Errorf("region language = %q, want none established", got.Lang) + } + if got.Chars == 0 { + t.Error("an unnamed page's characters were not counted; size does not depend on naming") + } +} + +func TestPageRegionsSkipAPageWithNothingOnIt(t *testing.T) { + page := &doc.PageRuns{No: 3, Width: testRegionPageWidth, Height: 850} + if got := doc.PageRegions(page, nil, doc.PageResolution{}, nil); len(got) != 0 { + t.Errorf("got %d regions for an empty page, want none", len(got)) + } +} + +// TestRegionCharsExcludeWhatIsNotText guards the measurement the size unit rests +// on. The column manual's text layer carries 522 sub-legible production slugs and +// parks 218 runs above the top edge of one page; counting those overstates that +// page by half. +func TestRegionCharsExcludeWhatIsNotText(t *testing.T) { + page := regionPage(9, fill(german, 10)) + clean := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, nil)) + + page.Runs = append(page.Runs, + // A production slug: real text in the file, two units tall, invisible on paper. + doc.TextRun{ + X: 30, Y: 400, Width: 250, Height: 2, + Text: "Job_4417_Manual_v3_export_2019-11-08.indd 1 08.11.19 10:16", + }, + // A run parked above the page, which is where a superseded address list lives. + doc.TextRun{ + X: 30, Y: -38, Width: 250, Height: 14, Text: "Superseded address list line", + }) + + withJunk := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, nil)) + if withJunk.Chars != clean.Chars { + t.Errorf("characters went from %d to %d when a sub-legible slug and an off-page "+ + "run were added; neither is text on the page", clean.Chars, withJunk.Chars) + } +} + +// TestRegionCharsCountRunesNotBytes is the convention this project has already been +// bitten by: half a real manual is Cyrillic or CJK, where the same writing runs a +// third more bytes. +func TestRegionCharsCountRunesNotBytes(t *testing.T) { + latin := onlyRegion(t, doc.PageRegions(regionPage(1, fill("aaaaa", 10)), nil, doc.PageResolution{}, nil)) + cyrillic := onlyRegion(t, doc.PageRegions(regionPage(1, fill("ааааа", 10)), nil, doc.PageResolution{}, nil)) + + if latin.Chars != cyrillic.Chars { + t.Errorf("five Latin letters counted %d and five Cyrillic %d; runes, not bytes", + latin.Chars, cyrillic.Chars) + } +} + +// TestPageRegionsRefuseAPageNamedByAMinorityOfItsColumns is the column manual's +// back page: three columns of service addresses in six languages, one of which the +// alphabet reads as Turkish while the other two decline. Naming the page from a +// third of it is naming it on weak evidence. +// +// This was previously suppressed by accident — the address page was misread as a +// contents table, and the contents guard caught it. Fixing the index parser removed +// that accident and exposed the reading, which is why the refusal is now explicit. +func TestPageRegionsRefuseAPageNamedByAMinorityOfItsColumns(t *testing.T) { + page := regionPage(68, + fill("Service 1234 5678 90", 10), + fill("Servis 9876 5432 10", 10), + fill("Huolto 5555 4444 33 Jyväskylä Töölö", 10), + ) + got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, nil)) + + if got.Lang != "" { + t.Errorf("page named %q from one of three columns; the other two established nothing", + got.Lang) + } + if !strings.Contains(got.Note, "minority") { + t.Errorf("note does not say why the reading was refused: %q", got.Note) + } + if got.Chars == 0 { + t.Error("the page's characters must still be counted; size does not depend on naming") + } +} + +// TestPageRegionsStillNameAPageAllOfWhoseColumnsAgree is the other side of that +// guard: it must not refuse the ordinary case of two columns in one language, which +// is pages 6 to 10 of the same manual. +func TestPageRegionsStillNameAPageAllOfWhoseColumnsAgree(t *testing.T) { + page := regionPage(6, fill(german, 10), fill(german, 10)) + if got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, nil)); got.Lang != "de" { + t.Errorf("region language = %q, want de; both columns named it", got.Lang) + } +} + +// The cell-divider rule. These four are page 57 of the column manual, reduced to +// the shape that produces its one language error and back again. +// +// That page has no language columns at all: it has two ruled tables, and +// doc.DetectColumns found their cell dividers. Its narrow question-cell column is +// 289 runes of short German labels carrying ä and ö and no ü or ß, which reads as +// Finnish on its own and as German once the table's two cell columns are read as +// one thing. So the case is built the same way here — labels that really do read as +// Finnish alone — rather than with text chosen to make the test pass. +const tableLabels = "Saugkraft lässt allmählich nach. Viele Wassertropfen an der " + + "Innenseite des Gehäusedeckels. Beim Saugen tritt Staub aus." + +// ruledTable builds a table spanning x0 to x1 whose cells divide at each of the +// given interior x positions, with two rows so it has a grid at all. +func ruledTable(x0, x1, y0, y1 float64, dividers ...float64) doc.RuledTable { + edges := append(append([]float64{x0}, dividers...), x1) + t := doc.RuledTable{ + Box: doc.CellRect{X0: x0, Y0: y0, X1: x1, Y1: y1}, + Rows: 2, + Cols: len(edges) - 1, + } + for row := 0; row < 2; row++ { + top := y0 + float64(row)*(y1-y0)/2 + for c := 0; c+1 < len(edges); c++ { + t.Cells = append(t.Cells, doc.RuledCell{ + Row: row, Col: c, ColSpan: 1, Chars: 40, + Rect: doc.CellRect{X0: edges[c], Y0: top, X1: edges[c+1], Y1: top + (y1-y0)/2}, + }) + } + } + return t +} + +// TestPageRegionsDivideATablesCellsWithoutItsRuledLines is the state that shipped, +// pinned so that the fix below is visibly a fix and not a restatement. Given no +// tables — pdftocairo absent, or the ruled lines not read — the two cell columns +// are two languages, and the left one is wrong. +func TestPageRegionsDivideATablesCellsWithoutItsRuledLines(t *testing.T) { + page := regionPage(57, fill(tableLabels, 10), fill(german, 10)) + got := doc.PageRegions(page, nil, doc.PageResolution{}, nil) + + if len(got) != 2 { + t.Fatalf("got %d regions, want the 2 this page produced before ruled lines were read", len(got)) + } + if got[0].Lang != "fi" || got[1].Lang != "de" { + t.Fatalf("columns read as %q and %q; this test only means something if the left "+ + "one is the wrong answer the fix removes", got[0].Lang, got[1].Lang) + } +} + +// TestPageRegionsDoNotDivideOnATablesCellDividers is the fix: the same page, with +// the ruled lines read. One region, in the language the table's whole text is in, +// and the page's characters all still counted. +func TestPageRegionsDoNotDivideOnATablesCellDividers(t *testing.T) { + page := regionPage(57, fill(tableLabels, 10), fill(german, 10)) + // The gutter runs 280 to 320, so a cell divider at 300 is the boundary the + // detector reported, drawn by the table. + table := ruledTable(20, 870, 10, 800, 300) + + got := onlyRegion(t, doc.PageRegions(page, nil, doc.PageResolution{}, []doc.RuledTable{table})) + if got.Lang != "de" { + t.Errorf("region language = %q, want de: read as one table the text is German", got.Lang) + } + if got.X0 != 0 || got.X1 != testRegionPageWidth { + t.Errorf("region is boxed at x=%.0f-%.0f; a page that does not divide is whole-page", + got.X0, got.X1) + } + // The table's area is excluded from deciding where the page divides, never from + // what the page holds. Both cell columns' text is still charged for. + if got.Chars < 1000 { + t.Errorf("region holds %d characters; both cell columns' text must still be counted", + got.Chars) + } +} + +// TestPageRegionsKeepColumnsATableDoesNotExplain is the guard that keeps this from +// running the other way, and it is the case the fix could most easily break: a +// table printed across two genuine language columns must not weld them into one. +// +// The discriminator is measured rather than assumed. On page 57 every one of the +// four coincidences is within about five units, so a table only merges a boundary +// it can be shown to have drawn. +func TestPageRegionsKeepColumnsATableDoesNotExplain(t *testing.T) { + page := regionPage(2, fill(german, 10), fill(polish, 10)) + // A table across the whole measure whose only interior divider is at 150 — + // nowhere near the 300 the page divides at. + table := ruledTable(20, 870, 10, 800, 150) + + got := doc.PageRegions(page, nil, doc.PageResolution{}, []doc.RuledTable{table}) + if len(got) != 2 { + t.Fatalf("got %d regions, want 2: this table explains neither boundary", len(got)) + } + if got[0].Lang != "de" || got[1].Lang != "pl" { + t.Errorf("regions read as %q and %q, want de and pl", got[0].Lang, got[1].Lang) + } +} + +// TestPageRegionsIgnoreATableInsideOneColumn is the column manual's pages 52 to 56: +// a small parts table set inside one of three same-language columns. A table that +// covers one column has nothing to merge, and the page reads as it did. +func TestPageRegionsIgnoreATableInsideOneColumn(t *testing.T) { + page := regionPage(52, fill(german, 10), fill(polish, 10), fill(rus, 10)) + table := ruledTable(25, 285, 400, 700, 150) + + got := doc.PageRegions(page, nil, doc.PageResolution{}, []doc.RuledTable{table}) + if len(got) != 3 { + t.Fatalf("got %d regions, want the 3 languages this page prints", len(got)) + } +} diff --git a/internal/doc/repertoire.go b/internal/doc/repertoire.go new file mode 100644 index 0000000..b0bcc30 --- /dev/null +++ b/internal/doc/repertoire.go @@ -0,0 +1,531 @@ +package doc + +import ( + "fmt" + "sort" + "strings" + "unicode" +) + +// The character-repertoire signal: which language's alphabet a page is actually +// written with. +// +// [DominantScript] narrows a Cyrillic page to seven candidate languages and +// stops. That is not a gap in the implementation — a script table cannot know +// more. But languages sharing a script do not share an *alphabet*, and the +// letters only some of them can write cost nothing to count. +// +// Measured per column of a three-column Cyrillic page in a real manual: +// +// column Ukrainian marks Russian marks Kazakh marks verdict +// left 0 40 0 Russian +// middle 83 0 0 Ukrainian +// right 78 111 143 Kazakh +// +// consistent across 18 of the 19 such pages; the exception is a page of contact +// addresses rather than content. Three languages, one script, one page, one per +// column — exactly what script alone cannot resolve. +// +// The right column is why a maximum over those counts is the wrong reading. +// Kazakh's alphabet contains the і it shares with Ukrainian and the ы it shares +// with Russian, so overlapping counts are the normal case, not an anomaly. What +// decides is whether one language's alphabet can account for *everything* +// observed, and how much of that alphabet the text actually exercises. +// +// See docs/design/language-detection.md. + +// repertoire is one language's distinctive characters within a script. +// +// Deliberately not the language's whole alphabet. The letters every candidate +// shares carry no information, and counting them would swamp the ones that do: +// Cyrillic и appears in six of these seven languages and roughly seven times per +// hundred letters, which would bury the ы that actually names Russian. +type repertoire struct { + lang string + marks string +} + +// cyrillicRepertoires are the letters that tell the Cyrillic languages apart. +// +// Two entries look wrong and are not. Kazakh lists ё ъ ы э and і as well as its +// own nine letters, because Kazakh genuinely writes them — omitting them would +// make a Kazakh page look like a contradiction rather than a Kazakh page. +// Bulgarian lists a single letter because its alphabet is a strict subset of +// Russian's: it has no exclusive letter at all, and what identifies it is ъ +// appearing in quantity while ы, э and ё never do. +var cyrillicRepertoires = []repertoire{ + {"ru", "ёъыэ"}, + {"uk", "ґєії"}, + {"be", "ёіўыэ"}, + {"bg", "ъ"}, + {"sr", "ђјљњћџ"}, + {"mk", "ѓѕјљњќџ"}, + {"kk", "әғқңөұүһіёъыэ"}, +} + +// latinRepertoires are the letters that tell the Latin-script languages apart. +// +// English, Indonesian and Malay are listed with nothing, which is the honest +// entry: they write no letter outside a-z, so this signal cannot see them and +// must never name them. Carrying them in the table rather than omitting them is +// what lets [RepertoireTies] report that fact instead of staying silent about it. +// +// Serbian appears here as well as in the Cyrillic table. It is written in both, +// and its Latin form is one of this signal's blind spots — see [RepertoireTies]. +var latinRepertoires = []repertoire{ + {"pl", "ąćęłńóśźż"}, + {"cs", "áčďéěíňóřšťúůýž"}, + {"sk", "áäčďéíĺľňóôŕšťúýž"}, + {"hu", "áéíóöőúüű"}, + {"ro", "ăâîșț"}, + {"tr", "çğıöşü"}, + {"lt", "ąčėęįšūųž"}, + {"lv", "āčēģīķļņšūž"}, + {"et", "äöõüšž"}, + {"sl", "čšž"}, + {"hr", "čćđšž"}, + {"bs", "čćđšž"}, + {"sr", "čćđšž"}, + {"de", "äöüß"}, + {"fr", "àâæçèéêëîïôœùûüÿ"}, + {"es", "áéíñóúü"}, + {"it", "àèéìòù"}, + {"pt", "àáâãçéêíóôõú"}, + // Dutch writes no diacritic it cannot do without, so most Dutch pages carry + // none of these and this signal simply has nothing to say about them. The + // entry exists so that the tremas Dutch does write are not read as French. + {"nl", "éëïü"}, + // é is not decoration in these two: Danish marks stress with it (idé, allé, + // kontrollér) and Norwegian writes én. Leaving it out made a real Danish + // paragraph contradict Danish. + {"da", "æøåé"}, + // Bokmål and Nynorsk share this alphabet too, so nb and nn are equally + // indistinguishable from Danish here. Manuals print NO, which is the code + // listed. + {"no", "æøåé"}, + {"sv", "åäö"}, + // Finnish differs from Swedish only by å, which belongs to the Finnish + // alphabet but appears almost solely in Swedish loan names. Listing it would + // make Swedish unidentifiable; leaving it out means a Swedish sample carrying + // no å reads as Finnish. That trade is stated in the design doc. + {"fi", "äö"}, + {"is", "áðéíóúýþæö"}, + {"en", ""}, + {"id", ""}, + {"ms", ""}, +} + +// langMarks is a prepared repertoire: a set for testing membership and a sorted +// slice for reporting. +type langMarks struct { + lang string + set map[rune]bool + all []rune +} + +// preparedRepertoires and repertoireUniverse are the tables above, indexed for +// use: per script, one entry per language, plus every mark any of them uses. +var preparedRepertoires, repertoireUniverse = prepareRepertoires() + +func prepareRepertoires() (prepared map[string][]langMarks, universe map[string]map[rune]bool) { + byScript := map[string][]repertoire{ + ScriptCyrillic: cyrillicRepertoires, + ScriptLatin: latinRepertoires, + } + + prepared = make(map[string][]langMarks, len(byScript)) + universe = make(map[string]map[rune]bool, len(byScript)) + for script, langs := range byScript { + all := make(map[rune]bool, 64) + out := make([]langMarks, 0, len(langs)) + for i := range langs { + marks := []rune(langs[i].marks) + set := make(map[rune]bool, len(marks)) + for j, r := range marks { + marks[j] = foldMark(r) + set[marks[j]] = true + all[marks[j]] = true + } + sort.Slice(marks, func(a, b int) bool { return marks[a] < marks[b] }) + out = append(out, langMarks{lang: langs[i].lang, set: set, all: marks}) + } + prepared[script] = out + universe[script] = all + } + return prepared, universe +} + +// foldMark folds the character variants that are one letter typeset two ways. +// +// Romanian's ș and ț are routinely printed with a cedilla (ş, ţ) by fonts +// predating Unicode 3.0, which is the same letter and not Turkish. Folding them +// together stops a typesetting choice from reading as a different language, and +// applying the fold to the tables as well as to the text keeps both entries +// written the way their own language writes them. +func foldMark(r rune) rune { + switch r { + case 'ş': + return 'ș' + case 'ţ': + return 'ț' + } + return r +} + +// foreignMarkFraction is the share of the observed distinctive characters a +// language may be unable to write and still be considered. +// +// Not zero, because a page of one language routinely carries a brand name, a +// quoted term or a foreign address, and one character it cannot write is not +// evidence of a different language. Small, because the real cases are nowhere +// near the line: on the measured Cyrillic page's right column, Russian cannot +// account for 67% of the characters and Ukrainian 76%. This is a judgement +// rather than a measurement, and nothing below was tuned to it. +const foreignMarkFraction = 0.05 + +// strayForeignMarks is how many contradicting characters are forgiven outright, +// whatever the fraction says. +// +// A fraction alone is too harsh on short text, and short text is the normal case +// for a caption, a heading, or one column of a page. Eleven distinctive +// characters make a single stray worth 9%, which ruled out Danish for a Danish +// paragraph over one character. One character is never evidence of a language. +const strayForeignMarks = 1 + +// minRepertoireMarks is how many distinctive characters a text needs before this +// signal will name a language at all. +// +// One is a brand name: Wałęsa on an English page is not a Polish page. A page of +// prose in a language that has distinctive letters carries dozens — the measured +// columns carried 40, 83 and 332. Three is the smallest count that is not a +// single stray word, and below it the marks are still reported so a caller can +// see what was there. +const minRepertoireMarks = 3 + +// scoreEpsilon is how close two scores must be to count as tied. Equal +// repertoires produce bit-identical scores, so this exists only so that a tie +// never depends on floating-point luck. +const scoreEpsilon = 1e-9 + +// RepertoireMatch is what the character-repertoire signal concluded about a text. +// +// An empty Candidates is a normal and deliberate outcome: this signal reports +// nothing rather than guessing. Marks distinguishes the two reasons — zero means +// the text carries no distinctive characters at all, non-zero means it carries +// some that no single language accounts for, which is what a page mixing two +// languages looks like. +type RepertoireMatch struct { + // Script is the script whose distinctive characters were read. + Script string `json:"script"` + // Marks is how many distinctive-character occurrences were found in it. + Marks int `json:"marks"` + // Candidates are the languages that can account for those characters, best + // first. + Candidates []RepertoireCandidate `json:"candidates,omitempty"` + // Ambiguous reports that the leading candidates scored identically. The + // signal has narrowed the language and cannot name it; see [RepertoireTies]. + Ambiguous bool `json:"ambiguous"` + // Note says in checkable terms which characters produced this outcome. + Note string `json:"note,omitempty"` +} + +// RepertoireCandidate is one language's fit to the characters observed. +type RepertoireCandidate struct { + // Lang is the language subtag. + Lang string `json:"lang"` + // Score is Matched/Marks × Used/Total, in 0 to 1. + // + // The first factor is how much of the evidence this language can write, the + // second how much of this language the evidence exercises. Both are needed. + // Coverage alone cannot separate a language from one whose alphabet contains + // it — Kazakh writes every Russian letter, so it explains a Russian page + // perfectly — and the second factor is what settles that: on a Russian page + // none of Kazakh's own nine letters appear. + Score float64 `json:"score"` + // Matched and Foreign are how many of the observed characters this language + // can and cannot write. + Matched int `json:"matched"` + Foreign int `json:"foreign"` + // Used and Total are how many of this language's distinctive characters + // appeared, and how many it has. + Used int `json:"used"` + Total int `json:"total"` + // Evidence is the characters this language accounts for and their counts, + // most frequent first: "і×78 ы×40". + Evidence string `json:"evidence"` + // Missing is this language's distinctive characters that never appeared. + Missing string `json:"missing,omitempty"` +} + +// MatchRepertoire reads a text's distinctive characters and reports which +// languages of its script can account for them. +// +// Pure, free, and needs no model, network or dependency. It is the fifth +// language signal and it is not authoritative: it answers "whose alphabet is +// this", which is not the same question as "what language is this", and there +// are pairs it cannot separate at all. Use [RepertoireMatch.Language] to get an +// answer only when there is one. +func MatchRepertoire(s string) RepertoireMatch { + script := DominantScript(s) + m := RepertoireMatch{Script: script} + + if script == "" { + m.Note = "no letters to read" + return m + } + langs := preparedRepertoires[script] + if len(langs) == 0 { + // Every other script this package recognises is already resolved to one + // language by the script itself, so there is nothing left to separate. + m.Note = fmt.Sprintf("%s script needs no repertoire table", script) + return m + } + + counts := RepertoireMarks(script, s) + for _, n := range counts { + m.Marks += n + } + if m.Marks == 0 { + m.Note = fmt.Sprintf("no %s character here belongs to one language rather than another", script) + return m + } + if m.Marks < minRepertoireMarks { + m.Note = fmt.Sprintf("only %d distinctive characters (%s), too few to name a language", + m.Marks, markList(counts, nil)) + return m + } + + all := make([]RepertoireCandidate, len(langs)) + admissible := make([]RepertoireCandidate, 0, len(langs)) + for i := range langs { + l := &langs[i] + c := RepertoireCandidate{Lang: l.lang, Total: len(l.all)} + for r, n := range counts { + if l.set[r] { + c.Matched += n + c.Used++ + } else { + c.Foreign += n + } + } + all[i] = c + + if c.Total == 0 || c.Matched == 0 { + continue + } + if c.Foreign > strayForeignMarks && float64(c.Foreign)/float64(m.Marks) > foreignMarkFraction { + continue + } + c.Score = float64(c.Matched) / float64(m.Marks) * float64(c.Used) / float64(c.Total) + c.Evidence = markList(counts, l.set) + c.Missing = missingMarks(l, counts) + admissible = append(admissible, c) + } + + if len(admissible) == 0 { + m.Note = noSingleLanguageNote(m.Marks, all, counts) + return m + } + + sort.Slice(admissible, func(i, j int) bool { + if admissible[i].Score != admissible[j].Score { + return admissible[i].Score > admissible[j].Score + } + return admissible[i].Lang < admissible[j].Lang + }) + m.Candidates = admissible + m.Ambiguous = len(admissible) > 1 && + admissible[0].Score-admissible[1].Score < scoreEpsilon + m.Note = decidedNote(m) + return m +} + +// Language returns the one language the characters name, and whether the signal +// is prepared to name one. It is false whenever the evidence was absent, +// contradictory, or fits two languages equally — a confident wrong answer being +// worse than an absent one. +func (m RepertoireMatch) Language() (string, bool) { + if len(m.Candidates) == 0 || m.Ambiguous { + return "", false + } + return m.Candidates[0].Lang, true +} + +// Tied returns the languages that share the leading score, in order. It is the +// answer when [RepertoireMatch.Language] declines: the signal has narrowed the +// page to these and cannot go further. +func (m RepertoireMatch) Tied() []string { + if len(m.Candidates) == 0 { + return nil + } + var tied []string + for i := range m.Candidates { + if m.Candidates[0].Score-m.Candidates[i].Score >= scoreEpsilon { + break + } + tied = append(tied, m.Candidates[i].Lang) + } + return tied +} + +// RepertoireMarks returns the characters in s that distinguish one language of +// the given script from another, and how often each occurs. Characters of other +// scripts are ignored rather than counted against anything, so a Latin brand +// name on a Russian page is not evidence about the Russian. +// +// Case is folded, so an all-capitals heading counts the same as body text. Only +// precomposed characters are seen: a decomposed é (e plus a combining accent) +// reads as a plain e and is silently not evidence, which loses the signal rather +// than misdirecting it. +func RepertoireMarks(script, s string) map[rune]int { + universe := repertoireUniverse[script] + if len(universe) == 0 { + return nil + } + counts := make(map[rune]int, 8) + for _, r := range s { + if !unicode.IsLetter(r) { + continue + } + if r = foldMark(unicode.ToLower(r)); universe[r] { + counts[r]++ + } + } + return counts +} + +// RepertoireTies returns the languages whose distinctive characters are exactly +// lang's, including lang itself, sorted. More than one entry means this signal +// can never separate them and will report them tied rather than pick one. +// +// It is derived from the same tables the signal scores against, so it cannot +// drift from what the signal actually does. Known groups: +// +// da no identical: æ ø å. Bokmål and Nynorsk share it too. +// bs hr sr identical in Latin script: č ć đ š ž. +// en id ms nothing at all: they write no letter outside a-z. +// +// A language written in two scripts is reported against every script it appears +// in, so Serbian ties with Bosnian and Croatian on the strength of its Latin +// form even though its Cyrillic form is unmistakable. +// +// Ties are not the only limit — a language whose repertoire is a subset of +// another's is separated only by the larger one's letters being absent, which +// short text cannot establish. See docs/design/language-detection.md. +func RepertoireTies(lang string) []string { + base := BaseLanguage(lang) + if base == "" { + base = lang + } + + tied := make(map[string]bool, 4) + for _, langs := range preparedRepertoires { + key, found := "", false + for i := range langs { + if langs[i].lang == base { + key, found = string(langs[i].all), true + break + } + } + if !found { + continue + } + for i := range langs { + if string(langs[i].all) == key { + tied[langs[i].lang] = true + } + } + } + if len(tied) == 0 { + return nil + } + + out := make([]string, 0, len(tied)) + for l := range tied { + out = append(out, l) + } + sort.Strings(out) + return out +} + +// decidedNote explains an outcome that produced candidates, in the terms a +// reader can check against the page: which characters were counted, and why the +// runner-up lost. +func decidedNote(m RepertoireMatch) string { + best := &m.Candidates[0] + if m.Ambiguous { + return fmt.Sprintf("%s write the same distinctive characters (%s) and cannot be told apart here", + strings.Join(m.Tied(), " and "), best.Evidence) + } + if len(m.Candidates) == 1 { + return fmt.Sprintf("%s: %s; no other %s language accounts for them", + best.Lang, best.Evidence, m.Script) + } + next := &m.Candidates[1] + return fmt.Sprintf("%s: %s; %s fits too but %d of its distinctive letters never appear (%s)", + best.Lang, best.Evidence, next.Lang, next.Total-next.Used, next.Missing) +} + +// noSingleLanguageNote describes marks that no one language can account for. +// Naming the two biggest contributors is what makes a mixed page reportable +// rather than merely unanswered. +func noSingleLanguageNote(marks int, all []RepertoireCandidate, counts map[rune]int) string { + ranked := make([]RepertoireCandidate, len(all)) + copy(ranked, all) + sort.Slice(ranked, func(i, j int) bool { + if ranked[i].Matched != ranked[j].Matched { + return ranked[i].Matched > ranked[j].Matched + } + return ranked[i].Lang < ranked[j].Lang + }) + + var named []string + for i := range ranked { + if len(named) == 2 || ranked[i].Matched == 0 { + break + } + named = append(named, fmt.Sprintf("%s accounts for %d", ranked[i].Lang, ranked[i].Matched)) + } + if len(named) == 0 { + return fmt.Sprintf("%d distinctive characters (%s) belong to no language in this table", + marks, markList(counts, nil)) + } + return fmt.Sprintf("%d distinctive characters (%s) fit no single language: %s", + marks, markList(counts, nil), strings.Join(named, ", ")) +} + +// markList renders characters and their counts as "і×78 ы×40", most frequent +// first. only restricts it to one language's characters; nil renders all of them. +func markList(counts map[rune]int, only map[rune]bool) string { + runes := make([]rune, 0, len(counts)) + for r := range counts { + if only == nil || only[r] { + runes = append(runes, r) + } + } + sort.Slice(runes, func(i, j int) bool { + if counts[runes[i]] != counts[runes[j]] { + return counts[runes[i]] > counts[runes[j]] + } + return runes[i] < runes[j] + }) + + parts := make([]string, len(runes)) + for i, r := range runes { + parts[i] = fmt.Sprintf("%c×%d", r, counts[r]) + } + return strings.Join(parts, " ") +} + +// missingMarks lists the language's distinctive characters that did not appear. +// It is the evidence *against* a language that otherwise fits, and the reason a +// Russian page is not read as Kazakh. +func missingMarks(l *langMarks, counts map[rune]int) string { + var b strings.Builder + for _, r := range l.all { + if counts[r] == 0 { + b.WriteRune(r) + } + } + return b.String() +} diff --git a/internal/doc/repertoire_test.go b/internal/doc/repertoire_test.go new file mode 100644 index 0000000..1c367ad --- /dev/null +++ b/internal/doc/repertoire_test.go @@ -0,0 +1,662 @@ +package doc_test + +import ( + "sort" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Hermetic tests for the character-repertoire signal. No PDF and no poppler: +// every string is built here. The prose samples are ordinary appliance-manual +// sentences written in each language, because the signal reads the alphabet a +// page is actually typeset with and a contrived alphabet soup would prove +// nothing about real pages. + +// markText builds text containing exactly the given characters at the given +// counts, spread through filler made only of letters no language in the table +// claims. It is how the measured per-column counts are reproduced exactly. +func markText(counts map[rune]int, filler string) string { + runes := make([]rune, 0, len(counts)) + for r := range counts { + runes = append(runes, r) + } + sort.Slice(runes, func(i, j int) bool { return runes[i] < runes[j] }) + + var b strings.Builder + for _, r := range runes { + for range counts[r] { + b.WriteString(filler) + b.WriteRune(r) + } + } + return b.String() +} + +// cyrillicFiller is shared Cyrillic prose: not one of its letters distinguishes +// any language from any other, so it sets the dominant script and nothing else. +const cyrillicFiller = " робот пилосос " + +// The three columns of the measured page, at the counts measured on it: +// +// column Ukrainian marks Russian marks Kazakh marks +// left 0 40 0 +// middle 83 0 0 +// right 78 111 143 +var ( + leftColumnMarks = map[rune]int{'ы': 25, 'э': 10, 'ъ': 3, 'ё': 2} + middleColumnMarks = map[rune]int{'і': 50, 'ї': 20, 'є': 10, 'ґ': 3} + rightColumnMarks = map[rune]int{ + 'і': 78, + 'ы': 80, 'э': 25, 'ъ': 4, 'ё': 2, + 'ә': 25, 'ғ': 20, 'қ': 30, 'ң': 20, 'ө': 18, 'ұ': 15, 'ү': 10, 'һ': 5, + } +) + +func TestRepertoireSeparatesThreeCyrillicLanguagesOnOnePage(t *testing.T) { + // This is what the signal exists for. ScriptRuns narrows a Cyrillic page to + // seven candidates and stops; the measured fixture has a page whose three + // columns are three different Cyrillic languages, one per column. + cases := []struct { + column string + marks map[rune]int + want string + }{ + {"left", leftColumnMarks, "ru"}, + {"middle", middleColumnMarks, "uk"}, + {"right", rightColumnMarks, "kk"}, + } + + for _, c := range cases { + text := markText(c.marks, cyrillicFiller) + if s := doc.DominantScript(text); s != doc.ScriptCyrillic { + t.Fatalf("%s column: script = %q, want Cyrillic", c.column, s) + } + + m := doc.MatchRepertoire(text) + got, ok := m.Language() + if !ok || got != c.want { + t.Errorf("%s column: got %q ok=%v, want %q\n%s", c.column, got, ok, c.want, m.Note) + } + } +} + +func TestRepertoireReadsSharedCharactersAsSharedNotAsAVote(t *testing.T) { + // The right column is why a maximum over per-language counts is the wrong + // reading. Kazakh writes the і it shares with Ukrainian and the ы it shares + // with Russian, so the overlapping counts the measurement recorded — 78 + // Ukrainian, 111 Russian, 143 Kazakh — are all Kazakh's, and the two rivals + // are ruled out by what they *cannot* write rather than out-counted. + m := doc.MatchRepertoire(markText(rightColumnMarks, cyrillicFiller)) + + if m.Marks != 78+111+143 { + t.Fatalf("marks = %d, want %d", m.Marks, 78+111+143) + } + + byLang := make(map[string]doc.RepertoireCandidate, len(m.Candidates)) + for i := range m.Candidates { + byLang[m.Candidates[i].Lang] = m.Candidates[i] + } + if _, admitted := byLang["ru"]; admitted { + t.Error("Russian was admitted, though it cannot write і or any Kazakh letter") + } + if _, admitted := byLang["uk"]; admitted { + t.Error("Ukrainian was admitted, though it cannot write ы or any Kazakh letter") + } + + kk, ok := byLang["kk"] + if !ok { + t.Fatalf("Kazakh was not a candidate: %s", m.Note) + } + if kk.Matched != m.Marks || kk.Foreign != 0 { + t.Errorf("Kazakh matched %d of %d with %d foreign, want all of them and none", + kk.Matched, m.Marks, kk.Foreign) + } + // The whole point: the note has to let a human check the reading against the + // page rather than take the score on trust. + for _, want := range []string{"і×78", "ы×80", "ә×25"} { + if !strings.Contains(m.Note, want) { + t.Errorf("note does not show %s: %s", want, m.Note) + } + } +} + +func TestRepertoireDeclinesTheWholeThreeColumnPage(t *testing.T) { + // Read as one blob the same page has no answer, and saying so is the correct + // behaviour: the characters of three languages fit none of them. This is what + // makes the signal safe to run before column splitting rather than after. + page := markText(leftColumnMarks, cyrillicFiller) + + markText(middleColumnMarks, cyrillicFiller) + + markText(rightColumnMarks, cyrillicFiller) + + m := doc.MatchRepertoire(page) + if got, ok := m.Language(); ok { + t.Errorf("named %q for a page of three languages: %s", got, m.Note) + } + if m.Marks == 0 { + t.Error("reported no distinctive characters, though the page is full of them") + } + // Absent is not the same as blind: the note must still say what was found. + if !strings.Contains(m.Note, "kk") { + t.Errorf("note does not name the largest contributor: %s", m.Note) + } +} + +// manualProse is one ordinary paragraph of appliance-manual copy per language. +var manualProse = map[string]string{ + "ru": "Перед первым использованием робота-пылесоса внимательно прочитайте это руководство. " + + "Не используйте устройство, если кабель повреждён. Съёмный контейнер для пыли следует " + + "очищать после каждой уборки, а фильтр промывать тёплой водой без моющих средств. " + + "Этот прибор не предназначен для использования детьми. Мыть щётку нельзя.", + "uk": "Перед першим використанням робота-пилососа уважно прочитайте цей посібник. " + + "Не використовуйте пристрій, якщо кабель пошкоджено. Знімний контейнер для пилу слід " + + "очищати після кожного прибирання, а фільтр промивати теплою водою без мийних засобів. " + + "Цей прилад не призначений для використання дітьми. Ґудзик живлення знаходиться збоку. " + + "Якщо щітка забруднена, її потрібно зняти та промити. Є декілька режимів прибирання.", + "kk": "Робот-шаңсорғышты алғаш рет пайдаланар алдында осы нұсқаулықты мұқият оқып шығыңыз. " + + "Кабель зақымдалған болса, құрылғыны пайдаланбаңыз. Шаңға арналған алмалы контейнерді " + + "әрбір тазалаудан кейін тазалап отырыңыз, ал сүзгіні жылы сумен жуыңыз. Бұл құрылғы " + + "балалардың пайдалануына арналмаған.", + "be": "Перад першым выкарыстаннем робата-пыласоса ўважліва прачытайце гэта кіраўніцтва. " + + "Не выкарыстоўвайце прыладу, калі кабель пашкоджаны. Здымны кантэйнер для пылу трэба " + + "чысціць пасля кожнай уборкі, а фільтр прамываць цёплай вадой. Гэты прыбор не " + + "прызначаны для выкарыстання дзецьмі.", + "bg": "Преди първата употреба на прахосмукачката робот прочетете внимателно това ръководство. " + + "Не използвайте уреда, ако кабелът е повреден. Изваждащият се контейнер за прах трябва " + + "да се почиства след всяко почистване, а филтърът да се измива с топла вода. Този уред " + + "не е предназначен за употреба от деца. Съхранявайте ръководството.", + "sr": "Пре прве употребе робота усисивача пажљиво прочитајте ово упутство. Немојте " + + "користити уређај ако је кабл оштећен. Уклоњиви контејнер за прашину треба очистити " + + "после сваког чишћења, а филтер испрати топлом водом. Овај уређај није намењен деци.", + "mk": "Пред првата употреба на роботот правосмукалка внимателно прочитајте го ова упатство. " + + "Не го користете уредот ако кабелот е оштетен. Подвижниот контејнер за прашина треба " + + "да се исчисти по секое чистење, а филтерот да се измие со топла вода. Уредот ќе се " + + "врати на базата автоматски. Не фрлајте ѓубре во контејнерот и не го ставајте до ѕидот.", + + "de": "Lesen Sie diese Anleitung vor der ersten Verwendung des Saugroboters sorgfältig durch. " + + "Verwenden Sie das Gerät nicht, wenn das Kabel beschädigt ist. Der abnehmbare Staubbehälter " + + "muss nach jeder Reinigung geleert werden, und der Filter ist mit warmem Wasser zu spülen. " + + "Größere Fremdkörper müssen vorher entfernt werden. Öffnen Sie das Gehäuse nicht.", + "fr": "Lisez attentivement ce manuel avant la première utilisation du robot aspirateur. " + + "N'utilisez pas le robot si le câble est endommagé. Le bac à poussière amovible doit être " + + "vidé après chaque nettoyage, et le filtre rincé à l'eau tiède. Ce produit n'est pas " + + "destiné à être utilisé par des enfants. Vérifiez que la brosse est propre.", + "es": "Lea atentamente este manual antes de utilizar el robot aspirador por primera vez. " + + "No utilice el aparato si el cable está dañado. El depósito de polvo extraíble debe vaciarse " + + "después de cada limpieza y el filtro debe lavarse con agua tibia. Este aparato no está " + + "diseñado para ser utilizado por niños pequeños.", + "it": "Leggere attentamente questo manuale prima di utilizzare il robot aspirapolvere. " + + "Non utilizzare l'apparecchio se il cavo è danneggiato. Il contenitore della polvere " + + "estraibile può essere svuotato dopo ogni pulizia, però il filtro va risciacquato con " + + "acqua tiepida. Così l'apparecchio durerà più a lungo. Perché è necessario?", + "pt": "Leia atentamente este manual antes da primeira utilização do robô aspirador. " + + "Não utilize o aparelho se o cabo estiver danificado. O depósito de pó amovível deve ser " + + "esvaziado após cada limpeza e o filtro lavado com água morna. Este aparelho não se destina " + + "a ser utilizado por crianças. Verifique a posição da escova.", + "pl": "Przed pierwszym użyciem robota odkurzającego należy uważnie przeczytać tę instrukcję. " + + "Nie należy używać urządzenia, jeśli przewód jest uszkodzony. Wyjmowany pojemnik na kurz " + + "należy opróżniać po każdym sprzątaniu, a filtr płukać ciepłą wodą. To urządzenie nie jest " + + "przeznaczone do obsługi przez dzieci. Sprawdź, czy szczotka jest czysta.", + "cs": "Před prvním použitím robotického vysavače si pečlivě přečtěte tento návod. " + + "Nepoužívejte přístroj, pokud je kabel poškozený. Vyjímatelnou nádobu na prach je třeba " + + "vyprázdnit po každém úklidu a filtr propláchnout vlažnou vodou. Tento přístroj není určen " + + "pro použití dětmi. Zkontrolujte, zda je kartáč čistý.", + "sk": "Pred prvým použitím robotického vysávača si pozorne prečítajte tento návod. " + + "Nepoužívajte prístroj, ak je kábel poškodený. Vyberateľnú nádobu na prach je potrebné " + + "vyprázdniť po každom upratovaní a filter prepláchnuť vlažnou vodou. Tento prístroj nie je " + + "určený na používanie deťmi. Ľavá strana zariadenia musí byť voľná. Ôsmy krok je dôležitý.", + "hu": "A robotporszívó első használata előtt figyelmesen olvassa el ezt az útmutatót. " + + "Ne használja a készüléket, ha a kábel sérült. A kivehető portartályt minden takarítás után " + + "ki kell üríteni, a szűrőt pedig langyos vízzel kell öblíteni. Ez a készülék nem gyermekek " + + "általi használatra készült. Ellenőrizze a kefe állapotát.", + "ro": "Citiți cu atenție acest manual înainte de prima utilizare a robotului aspirator. " + + "Nu utilizați aparatul dacă cablul este deteriorat. Recipientul detașabil pentru praf " + + "trebuie golit după fiecare curățare, iar filtrul trebuie clătit cu apă călduță. Acest " + + "aparat nu este destinat utilizării de către copii. Verificați starea periei.", + "tr": "Robot süpürgeyi ilk kez kullanmadan önce bu kılavuzu dikkatlice okuyun. " + + "Kablo hasarlıysa cihazı kullanmayın. Çıkarılabilir toz haznesi her temizlikten sonra " + + "boşaltılmalı ve filtre ılık suyla yıkanmalıdır. Bu cihaz çocuklar tarafından " + + "kullanılmak üzere tasarlanmamıştır. Fırçanın temiz olduğunu kontrol edin.", + "lt": "Prieš pirmą kartą naudodami robotą dulkių siurblį, atidžiai perskaitykite šį vadovą. " + + "Nenaudokite prietaiso, jei laidas pažeistas. Išimamą dulkių talpyklą reikia ištuštinti " + + "po kiekvieno valymo, o filtrą praplauti šiltu vandeniu. Šis prietaisas nėra skirtas " + + "naudoti vaikams. Patikrinkite, ar šepetys švarus.", + "lv": "Pirms putekļu sūcēja robota pirmās lietošanas rūpīgi izlasiet šo rokasgrāmatu. " + + "Nelietojiet ierīci, ja kabelis ir bojāts. Izņemamā putekļu tvertne jāiztukšo pēc katras " + + "tīrīšanas, bet filtrs jāizskalo ar siltu ūdeni. Šī ierīce nav paredzēta lietošanai " + + "bērniem. Pārbaudiet, vai birste ir tīra.", + "et": "Enne robottolmuimeja esmakordset kasutamist lugege see juhend hoolikalt läbi. " + + "Ärge kasutage seadet, kui kaabel on kahjustatud. Eemaldatav tolmumahuti tuleb pärast iga " + + "koristamist tühjendada ja filter loputada leige veega. Käesolev seade ei ole mõeldud " + + "lastele kasutamiseks. Kontrollige, kas hari on puhas. Õhufilter tuleb vahetada.", + "sl": "Pred prvo uporabo robotskega sesalnika natančno preberite ta priročnik. " + + "Naprave ne uporabljajte, če je kabel poškodovan. Snemljivo posodo za prah je treba " + + "izprazniti po vsakem čiščenju, filter pa sprati z mlačno vodo. Ta naprava ni namenjena " + + "uporabi otrok. Preverite, ali je krtača čista.", + "sv": "Läs denna bruksanvisning noggrant innan du använder robotdammsugaren för första gången. " + + "Använd inte apparaten om kabeln är skadad. Den avtagbara dammbehållaren måste tömmas " + + "efter varje rengöring och filtret sköljas i ljummet vatten. Den här apparaten är inte " + + "avsedd att användas av barn. Kontrollera att borsten är ren.", + "fi": "Lue tämä käyttöohje huolellisesti ennen robotti-imurin ensimmäistä käyttökertaa. " + + "Älä käytä laitetta, jos johto on vaurioitunut. Irrotettava pölysäiliö on tyhjennettävä " + + "jokaisen siivouksen jälkeen ja suodatin huuhdeltava haalealla vedellä. Tätä laitetta ei " + + "ole tarkoitettu lasten käyttöön. Tarkista, että harja on puhdas.", + "is": "Lesið þessar leiðbeiningar vandlega áður en ryksuguvélmennið er notað í fyrsta sinn. " + + "Notið ekki tækið ef snúran er skemmd. Tæma þarf lausa rykhólfið eftir hverja þrif og " + + "skola síuna með volgu vatni. Þetta tæki er ekki ætlað börnum. Athugið hvort burstinn sé " + + "hreinn. Öll aukahlutir fylgja.", +} + +// Prose that this signal must decline, and why. +var undetectableProse = map[string]string{ + "en": "Read this manual carefully before using the robot vacuum for the first time. " + + "Do not use the appliance if the cable is damaged. The removable dust bin must be emptied " + + "after every cleaning cycle and the filter rinsed in lukewarm water.", + "id": "Bacalah petunjuk ini dengan saksama sebelum menggunakan robot penyedot debu untuk " + + "pertama kali. Jangan gunakan perangkat jika kabel rusak. Wadah debu yang dapat dilepas " + + "harus dikosongkan setelah setiap pembersihan dan filter dibilas dengan air hangat.", + "ms": "Baca manual ini dengan teliti sebelum menggunakan robot penyedut habuk buat kali " + + "pertama. Jangan gunakan perkakas jika kabel rosak. Bekas habuk yang boleh ditanggalkan " + + "mesti dikosongkan selepas setiap pembersihan dan penapis dibilas dengan air suam.", + "nl": "Lees deze handleiding zorgvuldig door voordat u de robotstofzuiger voor het eerst " + + "gebruikt. Gebruik het apparaat niet als de kabel beschadigd is. Het uitneembare " + + "stofreservoir moet na elke schoonmaakbeurt worden geleegd en het filter met lauw water " + + "worden gespoeld.", +} + +func TestRepertoireNamesTheLanguageOfOrdinaryProse(t *testing.T) { + langs := make([]string, 0, len(manualProse)) + for lang := range manualProse { + langs = append(langs, lang) + } + sort.Strings(langs) + + for _, want := range langs { + m := doc.MatchRepertoire(manualProse[want]) + got, ok := m.Language() + if !ok { + t.Errorf("%s: declined to name a language: %s", want, m.Note) + continue + } + if got != want { + t.Errorf("%s: got %q — %s", want, got, m.Note) + } + } +} + +func TestRepertoireSeparatesLatinLanguagePairs(t *testing.T) { + // Pairs that a trigram detector confuses or that share most of an alphabet. + // Czech and Slovak are here deliberately: they are usually listed together as + // a hard pair, and by repertoire they are not, because Czech ř ě ů and Slovak + // ľ ô ä are frequent enough in ordinary prose to contradict the other outright. + pairs := [][2]string{ + {"cs", "sk"}, + {"es", "pt"}, + {"fi", "sv"}, + {"de", "et"}, + {"fr", "it"}, + {"lt", "lv"}, + } + + for _, pair := range pairs { + for _, want := range pair { + m := doc.MatchRepertoire(manualProse[want]) + got, ok := m.Language() + if !ok || got != want { + t.Errorf("%s vs %s: %s prose read as %q (ok=%v) — %s", + pair[0], pair[1], want, got, ok, m.Note) + } + } + } +} + +func TestRepertoireRulesOutRivalsByWhatTheyCannotWrite(t *testing.T) { + // A rival is not out-scored, it is contradicted. Czech is not admitted for + // Slovak prose at all, because ľ and ô are letters Czech does not have. + m := doc.MatchRepertoire(manualProse["sk"]) + for i := range m.Candidates { + if m.Candidates[i].Lang == "cs" { + t.Errorf("Czech was admitted for Slovak prose: %s", m.Note) + } + } + if got, _ := m.Language(); got != "sk" { + t.Fatalf("Slovak prose read as %q: %s", got, m.Note) + } +} + +func TestRepertoireKeepsASupersetLanguageAsARankedRunnerUp(t *testing.T) { + // Russian's alphabet is contained in Kazakh's, so Kazakh explains a Russian + // page perfectly and coverage alone cannot separate them. What separates them + // is that none of Kazakh's own nine letters appear. Kazakh is ranked below + // rather than discarded, because "it could be this" is a true statement and + // the caller is entitled to see it. + m := doc.MatchRepertoire(manualProse["ru"]) + + got, ok := m.Language() + if !ok || got != "ru" { + t.Fatalf("Russian prose read as %q (ok=%v): %s", got, ok, m.Note) + } + + var kk *doc.RepertoireCandidate + for i := range m.Candidates { + if m.Candidates[i].Lang == "kk" { + kk = &m.Candidates[i] + } + } + if kk == nil { + t.Fatalf("Kazakh was dropped rather than ranked: %s", m.Note) + } + if kk.Foreign != 0 { + t.Errorf("Kazakh reported %d foreign characters; it can write every Russian letter", kk.Foreign) + } + if kk.Score >= m.Candidates[0].Score { + t.Errorf("Kazakh scored %.3f against Russian's %.3f", kk.Score, m.Candidates[0].Score) + } + if kk.Missing == "" { + t.Error("Kazakh's absent letters were not reported, so the reason it lost is invisible") + } +} + +func TestRepertoirePrefersTheSmallerAlphabetThatFitsExactly(t *testing.T) { + // Bulgarian has no exclusive letter at all: its alphabet is a strict subset of + // Russian's. What identifies it is ъ in quantity while ы, э and ё never + // appear, so Bulgarian must beat Russian on Bulgarian prose — and Russian must + // still be listed, because on a short enough sample it would be the truth. + m := doc.MatchRepertoire(manualProse["bg"]) + + got, ok := m.Language() + if !ok || got != "bg" { + t.Fatalf("Bulgarian prose read as %q (ok=%v): %s", got, ok, m.Note) + } + found := false + for i := range m.Candidates { + if m.Candidates[i].Lang == "ru" { + found = true + } + } + if !found { + t.Errorf("Russian was not offered as a runner-up: %s", m.Note) + } + if !strings.Contains(m.Note, "ыэё") { + t.Errorf("note does not say which Russian letters are missing: %s", m.Note) + } +} + +func TestRepertoireReportsBlindSpotsAsTiesRatherThanPickingOne(t *testing.T) { + // The pairs this signal provably cannot separate. Each must come back tied, + // with Language() declining, and with every tied language named. + cases := []struct { + name string + text string + want []string + }{ + { + "Danish and Norwegian", + "Læs denne vejledning grundigt igennem, før du bruger robotstøvsugeren første gang. " + + "Brug ikke apparatet, hvis kablet er beskadiget. Den aftagelige støvbeholder skal " + + "tømmes efter hver rengøring. Åbn ikke kabinettet. Kontrollér, at børsten er ren.", + []string{"da", "no"}, + }, + { + "Bosnian, Croatian and Serbian in Latin script", + "Prije prve uporabe robotskog usisavača pažljivo pročitajte ovaj priručnik. " + + "Nemojte koristiti uređaj ako je kabel oštećen. Odvojivi spremnik za prašinu treba " + + "isprazniti nakon svakog čišćenja, a filtar isprati mlakom vodom.", + []string{"bs", "hr", "sr"}, + }, + } + + for _, c := range cases { + m := doc.MatchRepertoire(c.text) + if got, ok := m.Language(); ok { + t.Errorf("%s: named %q instead of reporting a tie — %s", c.name, got, m.Note) + } + if !m.Ambiguous { + t.Errorf("%s: not marked ambiguous — %s", c.name, m.Note) + } + tied := m.Tied() + if len(tied) != len(c.want) { + t.Errorf("%s: tied = %v, want %v", c.name, tied, c.want) + continue + } + sort.Strings(tied) + for i := range tied { + if tied[i] != c.want[i] { + t.Errorf("%s: tied = %v, want %v", c.name, tied, c.want) + break + } + } + for _, lang := range c.want { + if !strings.Contains(m.Note, lang) { + t.Errorf("%s: note does not name %s: %s", c.name, lang, m.Note) + } + } + } +} + +func TestRepertoireIsBlindToLanguagesWithoutDistinctiveCharacters(t *testing.T) { + // English, Indonesian and Malay write nothing outside a-z, and Dutch writes + // nothing it cannot do without. The signal must return no candidates at all + // for them, rather than reaching for the nearest language that fits nothing. + langs := make([]string, 0, len(undetectableProse)) + for lang := range undetectableProse { + langs = append(langs, lang) + } + sort.Strings(langs) + + for _, lang := range langs { + m := doc.MatchRepertoire(undetectableProse[lang]) + if got, ok := m.Language(); ok { + t.Errorf("%s prose was named %q: %s", lang, got, m.Note) + } + if len(m.Candidates) != 0 { + t.Errorf("%s prose produced %d candidates: %s", lang, len(m.Candidates), m.Note) + } + if m.Marks != 0 { + t.Errorf("%s prose reported %d distinctive characters, want 0", lang, m.Marks) + } + } +} + +func TestRepertoireTiesNamesTheIndistinguishableLanguages(t *testing.T) { + cases := []struct { + lang string + want []string + }{ + {"da", []string{"da", "no"}}, + {"no", []string{"da", "no"}}, + {"hr", []string{"bs", "hr", "sr"}}, + {"bs", []string{"bs", "hr", "sr"}}, + // Serbian is written in both scripts. Its Cyrillic form is unmistakable, + // but the tie its Latin form is in still has to be reported. + {"sr", []string{"bs", "hr", "sr"}}, + // No distinctive characters at all is the same statement about all three. + {"id", []string{"en", "id", "ms"}}, + {"ms", []string{"en", "id", "ms"}}, + {"en", []string{"en", "id", "ms"}}, + // Separable, so each stands alone. + {"cs", []string{"cs"}}, + {"sk", []string{"sk"}}, + {"ru", []string{"ru"}}, + {"uk", []string{"uk"}}, + // A language this signal knows nothing about at all. + {"ja", nil}, + } + + for _, c := range cases { + got := doc.RepertoireTies(c.lang) + if len(got) != len(c.want) { + t.Errorf("RepertoireTies(%q) = %v, want %v", c.lang, got, c.want) + continue + } + for i := range got { + if got[i] != c.want[i] { + t.Errorf("RepertoireTies(%q) = %v, want %v", c.lang, got, c.want) + break + } + } + } +} + +func TestRepertoireSaysNothingWithoutEvidence(t *testing.T) { + cases := []struct { + name, text string + wantMarks int + }{ + {"empty", "", 0}, + {"whitespace", " \n\t ", 0}, + {"digits and punctuation", "12.5 kg — 230 V / 50 Hz (±10%)", 0}, + {"a model number", "L40-U2400-B", 0}, + {"Greek, which script alone already settles", "Διαβάστε προσεκτικά αυτό το εγχειρίδιο.", 0}, + {"plain Latin prose", "Empty the dust bin after every cleaning cycle.", 0}, + // Two accented characters in an otherwise English caption are a brand + // name, not a language. + {"a brand name in English text", "Connect the Citroën adapter to the café socket.", 2}, + } + + for _, c := range cases { + m := doc.MatchRepertoire(c.text) + if got, ok := m.Language(); ok { + t.Errorf("%s: named %q — %s", c.name, got, m.Note) + } + if len(m.Candidates) != 0 { + t.Errorf("%s: produced %d candidates — %s", c.name, len(m.Candidates), m.Note) + } + if m.Marks != c.wantMarks { + t.Errorf("%s: marks = %d, want %d", c.name, m.Marks, c.wantMarks) + } + if m.Note == "" { + t.Errorf("%s: no note, so a caller cannot tell why nothing came back", c.name) + } + } +} + +func TestRepertoireDeclinesTextMixingTwoLanguages(t *testing.T) { + // Neither language can account for the other's characters, so neither is + // admitted and the signal reports nothing — but the note names both, because + // "these two are both here" is the useful part of the answer. + m := doc.MatchRepertoire(manualProse["de"] + " " + manualProse["pl"]) + + if got, ok := m.Language(); ok { + t.Errorf("named %q for German mixed with Polish: %s", got, m.Note) + } + if len(m.Candidates) != 0 { + t.Errorf("produced %d candidates: %s", len(m.Candidates), m.Note) + } + if m.Marks == 0 { + t.Fatal("reported no distinctive characters for two languages full of them") + } + for _, lang := range []string{"de", "pl"} { + if !strings.Contains(m.Note, lang) { + t.Errorf("note does not name %s: %s", lang, m.Note) + } + } +} + +func TestRepertoireForgivesASingleForeignCharacter(t *testing.T) { + // A page of one language routinely carries a foreign name. One character it + // cannot write must not rule the language out, and on a short paragraph a + // percentage alone does exactly that: one stray in eleven marks is 9%. + m := doc.MatchRepertoire(manualProse["de"] + " Zubehör von Nestlé.") + + got, ok := m.Language() + if !ok || got != "de" { + t.Errorf("German with one French accent read as %q (ok=%v): %s", got, ok, m.Note) + } +} + +func TestRepertoireReadsCedillaAndCommaAsTheSameLetter(t *testing.T) { + // Romanian ș and ț are routinely typeset with a cedilla by fonts predating + // Unicode 3.0. That is a typesetting choice, not a different language, and + // both spellings of the same paragraph must reach the same answer. + comma := manualProse["ro"] + cedilla := strings.NewReplacer("ș", "ş", "ț", "ţ", "Ș", "Ş", "Ț", "Ţ").Replace(comma) + if cedilla == comma { + t.Fatal("the cedilla variant is identical to the comma variant; the test proves nothing") + } + + want := doc.MatchRepertoire(comma) + got := doc.MatchRepertoire(cedilla) + + wantLang, wantOK := want.Language() + gotLang, gotOK := got.Language() + if wantLang != "ro" || !wantOK { + t.Fatalf("comma-below Romanian read as %q (ok=%v): %s", wantLang, wantOK, want.Note) + } + if gotLang != wantLang || gotOK != wantOK { + t.Errorf("cedilla Romanian read as %q (ok=%v), comma-below as %q: %s", + gotLang, gotOK, wantLang, got.Note) + } + if got.Marks != want.Marks { + t.Errorf("cedilla spelling found %d marks, comma-below %d", got.Marks, want.Marks) + } +} + +func TestRepertoireIgnoresCharactersOfAnotherScript(t *testing.T) { + // Cyrillic pages carry Latin furniture — model numbers, web addresses, brand + // names. Counting those against the Cyrillic reading would let a product name + // decide the language of the page it sits on. + clean := manualProse["ru"] + withLatin := clean + " Dreame L40 Ultra — Größe: 350 mm. Voir aussi: dreametech.com/support" + + before := doc.MatchRepertoire(clean) + after := doc.MatchRepertoire(withLatin) + + if after.Script != doc.ScriptCyrillic { + t.Fatalf("script = %q, want Cyrillic", after.Script) + } + if after.Marks != before.Marks { + t.Errorf("Latin furniture changed the mark count from %d to %d", before.Marks, after.Marks) + } + got, ok := after.Language() + if !ok || got != "ru" { + t.Errorf("Russian with Latin furniture read as %q (ok=%v): %s", got, ok, after.Note) + } +} + +func TestRepertoireFoldsCase(t *testing.T) { + // Headings and warning banners are set in capitals, and a page that is all + // heading is exactly the short sample this signal is most needed for. + lower := manualProse["hu"] + upper := strings.ToUpper(lower) + + got := doc.MatchRepertoire(upper) + want := doc.MatchRepertoire(lower) + + if got.Marks != want.Marks { + t.Errorf("upper case found %d marks, lower case %d", got.Marks, want.Marks) + } + gotLang, gotOK := got.Language() + if !gotOK || gotLang != "hu" { + t.Errorf("upper-case Hungarian read as %q (ok=%v): %s", gotLang, gotOK, got.Note) + } +} + +func TestRepertoireMarksReportTheEvidenceItself(t *testing.T) { + // The counts a caller can check by hand. Without these the score is an + // assertion rather than a finding. + counts := doc.RepertoireMarks(doc.ScriptCyrillic, markText(leftColumnMarks, cyrillicFiller)) + + for r, want := range leftColumnMarks { + if counts[r] != want { + t.Errorf("%c counted %d, want %d", r, counts[r], want) + } + } + if len(counts) != len(leftColumnMarks) { + t.Errorf("counted %d distinct characters, want %d", len(counts), len(leftColumnMarks)) + } + if doc.RepertoireMarks(doc.ScriptGreek, "Διαβάστε") != nil { + t.Error("Greek has no repertoire table and must report no marks") + } +} + +func BenchmarkMatchRepertoire(b *testing.B) { + // One manual page is ~1700 characters (docs/design/ingest.md), so the sample + // is padded to that length: the cost that matters is per page, over 560 of them. + page := strings.Repeat(manualProse["ru"], 1700/len([]rune(manualProse["ru"]))+1) + b.ReportAllocs() + for b.Loop() { + doc.MatchRepertoire(page) + } +} diff --git a/internal/doc/rules.go b/internal/doc/rules.go new file mode 100644 index 0000000..b716dad --- /dev/null +++ b/internal/doc/rules.go @@ -0,0 +1,1692 @@ +package doc + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "math" + "os/exec" + "sort" + "strconv" + "strings" + + "github.com/gordon2/manualbox/internal/extern" +) + +// A table is found from the lines the page draws, which is a different input +// rather than a cleverer reading of the old one. +// +// [DetectColumns] and docs/design/layouts.md both record that the *geometry of +// the text* cannot tell a table cell from a text column, and that stands: a +// two-column page and a two-column table project identically. What separates +// them here is not a better statistic over the same runs but a second source — +// the vector graphics. A ruled table is ruled because the document drew lines, +// and those lines are recoverable exactly. +// +// They are not in the input the probe already reads, and that was measured +// rather than assumed. On the parallel-columns fixture's page 57, a page whose +// tables are plainly visible in a render, `pdftohtml -xml` emits four kinds of +// element — pdf, page, fontspec, text — and no path of any sort. So the rules +// are absent from that output, not merely awkward to extract. `pdftocairo -svg` +// reports every one of them. +// +// Coordinates are multiplied by [svgPointScale] on the way out, because cairo +// writes the PDF's own points and everything else in this package works in +// poppler's 1.5-scaled space. That is what keeps a cell checkable: a +// `pdftoppm -r 108` raster matches the output of this file 1:1, so a detected +// cell can be drawn on the rendered page and looked at, which is how the counts +// in docs/design/conversion.md were arrived at. +// +// Five properties of real cairo output shape this code, and every one of them +// was found by reading output that came back wrong first. +// +// **Glyph outlines are paths too.** Cairo writes each glyph as a filled path in +// under `` and draws text by referencing it. A page of +// text is therefore tens of thousands of paths, some of them thin slivers. They +// are excluded structurally, by that id, rather than by any thinness heuristic — +// a heuristic would have to distinguish the stem of an l from a hairline rule, +// and cannot. +// +// **Some table rules are also inside , and skipping loses them.** +// The columns fixture draws page 57's tables in a blend group. Cairo hoists such +// a group out into `` and pulls it back at the use +// site with ``, while +// the hoisted group itself carries `translate(+tx,+ty)`. Measured, both wrong +// answers first: skipping returns 18 rules for that page, all of them +// footer crop marks, and every table rule is lost; entering without +// composing the transform of the element that referenced it shifts every +// coordinate by the filter region's origin, on that page by about (89, 85) +// units, a fifth of the page. So is entered, but only through the +// reference that pulls it in, carrying that reference's matrix. +// +// **A rule may be stroked or filled.** A hairline border arrives as a stroked +// path; a thick divider or a shaded rule arrives as a filled sliver whose +// bounding box is thin in one direction. Both occur in both fixtures and both +// are collected — though the filled half turns out to matter less than it looks, +// and [ruleWalker.filled] records how much. +// +// **A table's outer border may not be drawn at all.** The columns fixture's +// troubleshooting tables stroke every row rule and one interior column rule and +// nothing else. The page reads as bordered only because every row rule stops at +// the same x. So a cell boundary counts as present when a rule is drawn there, +// or when the row rules above and below the cell both terminate there — see +// [cellsOfTable]. +// +// **A path is not drawn at its own length; it is drawn inside a clip.** Cairo +// writes `clip-path` on the group and nests two of them around most page content, +// so a stroke's extent in the file can run past what is painted. It is read by +// clip.go and intersected in both walkers, and it is measured on page 38 of the +// columns manual: that page's frame draws a left edge to y=268.7 where a 432 dpi +// render shows the stroke ending at 238, and that 30 units of phantom rule was +// closing a table cell on a page of framed illustrations. Every cell count of the +// five ground-truth pages is unchanged by reading it. +// +// Nothing here knows what a block is. This file answers only "where are the +// lines, and what cells do they enclose"; assembling a cell's text into readable +// content is a separate stage. + +// Bounds on what counts as a rule, a table and a cell. +// +// Every one is measured against the two fixtures, in the 1.5-scaled space +// described above, on the pages whose printed cells were counted by eye in +// docs/design/conversion.md: page 57 of testdata/fixtures/thomas-drybox-amfibia +// (25 cells) and pages 15, 20, 21 and 100 of dreame-l40-ultra (37, 12, 32, 16). +// The sensitivity range on each is from a sweep over those five pages, one +// constant at a time, holding the rest: the range is every value for which all +// five counts stay right. Two things it says are worth reading before tuning +// anything. Most of these are wide, so they are guards against a page not yet +// seen rather than values fitted to these two documents. And the three that are +// narrow — snapTolerance, maxSegmentGap, minSideCoverage — are narrow because a +// real page sits just outside them, which is recorded on each. +const ( + // svgPointScale converts cairo's PDF points into the space the rest of this + // package uses. Not a tunable: it is 108 dpi over 72, the same ratio + // [ExtractRuns] documents, and it is checked against the fixtures' page boxes + // by TestRuleCoordinateSpaceMatchesTheRuns rather than trusted. + svgPointScale = 1.5 + + // axisTolerance is how far from level a segment may be and still be a rule, + // in output units. It is arithmetic slack, not a tolerance for sloping lines: + // every ruled line in both fixtures is exactly axis-aligned in the file, but a + // composed matrix — cairo writes scales like 0.998785 and negative y — leaves + // a level line off level in the last bits. + // + // Which is the whole measurement, and it surprised the first guess. The five + // pages are unchanged for every value from 0.01 to 60, so on these documents + // this threshold discriminates nothing whatever. The one value that breaks it + // is exactly 0, which returns no cells at all on any of the five. Set for a + // document not yet seen, then: 0.6 is a fifteenth of the thinnest rule here, + // so a line sloping enough to be a real diagonal is still rejected. + axisTolerance = 0.6 + + // maxRuleThickness is how thick a filled sliver may be and still be read as + // a rule rather than as a filled area, and the cap binds: the thickest filled + // sliver accepted is 3.93 units in the columns manual and 3.99 in the + // sequential one, both just inside it. + // + // The five pages are unchanged from 0.5 up to 22. At 24 page 57 gains 3 + // spurious cells, because at that thickness the shaded background behind its + // header row is read as a rule and adds a row boundary through the middle of + // it. So 4 is well clear of the upper edge, and the lower edge is not a + // constraint on these pages at all — see [ruleWalker.filled] for what filled + // slivers are actually worth here, which is less than expected. + maxRuleThickness = 4.0 + + // minRuleLength is how long a segment must be to be a rule at all. It drops + // the corner joins and one-unit stubs a rounded rectangle decomposes into. + // The five pages are unchanged from 0 to 24 and page 15 of the sequential + // manual loses a cell at 25, which puts its shortest real rule at about + // 25 units — a quarter of the way across one of its cells. 6 is a third of a + // line of body text and leaves a factor of four of headroom. + minRuleLength = 6.0 + + // snapTolerance is how far apart two rules may be and still be the same + // printed line. A printed rule is drawn once but arrives twice when a table is + // drawn in a blend group, and the two copies do not land in the same place. + // + // Narrow, and both edges are a real page. The five are correct from 0.4 to + // 3.5. At 0.3 page 57 returns *no cells at all*: its rules arrive in pairs + // about 0.35 apart, and unclustered every row boundary becomes two with an + // unusable sliver between them. At 3.6 that page drops to 21, because its + // header rules are close enough to start welding. 2.5 is near the middle of a + // window whose ends are 9x apart. + snapTolerance = 2.5 + + // minSideCoverage is the fraction of a candidate cell edge that must actually + // be drawn. A fraction rather than a gap, because a cell edge is as long as + // the cell and a 5-unit undrawn stretch means one thing in a 30-unit cell and + // another in a 300-unit one. + // + // The upper edge is what matters and it is close: the five pages hold from + // 0.05 to 0.98, and at 0.99 page 20 of the sequential manual drops from 12 + // cells to 10 while page 21 drops from 32 to 27. At 1.0 every page collapses — + // 9 cells instead of 25 on page 57. What that measures is real: these tables + // have rounded corners, so a rule genuinely stops a unit short of the corner + // it appears to reach, and demanding the whole edge rejects every corner cell. + // 0.9 leaves a tenth of every edge undrawn without asking why. + minSideCoverage = 0.9 + + // minCellSize is how small a rectangle may be and still be a cell. The five + // pages are unchanged from 0 to 20 and break at 25, where page 57 drops to 22 + // cells and page 15 of the sequential manual to 36: the smallest genuine cell + // on them is a row about 22 units tall. 8 is a comfortable third of that and + // still discards the 3-unit slivers a doubled rule leaves. + // + // This is not the legibility threshold — see [minLegibleCellWidth], which is + // larger and applies only to the guards. + minCellSize = 8.0 + + // minSharedTermini is how many of a table's row rules must stop at the same x + // before that x is believed to be a column boundary nobody drew. + // + // The five pages are unchanged from 1 to 8 and page 57 returns nothing at 10, + // which is the measurement that matters: its left table has exactly 9 row + // rules stopping at x=29.7 and x=428.1, and those two implied edges are the + // only thing holding the table together, since it draws no outer vertical at + // all. 2 is the smallest value that can mean "several agree", and going lower + // would let a single heading underline's right-hand end cut a column. + minSharedTermini = 2 + + // maxSegmentGap is how far apart two collinear rules may be and still be one + // printed line. This is what stops a joint being read as a terminus: a row + // rule crossing a column divider arrives as two segments meeting at it, and + // unmerged, that meeting point looks like a place where the rule stops. + // + // Both edges are real pages and the window is wide between them. The five are + // correct from 0.1 to 21. At 0 page 57 gains 3 cells and loses every spanning + // cell it has, because its full-width section rows get cut at the divider they + // cross. At 21.5 three of the five drop sharply — page 15 to 15 cells, page 20 + // to 7 — because its two side-by-side tables both place a row rule at y=103.5 + // with 22 units of white between them, and beyond that the two weld into one + // grid. The segments of a single printed rule, by contrast, meet within 0.01. + maxSegmentGap = 3.0 + + // maxSVGDepth bounds the recursion through and filter references. Cairo + // nests about six deep on these documents; the depth limit is there because + // the references come from an untrusted file and could be made to form a + // cycle. The visited set below already breaks a simple cycle, so this is the + // second of two guards, not the only one. + maxSVGDepth = 40 + + // maxRuleSVGBytes caps one page's SVG held in memory, for the reason + // [maxExtractedBytes] caps a document's text — except that this is per page + // and needs a much larger headroom than the page count suggests. + // + // Measured over both fixtures, whole documents: the columns manual's 68 pages + // come to 80.4 MB of SVG, and 30.4 MB of that is page 42 alone, a single page + // of exploded parts diagrams. The sequential manual's largest of 560 pages is + // 10.1 MB. So a cap sized from the average would reject a real page of a real + // manual; 64 MB is twice the largest measured and still bounds a hostile file. + maxRuleSVGBytes = 64 << 20 + + // minTableRows, minTableCols and minTableCells are the shape guard. + // + // It is not a formality. "This page draws a ruled line" is true of 68 of the + // columns manual's 68 pages — every page carries footer crop marks — so on its + // own it separates nothing at all. Requiring a grid of legible cells leaves 13 + // of the 68. See [tableHasText] for why 13 is still three too many. + minTableRows = 2 + minTableCols = 2 + minTableCells = 4 + + // minLegibleCellWidth and minLegibleCellHeight are what "legible" means in + // the shape guard: big enough to have held a word. They are a second, larger + // threshold than [minCellSize], which only asks whether a rectangle is a cell + // at all, and the difference between the two is worth 5 pages of the columns + // manual: at 8 by 8 the shape guard passes 18 of its 68 pages, at 24 by 10 it + // passes 13. The five it drops — 18, 20, 28, 34 and 42 — are exploded parts + // diagrams whose leader lines and callout boxes enclose grids of 9-to-20-unit + // slivers, too narrow to print a word in. + // + // 24 units is about a word and a half of body text at this document's 14pt, + // and 10 is under one line, so a real single-line cell clears both. The sweep + // over the five ground-truth pages holds up to 70 wide and 20 tall before any + // count moves — at 100 wide page 100 of the sequential manual stops being a + // table, and at 30 tall page 21 loses half its cells — so 24 by 10 sits with a + // factor of three of clearance on the side that matters. + // + // They bound the guard only. A narrow cell inside a table that passes is still + // returned, because it holds real content — a numbering column is narrow on + // purpose — and dropping it would lose text to make a count tidier. + minLegibleCellWidth = 24.0 + minLegibleCellHeight = 10.0 + + // minCellsWithText is the fraction of a table's cells that must contain some + // text. See [tableHasText]; this is the second guard and it is what separates + // a table from a grid of framed illustrations. + minCellsWithText = 0.5 + + // cellTextMargin is how far outside a cell a run may start and still be + // counted as inside it, in output units. A cell rectangle is the centre line + // of the rules that enclose it, so text set tight against a border sits a + // fraction outside. 2 units is a fifth of a line of body text. + cellTextMargin = 2.0 +) + +// RuleDirection is whether a ruled line runs across the page or down it. Only +// these two exist here: a table is enclosed by axis-aligned lines, and anything +// diagonal is not a table rule. +type RuleDirection uint8 + +// Horizontal is the zero value, which carries no meaning beyond being one of the +// two — a Rule is never usefully zero, since it needs coordinates. +const ( + Horizontal RuleDirection = iota + Vertical +) + +func (d RuleDirection) String() string { + if d == Vertical { + return "vertical" + } + return "horizontal" +} + +// Rule is one axis-aligned line the page draws, in the same 1.5-scaled +// coordinate space as [PageRuns] and a `pdftoppm -r 108` raster. +type Rule struct { + // Dir is which way the line runs. + Dir RuleDirection `json:"dir"` + // At is the coordinate the line holds constant: its y when Horizontal, its x + // when Vertical. + At float64 `json:"at"` + // Start and End are the coordinates it spans in its own direction — x when + // Horizontal, y when Vertical — with Start <= End always, so a caller never + // has to normalise one. + Start float64 `json:"start"` + End float64 `json:"end"` + // Thickness is how thick the line is drawn. Kept because it is the only + // evidence available for telling a table's outer border from a hairline + // divider, and because a caller that disagrees with [maxRuleThickness] can + // filter on it rather than re-extract. + Thickness float64 `json:"thickness"` + // Filled reports that this came from a thin filled shape rather than a + // stroked path. Both are real rules in both fixtures; this says which, so a + // wrong answer can be traced back to the right half of [ruleWalker]. + Filled bool `json:"filled,omitempty"` +} + +// Length is how far the rule runs. +func (r *Rule) Length() float64 { return r.End - r.Start } + +// CellRect is an axis-aligned rectangle: a cell's own bounds, or a table's. +type CellRect struct { + X0, Y0, X1, Y1 float64 +} + +// Width and Height are the rectangle's extent. +func (c CellRect) Width() float64 { return c.X1 - c.X0 } +func (c CellRect) Height() float64 { return c.Y1 - c.Y0 } + +// RuledCell is one cell of a [RuledTable]: where it is, where it sits in the +// grid, and how much text it holds. +type RuledCell struct { + // Row and Col are 0-based positions in the table's own grid, Row counting + // down and Col counting across. + Row int `json:"row"` + Col int `json:"col"` + // ColSpan is how many grid columns the cell covers, at least 1. A cell spans + // when the column rule does not run alongside it, which is how a full-width + // section heading inside a table is expressed — page 57's "Allgemeine (alle + // Funktionen)" is one cell across the whole table, not two. + // + // There is deliberately no RowSpan. A vertically merged cell is currently + // dropped rather than spanned, 10 of 47 on one measured page; that is an + // omission in the row walk recorded in docs/design/conversion.md, and adding + // the field before the walk finds them would promise something untrue. + ColSpan int `json:"colSpan"` + // Rect is the cell in page coordinates, on the centre lines of the rules that + // enclose it. + Rect CellRect `json:"rect"` + // Chars is how many runes of text sit inside the cell — runes rather than + // bytes, for the reason CONTRIBUTING.md gives: half of a real manual is not + // Latin, and a byte count would make a Cyrillic cell look half again fuller + // than the German one beside it. + // + // It is the text guard's evidence, kept per cell rather than reduced to the + // verdict, so that a page rejected as a grid of illustrations can be shown to + // have been rejected for the right reason. + Chars int `json:"chars"` +} + +// RuledTable is a grid of cells recovered from the lines a page draws. +type RuledTable struct { + // Box bounds every cell. + Box CellRect `json:"box"` + // Rows and Cols are the grid's dimensions. Cols counts grid columns, so a + // table whose every row is one spanning cell still reports the columns its + // boundaries imply. + Rows int `json:"rows"` + Cols int `json:"cols"` + // Cells are in reading order: down, then across. + Cells []RuledCell `json:"cells"` + // Rules are the lines this table was built from, for checking a wrong answer + // against a render. + Rules []Rule `json:"rules,omitempty"` +} + +// CellsWithText is how many of the table's cells hold any text at all — the +// numerator of the guard in [tableHasText]. +func (t *RuledTable) CellsWithText() int { + var n int + for i := range t.Cells { + if t.Cells[i].Chars > 0 { + n++ + } + } + return n +} + +// ExtractRules reads one page's ruled lines with pdftocairo. +// +// Like [ExtractRuns] it never mutates the file and calls nothing remote, so it +// is a pure function of the bytes and safe to re-run, which is what lets the job +// that calls it be idempotent. +// +// One page per invocation, which is the opposite of the choice [ExtractRuns] and +// [ExtractText] make, and it is forced rather than preferred: `pdftocairo -svg` +// writes one SVG document per page and, given a range, concatenates them into +// something that is not valid XML. The cost of that decision is real and +// measured — see docs/design/conversion.md, which also records the untested +// PostScript route that would avoid it. +// +// pdftocairo is optional at runtime. The error from a missing tool is returned +// plainly so a caller can convert a document without cell structure rather than +// fail it. +func ExtractRules(ctx context.Context, path string, page int) ([]Rule, error) { + if page < 1 { + return nil, fmt.Errorf("doc: page %d is not a page number", page) + } + bin, err := extern.Require(extern.PDFToCairo) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, extractTimeout) + defer cancel() + + // "-" is the output file, which keeps the SVG in memory: writing it would put + // a derived file beside the content-addressed blob store, which is immutable + // by design. -f and -l bound the range to the one page, for the reason above. + // #nosec G204 -- see ProbeInfo: bin comes from extern's own tool table, path + // is a blob-store path derived from a validated SHA-256 digest, and page is + // an int. + cmd := exec.CommandContext(ctx, bin, "-svg", + "-f", strconv.Itoa(page), "-l", strconv.Itoa(page), path, "-") + out := &limitedBuffer{limit: maxRuleSVGBytes} + var errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = out, &errOut + if err := cmd.Run(); err != nil { + if errors.Is(err, errOutputTooLarge) { + return nil, fmt.Errorf("%w (limit %d bytes)", errOutputTooLarge, maxRuleSVGBytes) + } + return nil, fmt.Errorf("doc: pdftocairo failed on page %d: %w: %s", + page, err, redact(strings.TrimSpace(errOut.String()), path)) + } + + rules, err := parseRules(out.buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("doc: reading pdftocairo output for %s page %d: %w", + redact(path, path), page, err) + } + return rules, nil +} + +// PageTables reads one page's ruled lines and returns the tables they enclose. +// +// The page's text is taken as a parameter rather than extracted again: the text +// guard needs it, [ExtractRuns] already produces it, and the probe has already +// paid for that call. Passing nil skips the text guard, which is what the +// discrimination measurements in docs/design/conversion.md were taken with — +// and what the numbers there show is not safe in production. +func PageTables(ctx context.Context, path string, page *PageRuns) ([]RuledTable, error) { + if page == nil { + return nil, errors.New("doc: PageTables needs a page to read") + } + rules, err := ExtractRules(ctx, path, page.No) + if err != nil { + return nil, err + } + return FindRuledTables(rules, page), nil +} + +// FindRuledTables groups rules into tables and returns those that pass both +// guards. +// +// Both are needed and neither is sufficient, which is measured rather than +// argued. See [minTableCells] for the shape guard and [tableHasText] for the +// text guard. +// +// page supplies the text the second guard reads, and the page box the run filter +// needs. A nil page applies the shape guard alone. +func FindRuledTables(rules []Rule, page *PageRuns) []RuledTable { + var text []TextRun + if page != nil { + // The same filter [DetectColumns] uses, and for a reason that bites here + // specifically: the columns fixture's illustrations are placed PDFs that + // each brought an InDesign filename slug along, scaled down with the + // artwork. Those slugs sit inside the illustration frames — which are + // exactly the shapes the text guard exists to reject — so counting them as + // text would let three pages of framed pictures through as tables. + text = usableRuns(page.Runs, page.Width, page.Height, &DroppedRuns{}) + } + + var out []RuledTable + for _, group := range rejoinTableFragments(ruleComponents(rules)) { + table := cellsOfTable(group) + if table == nil { + continue + } + countCellText(table, text) + // Both guards judge the table by its legible cells, and only those. A + // sliver cell is evidence about the drawing, not about whether this is a + // table, and letting a dozen of them outvote four real cells is how a + // diagram of callout boxes gets called a table. + legible := legibleCells(table) + if !hasTableShape(legible) { + continue + } + if page != nil && !tableHasText(legible) { + continue + } + out = append(out, *table) + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].Box.Y0 != out[j].Box.Y0 { + return out[i].Box.Y0 < out[j].Box.Y0 + } + return out[i].Box.X0 < out[j].Box.X0 + }) + return out +} + +// tableHasText reports whether enough of a table's cells hold words. +// +// This guard is not optional and the measurement is the whole argument for it. +// After the shape guard, 13 pages of the columns fixture look like tables, and +// three of them — 22, 38 and 44 — are grids of framed illustrations. They are +// ruled by exactly the evidence a table is ruled by: a border round each +// picture, aligned in rows and columns, because that is what a figure grid is. +// No amount of geometry separates them, and none should be expected to. +// +// What separates them is whether the cells hold words. Measured: 14 of those +// three pages' 15 cells contain zero characters, while every cell of page 57's +// twelve-cell table contains some, the smallest 27 runes. With this guard the +// columns fixture yields 10 table pages and the sequential one 170 — and 170 is +// 34 languages times 5 table pages, exactly, which is the strongest evidence +// available that the guard is not simply discarding awkward pages. +// +// Half is a deliberately loose threshold. A real table often has an empty cell — +// a blank corner above a row-label column, an unanswered row — and page 15 of +// the sequential fixture has three. What it never has is almost all of them +// empty. +// +// The failure this leaves is recorded in docs/design/conversion.md: a figure +// grid whose captions sit inside the frames would pass, and nothing here would +// catch it. +func tableHasText(legible []RuledCell) bool { + var withText int + for i := range legible { + if legible[i].Chars > 0 { + withText++ + } + } + return float64(withText) >= minCellsWithText*float64(len(legible)) +} + +// legibleCells are the cells big enough to be evidence about whether this is a +// table. See [minLegibleCellWidth]. +func legibleCells(t *RuledTable) []RuledCell { + out := make([]RuledCell, 0, len(t.Cells)) + for i := range t.Cells { + if t.Cells[i].Rect.Width() >= minLegibleCellWidth && + t.Cells[i].Rect.Height() >= minLegibleCellHeight { + out = append(out, t.Cells[i]) + } + } + return out +} + +// hasTableShape is the shape guard: enough legible cells, spread over at least +// two rows and two columns. See [minTableCells] for why it is needed and +// [tableHasText] for why it is not enough. +func hasTableShape(legible []RuledCell) bool { + rows, cols := make(map[int]bool), make(map[int]bool) + for i := range legible { + rows[legible[i].Row] = true + cols[legible[i].Col] = true + } + return len(legible) >= minTableCells && + len(rows) >= minTableRows && len(cols) >= minTableCols +} + +// countCellText fills in each cell's Chars from the page's text. +// +// A run belongs to the cell that contains it whole, within [cellTextMargin]. A +// run straddling a border belongs to neither, which is the honest reading: it is +// either a heading printed over the table or evidence that the cell rectangle is +// wrong, and counting it would hide both. +func countCellText(t *RuledTable, text []TextRun) { + for i := range t.Cells { + c := &t.Cells[i] + c.Chars = 0 + for j := range text { + r := &text[j] + if r.X >= c.Rect.X0-cellTextMargin && r.right() <= c.Rect.X1+cellTextMargin && + r.Y >= c.Rect.Y0-cellTextMargin && r.bottom() <= c.Rect.Y1+cellTextMargin { + c.Chars += len([]rune(r.Text)) + } + } + } +} + +// ruleSpan is a stretch of one printed line, in the line's own direction. It +// is not columns.go's span, which counts integer projection buckets. +type ruleSpan struct{ lo, hi float64 } + +// ruleComponents groups a page's rules into candidate tables. +// +// One grid per page is wrong, and page 57 is why. It carries two independent +// tables at different row positions, plus a heading underline and footer crop +// marks. Building one grid from every rule on the page fragments the real +// tables: the underline beneath the left table's title ends at x=128, which +// becomes a column boundary and splits the first column of a table it is not +// part of. +// +// So rules are grouped first, two ways. A vertical rule joins every horizontal +// it crosses, which is what a grid is. And two collinear rules join when they +// touch, because one printed line arrives in segments — but only when they +// touch: page 57's two side-by-side tables both put a row rule at y=103.5, and +// joining those on collinearity alone welds the two tables into one grid. They +// are 22 units apart while the segments of one rule meet exactly, so the gap +// decides. See [maxSegmentGap]. +func ruleComponents(rules []Rule) [][]Rule { + parent := make([]int, len(rules)) + for i := range parent { + parent[i] = i + } + find := func(i int) int { + for parent[i] != i { + parent[i] = parent[parent[i]] + i = parent[i] + } + return i + } + union := func(i, j int) { parent[find(i)] = find(j) } + + var hs, vs []int + for i := range rules { + if rules[i].Dir == Vertical { + vs = append(vs, i) + } else { + hs = append(hs, i) + } + } + for _, i := range hs { + for _, j := range vs { + if rulesCross(&rules[i], &rules[j]) { + union(i, j) + } + } + } + for _, group := range [][]int{hs, vs} { + for a := range group { + for b := a + 1; b < len(group); b++ { + ra, rb := &rules[group[a]], &rules[group[b]] + if math.Abs(ra.At-rb.At) > snapTolerance { + continue + } + if math.Min(ra.End, rb.End)-math.Max(ra.Start, rb.Start) >= -maxSegmentGap { + union(group[a], group[b]) + } + } + } + } + + byRoot := make(map[int][]Rule) + var roots []int + for i := range rules { + r := find(i) + if _, seen := byRoot[r]; !seen { + roots = append(roots, r) + } + byRoot[r] = append(byRoot[r], rules[i]) + } + out := make([][]Rule, 0, len(roots)) + for _, r := range roots { + out = append(out, byRoot[r]) + } + return out +} + +// rulesCross reports whether a horizontal and a vertical rule meet. +func rulesCross(h, v *Rule) bool { + return h.Start-snapTolerance <= v.At && v.At <= h.End+snapTolerance && + v.Start-snapTolerance <= h.At && h.At <= v.End+snapTolerance +} + +// rejoinTableFragments merges the stacked fragments of one table. +// +// A row that spans every column interrupts the column rule, so the rules alone +// see a table with section headings as several stacked grids — page 57's left +// table breaks into three. Fragments of one table share their horizontal extent +// exactly, which is what identifies them; a different table on the same page +// sits at a different x, and page 57's second table is 420 units to the right. +func rejoinTableFragments(groups [][]Rule) [][]Rule { + var out [][]Rule + for _, g := range groups { + lo, hi, ok := horizontalExtent(g) + if !ok { + out = append(out, g) + continue + } + var merged bool + for i := range out { + olo, ohi, ook := horizontalExtent(out[i]) + if ook && math.Abs(lo-olo) <= snapTolerance && math.Abs(hi-ohi) <= snapTolerance { + out[i] = append(out[i], g...) + merged = true + break + } + } + if !merged { + out = append(out, g) + } + } + return out +} + +// horizontalExtent is the x-range the group's horizontal rules span. +func horizontalExtent(rules []Rule) (lo, hi float64, ok bool) { + lo, hi = math.Inf(1), math.Inf(-1) + for i := range rules { + if rules[i].Dir != Horizontal { + continue + } + lo, hi = math.Min(lo, rules[i].Start), math.Max(hi, rules[i].End) + ok = true + } + return lo, hi, ok +} + +// gridLines are the distinct printed lines of one candidate table: a clustered +// position, and the union of the segments drawn along it. +type gridLines struct { + at []float64 + spans [][]ruleSpan +} + +// linesOf clusters a group's rules into printed lines in one direction. +func linesOf(rules []Rule, dir RuleDirection) gridLines { + var positions []float64 + for i := range rules { + if rules[i].Dir == dir { + positions = append(positions, rules[i].At) + } + } + out := gridLines{at: clusterPositions(positions)} + out.spans = make([][]ruleSpan, len(out.at)) + for i := range rules { + if rules[i].Dir != dir { + continue + } + k := nearestIndex(out.at, rules[i].At) + out.spans[k] = append(out.spans[k], ruleSpan{rules[i].Start, rules[i].End}) + } + for i := range out.spans { + out.spans[i] = mergeSpans(out.spans[i]) + } + return out +} + +// lookup returns the segments of the printed line nearest to at, or nil when +// none is within [snapTolerance]. +func (g *gridLines) lookup(at float64) []ruleSpan { + if len(g.at) == 0 { + return nil + } + k := nearestIndex(g.at, at) + if math.Abs(g.at[k]-at) > snapTolerance { + return nil + } + return g.spans[k] +} + +// clusterPositions groups coordinates that are the same printed line into one, +// returned sorted. Clustering is chained deliberately — a rule 2 units from its +// neighbour and 4 from the next is one line with the first, not a line of its +// own — which is what handles a rule drawn twice for a blend. +func clusterPositions(vals []float64) []float64 { + if len(vals) == 0 { + return nil + } + sorted := make([]float64, len(vals)) + copy(sorted, vals) + sort.Float64s(sorted) + + var out []float64 + group := []float64{sorted[0]} + flush := func() { + var sum float64 + for _, v := range group { + sum += v + } + out = append(out, sum/float64(len(group))) + } + for _, v := range sorted[1:] { + if v-group[len(group)-1] <= snapTolerance { + group = append(group, v) + continue + } + flush() + group = []float64{v} + } + flush() + return out +} + +func nearestIndex(at []float64, v float64) int { + best := 0 + for i := 1; i < len(at); i++ { + if math.Abs(at[i]-v) < math.Abs(at[best]-v) { + best = i + } + } + return best +} + +// mergeSpans unions the touching or overlapping segments of one printed line. +// +// This is what stops a joint being read as a terminus. A row rule crossing a +// column divider arrives as two segments meeting at it — 29.7 to 173.3 and 173.3 +// to 428.1 on page 57 — and unmerged, 173.3 looks like a place where the rule +// stops, so the full-width section row above it is wrongly cut in two. +func mergeSpans(spans []ruleSpan) []ruleSpan { + if len(spans) == 0 { + return nil + } + sorted := make([]ruleSpan, len(spans)) + copy(sorted, spans) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].lo < sorted[j].lo }) + + out := []ruleSpan{sorted[0]} + for _, s := range sorted[1:] { + last := &out[len(out)-1] + if s.lo-last.hi <= maxSegmentGap { + last.hi = math.Max(last.hi, s.hi) + continue + } + out = append(out, s) + } + return out +} + +// coveredFraction is how much of [lo,hi] the union of spans covers. +func coveredFraction(spans []ruleSpan, lo, hi float64) float64 { + if hi <= lo { + return 0 + } + var total, cur float64 + cur = lo + clipped := make([]ruleSpan, 0, len(spans)) + for _, s := range spans { + if s.hi > lo && s.lo < hi { + clipped = append(clipped, ruleSpan{math.Max(s.lo, lo), math.Min(s.hi, hi)}) + } + } + sort.Slice(clipped, func(i, j int) bool { return clipped[i].lo < clipped[j].lo }) + for _, s := range clipped { + if s.hi <= cur { + continue + } + total += s.hi - math.Max(s.lo, cur) + cur = math.Max(cur, s.hi) + } + return total / (hi - lo) +} + +// cellsOfTable turns one group of rules into a table, or nil if it is not a grid. +// +// Cells are walked a row at a time rather than tested rectangle by rectangle +// over the whole grid, and that is what expresses a spanning cell: page 57's +// section rows interrupt the column rule, so between them the full width is one +// cell rather than two empty ones. A grid-rectangle walk would report the pair. +func cellsOfTable(rules []Rule) *RuledTable { + rows := linesOf(rules, Horizontal) + cols := linesOf(rules, Vertical) + if len(rows.at) < 2 { + return nil + } + top, bottom := rows.at[0], rows.at[len(rows.at)-1] + + // Column boundaries nobody drew: an x where several row rules stop. See + // [minSharedTermini], and [Rule] for why a table may have no outer verticals + // at all — the columns fixture's tables draw none. + termini := make(map[float64]map[int]bool) + for i, spans := range rows.spans { + for _, s := range spans { + for _, e := range []float64{s.lo, s.hi} { + k := math.Round(e*10) / 10 + if termini[k] == nil { + termini[k] = make(map[int]bool) + } + termini[k][i] = true + } + } + } + var candidates []float64 + for k, atLines := range termini { + if len(atLines) >= minSharedTermini { + candidates = append(candidates, k) + } + } + + // A drawn vertical that does not run alongside the rows is furniture rather + // than a column boundary. The columns fixture's footer crop marks sit at + // x=29.8 and 188.2 spanning y=821-828, two units below its last table rule at + // 819.7 — close enough to have been joined to the table, and enough to cut + // its wide right-hand column in two. + for i, at := range cols.at { + if coveredFraction(cols.spans[i], top, bottom)*(bottom-top) >= minCellSize { + candidates = append(candidates, at) + } + } + xs := clusterPositions(candidates) + if len(xs) < 2 { + return nil + } + + out := &RuledTable{Cols: len(xs) - 1, Rules: rules} + box := CellRect{X0: math.Inf(1), Y0: math.Inf(1), X1: math.Inf(-1), Y1: math.Inf(-1)} + for j := 0; j+1 < len(rows.at); j++ { + y0, y1 := rows.at[j], rows.at[j+1] + if y1-y0 < minCellSize { + continue + } + topSpans, botSpans := rows.spans[j], rows.spans[j+1] + + edges := make(map[int]bool) + for i, at := range xs { + if coveredFraction(cols.lookup(at), y0, y1) >= minSideCoverage { + edges[i] = true + continue + } + // Or the row rules above and below both stop here, which is the only + // evidence an undrawn outer border leaves. + if spansEndNear(topSpans, at) && spansEndNear(botSpans, at) { + edges[i] = true + } + } + ordered := make([]int, 0, len(edges)) + for i := range edges { + ordered = append(ordered, i) + } + sort.Ints(ordered) + + var rowCells []RuledCell + for k := 0; k+1 < len(ordered); k++ { + a, b := ordered[k], ordered[k+1] + x0, x1 := xs[a], xs[b] + if x1-x0 < minCellSize { + continue + } + if coveredFraction(topSpans, x0, x1) < minSideCoverage || + coveredFraction(botSpans, x0, x1) < minSideCoverage { + continue + } + rowCells = append(rowCells, RuledCell{ + Row: out.Rows, Col: a, ColSpan: b - a, + Rect: CellRect{X0: x0, Y0: y0, X1: x1, Y1: y1}, + }) + } + if len(rowCells) == 0 { + // A band that encloses nothing is not a row. Numbering it would leave + // gaps in Row and make a table of three rows report five. + continue + } + out.Rows++ + out.Cells = append(out.Cells, rowCells...) + box.X0, box.Y0 = math.Min(box.X0, rowCells[0].Rect.X0), math.Min(box.Y0, y0) + box.X1 = math.Max(box.X1, rowCells[len(rowCells)-1].Rect.X1) + box.Y1 = math.Max(box.Y1, y1) + } + if len(out.Cells) == 0 { + return nil + } + out.Box = box + return out +} + +// spansEndNear reports whether any of the line's segments terminates at at. +func spansEndNear(spans []ruleSpan, at float64) bool { + for _, s := range spans { + if math.Abs(s.lo-at) <= snapTolerance || math.Abs(s.hi-at) <= snapTolerance { + return true + } + } + return false +} + +// --- reading cairo's SVG --------------------------------------------------- + +// matrix is an SVG transform: a b c d e f, applied as x' = ax + cy + e. +type matrix [6]float64 + +var identity = matrix{1, 0, 0, 1, 0, 0} + +// compose returns m then n, i.e. the matrix that applies n's mapping inside m's. +func (m matrix) compose(n matrix) matrix { + return matrix{ + m[0]*n[0] + m[2]*n[1], m[1]*n[0] + m[3]*n[1], + m[0]*n[2] + m[2]*n[3], m[1]*n[2] + m[3]*n[3], + m[0]*n[4] + m[2]*n[5] + m[4], m[1]*n[4] + m[3]*n[5] + m[5], + } +} + +func (m matrix) apply(x, y float64) (px, py float64) { + return m[0]*x + m[2]*y + m[4], m[1]*x + m[3]*y + m[5] +} + +// scale is the mean of the matrix's two axis scale factors, for turning a +// stroke width in user units into one on the page. +func (m matrix) scale() float64 { + return (math.Hypot(m[0], m[1]) + math.Hypot(m[2], m[3])) / 2 +} + +// svgNode is the part of an SVG element this code reads. Attributes it does not +// read — colour, opacity, stroke-linecap — are dropped at parse time rather than +// carried, because a 30 MB page makes the difference measurable. +type svgNode struct { + tag string + id string + transform string + filter string + href string + d string + stroke string + fill string + strokeWidth string + // clip is the element's own `clip-path` attribute, unresolved. It is kept + // because a shape's box has to be its visible extent rather than its + // geometric one — see clip.go, which is where the reference is followed. + clip string + x, y, w, h float64 + kids []*svgNode +} + +// skippedTags are elements whose contents are never page ink: masking and +// gradient machinery, embedded rasters, stylesheets. is deliberately +// not here — it is not walked for ink either, but its feImage references are how +// a hoisted compositing group is found again. +// +// is not here either, and used to be. Its contents are not ink, but +// they are geometry a shape's box depends on, so it is read by [readClipPath] +// into a rectangle instead of being skipped — see clip.go. +var skippedTags = map[string]bool{ + "mask": true, "linearGradient": true, "radialGradient": true, + "pattern": true, "symbol": true, "image": true, "style": true, +} + +// svgDoc is a parsed page: the element tree, and the two indexes needed to +// follow a reference into . +type svgDoc struct { + root *svgNode + // byID resolves an href or a filter reference. First declaration wins, which + // matches how a browser resolves a duplicate id. + byID map[string]*svgNode + // filterRefs maps a filter's id to the ids its feImage children pull in. + filterRefs map[string][]string + // clips maps a 's id to the extent it admits, in its own user space. + // Kept unresolved for the reason clip.go's header gives: the same definition + // resolves to two different page rectangles depending on which reference + // pulled it in. + clips map[string]clipDef +} + +// parseSVG reads cairo's SVG into a tree. +// +// Go's decoder never resolves an external entity, which is the obvious worry +// about parsing XML derived from an untrusted PDF and is worth stating rather +// than leaving to be rediscovered: there is no entity expansion to disable here, +// and the DOCTYPE cairo emits is inert. What is not free is size, which is why +// the caller caps the bytes — see [maxRuleSVGBytes]. +// +// Glyph outlines are dropped as the tree is built rather than filtered later. +// Cairo writes every glyph of the page as a filled path under +// ``, and on a page of text those are almost all of the +// document: skipping the subtree is both the correct exclusion — see [Rule] for +// why it must be structural — and what keeps a 30 MB page from becoming a tree +// of a million nodes. +func parseSVG(data []byte) (*svgDoc, error) { + dec := xml.NewDecoder(bytes.NewReader(data)) + doc := &svgDoc{ + root: &svgNode{tag: "#document"}, + byID: make(map[string]*svgNode), + filterRefs: make(map[string][]string), + clips: make(map[string]clipDef), + } + stack := []*svgNode{doc.root} + + for { + tok, err := dec.Token() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + switch v := tok.(type) { + case xml.StartElement: + // A becomes a rectangle rather than a subtree, and is + // consumed here so its children never reach the tree at all. + if v.Name.Local == "clipPath" { + id := attrValue(&v, "id") + def, err := readClipPath(dec, &v) + if err != nil { + return nil, err + } + // First declaration wins, matching how byID resolves a duplicate. + if _, seen := doc.clips[id]; !seen && id != "" { + doc.clips[id] = def + } + continue + } + node := &svgNode{tag: v.Name.Local} + for _, a := range v.Attr { + switch a.Name.Local { + case "id": + node.id = a.Value + case "transform": + node.transform = a.Value + case "filter": + node.filter = a.Value + case "clip-path": + node.clip = a.Value + case "href": + // Both the xlink form and the plain one; cairo writes xlink:href, + // but the plain attribute is the current spelling and costs nothing + // to accept. + node.href = a.Value + case "d": + node.d = a.Value + case "stroke": + node.stroke = a.Value + case "fill": + node.fill = a.Value + case "stroke-width": + node.strokeWidth = a.Value + case "x": + node.x = parseFloat(a.Value) + case "y": + node.y = parseFloat(a.Value) + case "width": + node.w = parseFloat(a.Value) + case "height": + node.h = parseFloat(a.Value) + } + } + if skippedTags[node.tag] || strings.HasPrefix(node.id, "glyph-") { + if err := dec.Skip(); err != nil { + return nil, err + } + continue + } + parent := stack[len(stack)-1] + parent.kids = append(parent.kids, node) + if node.id != "" { + if _, seen := doc.byID[node.id]; !seen { + doc.byID[node.id] = node + } + } + stack = append(stack, node) + case xml.EndElement: + if len(stack) > 1 { + stack = stack[:len(stack)-1] + } + } + } + + for _, f := range collectByTag(doc.root, "filter") { + var refs []string + for _, fe := range collectAll(f) { + if strings.HasPrefix(fe.href, "#") { + refs = append(refs, fe.href[1:]) + } + } + doc.filterRefs[f.id] = refs + } + return doc, nil +} + +// attrValue reads one attribute off an element that is being consumed from the +// stream rather than built into a node. +func attrValue(e *xml.StartElement, name string) string { + for _, a := range e.Attr { + if a.Name.Local == name { + return a.Value + } + } + return "" +} + +func collectByTag(n *svgNode, tag string) []*svgNode { + var out []*svgNode + if n.tag == tag { + out = append(out, n) + } + for _, k := range n.kids { + out = append(out, collectByTag(k, tag)...) + } + return out +} + +func collectAll(n *svgNode) []*svgNode { + out := []*svgNode{n} + for _, k := range n.kids { + out = append(out, collectAll(k)...) + } + return out +} + +// visitKey identifies one visit to a node under one matrix. The same node is +// legitimately drawn twice under different transforms, and must be collected +// both times; visiting it twice under the same one is a reference cycle. +type visitKey struct { + node *svgNode + m [6]int64 +} + +// ruleWalker accumulates rules while walking the tree. +type ruleWalker struct { + doc *svgDoc + rules []Rule + visited map[visitKey]bool +} + +// parseRules reads every axis-aligned rule a page draws. +func parseRules(data []byte) ([]Rule, error) { + doc, err := parseSVG(data) + if err != nil { + return nil, err + } + w := &ruleWalker{doc: doc, visited: make(map[visitKey]bool)} + // Only the body is walked from the top. is entered exclusively through + // the reference that pulls a definition back in, because that reference + // carries the matrix which cancels the definition's own — see the compositing + // group trap in this file's header. + for _, kid := range doc.root.kids { + w.walkBody(kid, identity, clipBox{}, 0) + } + return dedupeRules(w.rules), nil +} + +// walkBody walks a subtree, skipping , which walk enters by reference. +func (w *ruleWalker) walkBody(n *svgNode, m matrix, clip clipBox, depth int) { + if n.tag == "defs" { + return + } + w.walk(n, m, clip, depth) +} + +func (w *ruleWalker) walk(n *svgNode, m matrix, clip clipBox, depth int) { + if depth > maxSVGDepth || strings.HasPrefix(n.id, "glyph-") { + return + } + key := visitKey{node: n} + for i, v := range m { + key.m[i] = int64(math.Round(v * 1e4)) + } + if w.visited[key] { + return + } + w.visited[key] = true + + m = m.compose(parseTransform(n.transform)) + // The element's own clip narrows its ancestors' — resolved after its transform, + // which is the user space the clip is written in, and the same composition that + // keeps a hoisted compositing group's coordinates right. + if box, ok := w.doc.clipAt(n.clip, m); ok { + clip = clip.intersect(box) + if clip.empty() { + return + } + } + + // A filtered group's real content was hoisted into ; follow it carrying + // this element's matrix, which is what cancels the hoisted group's own. + if id, ok := refID(n.filter); ok { + for _, ref := range w.doc.filterRefs[id] { + if target := w.doc.byID[ref]; target != nil { + w.walk(target, m, clip, depth+1) + } + } + } + if n.tag == "use" { + if strings.HasPrefix(n.href, "#") { + if target := w.doc.byID[n.href[1:]]; target != nil { + w.walk(target, m, clip, depth+1) + } + } + } + + switch { + case n.tag == "path" && n.stroke != "" && n.stroke != "none": + width := 1.0 + if n.strokeWidth != "" { + width = parseFloat(n.strokeWidth) + } + width *= m.scale() * svgPointScale + for _, sub := range subpaths(n.d) { + for i := 0; i+1 < len(sub); i++ { + w.stroked(m, clip, sub[i], sub[i+1], width) + } + } + case n.tag == "path" && n.fill != "" && n.fill != "none": + for _, sub := range subpaths(n.d) { + w.filled(m, clip, sub) + } + case n.tag == "rect" && n.fill != "" && n.fill != "none": + w.filled(m, clip, []point{ + {n.x, n.y}, {n.x + n.w, n.y}, {n.x + n.w, n.y + n.h}, {n.x, n.y + n.h}, + }) + } + + for _, kid := range n.kids { + w.walkBody(kid, m, clip, depth+1) + } +} + +// stroked records a stroked segment if it is an axis-aligned rule. +// +// The clip is applied to the segment's own extent, so a rule drawn longer than +// the window it is painted in is recorded at the length the page prints. It is +// applied here rather than only in [inkWalker] because a rule that is clipped +// away is not a printed line, and a cell boundary is read off where the rules +// end — see [cellsOfTable]. Measured on the five ground-truth pages, every cell +// count is unchanged, which says the tables of these two documents draw their +// rules inside their clips. +func (w *ruleWalker) stroked(m matrix, clip clipBox, p, q point, width float64) { + x0, y0 := m.apply(p.x, p.y) + x1, y1 := m.apply(q.x, q.y) + x0, y0, x1, y1 = x0*svgPointScale, y0*svgPointScale, x1*svgPointScale, y1*svgPointScale + box, visible := clip.apply(CellRect{ + X0: math.Min(x0, x1), Y0: math.Min(y0, y1), + X1: math.Max(x0, x1), Y1: math.Max(y0, y1), + }) + if !visible { + return + } + dx, dy := math.Abs(x1-x0), math.Abs(y1-y0) + switch { + case dy <= axisTolerance && box.Width() >= minRuleLength: + w.rules = append(w.rules, Rule{ + Dir: Horizontal, At: (y0 + y1) / 2, + Start: box.X0, End: box.X1, Thickness: width, + }) + case dx <= axisTolerance && box.Height() >= minRuleLength: + w.rules = append(w.rules, Rule{ + Dir: Vertical, At: (x0 + x1) / 2, + Start: box.Y0, End: box.Y1, Thickness: width, + }) + } +} + +// filled records a filled subpath if it is a thin sliver, which is one of the +// two ways a rule is drawn. Subpaths of more than six points are not slivers; +// bounding a curve's flattened outline would call a filled logo a rule. +// +// How much this is worth was measured by removing it, and the answer is less +// than the volume suggests. 800 of the columns manual's 4,360 rules are filled +// slivers and 103 of the sequential manual's 16,959, so 18% of one document's +// rules come from here. But with this branch disabled, both documents return the +// same tables: 13 pages passing the shape guard and 10 passing both in one, 171 +// and 170 in the other, and all five ground-truth pages come back with their +// exact cell counts. The only figure that moves is how many pages draw a rule at +// all, 226 down to 195 in the sequential manual. +// +// So no table in either fixture depends on a filled rule. It is kept because the +// shape is real, cheap to read and documented in poppler's output rather than +// guessed — a document that rules its tables with filled slivers instead of +// strokes is an ordinary thing for a designer to produce, and the next manual +// gets no say in which of the two this code understands. +func (w *ruleWalker) filled(m matrix, clip clipBox, sub []point) { + if len(sub) > 6 || len(sub) < 2 { + return + } + minX, minY := math.Inf(1), math.Inf(1) + maxX, maxY := math.Inf(-1), math.Inf(-1) + for _, p := range sub { + x, y := m.apply(p.x, p.y) + x, y = x*svgPointScale, y*svgPointScale + minX, maxX = math.Min(minX, x), math.Max(maxX, x) + minY, maxY = math.Min(minY, y), math.Max(maxY, y) + } + box, visible := clip.apply(CellRect{X0: minX, Y0: minY, X1: maxX, Y1: maxY}) + if !visible { + return + } + minX, minY, maxX, maxY = box.X0, box.Y0, box.X1, box.Y1 + width, height := maxX-minX, maxY-minY + switch { + case width >= minRuleLength && height > 0 && height <= maxRuleThickness: + w.rules = append(w.rules, Rule{ + Dir: Horizontal, At: (minY + maxY) / 2, + Start: minX, End: maxX, Thickness: height, Filled: true, + }) + case height >= minRuleLength && width > 0 && width <= maxRuleThickness: + w.rules = append(w.rules, Rule{ + Dir: Vertical, At: (minX + maxX) / 2, + Start: minY, End: maxY, Thickness: width, Filled: true, + }) + } +} + +// dedupeRules collapses one printed line drawn several times, keeping the +// thickest. A table drawn in a blend group is drawn twice by construction — once +// as the hoisted definition and once at the use site — so this is not a tidying +// pass but part of reading that output correctly. +func dedupeRules(rules []Rule) []Rule { + type key struct { + dir RuleDirection + at, start, end int64 + } + best := make(map[key]int, len(rules)) + var order []key + for i := range rules { + r := &rules[i] + k := key{r.Dir, int64(math.Round(r.At * 2)), + int64(math.Round(r.Start * 2)), int64(math.Round(r.End * 2))} + if j, seen := best[k]; seen { + if r.Thickness > rules[j].Thickness { + best[k] = i + } + continue + } + best[k] = i + order = append(order, k) + } + out := make([]Rule, 0, len(order)) + for _, k := range order { + out = append(out, rules[best[k]]) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Dir != out[j].Dir { + return out[i].Dir < out[j].Dir + } + if out[i].At != out[j].At { + return out[i].At < out[j].At + } + return out[i].Start < out[j].Start + }) + return out +} + +// point is a coordinate in an SVG element's own user space. +type point struct{ x, y float64 } + +// parseTransform reads an SVG transform list. The functions compose left to +// right, which is the order they are written in. +func parseTransform(t string) matrix { + m := identity + for i := 0; i < len(t); { + // The function name runs to the opening parenthesis. + for i < len(t) && (t[i] == ' ' || t[i] == ',' || t[i] == '\t' || t[i] == '\n') { + i++ + } + start := i + for i < len(t) && t[i] != '(' { + i++ + } + if i >= len(t) { + break + } + name := strings.TrimSpace(t[start:i]) + i++ + argStart := i + for i < len(t) && t[i] != ')' { + i++ + } + args := scanFloats(t[argStart:min(i, len(t))]) + if i < len(t) { + i++ + } + m = m.compose(transformMatrix(name, args)) + } + return m +} + +func transformMatrix(name string, a []float64) matrix { + at := func(i int) float64 { + if i < len(a) { + return a[i] + } + return 0 + } + switch name { + case "matrix": + if len(a) < 6 { + return identity + } + return matrix{a[0], a[1], a[2], a[3], a[4], a[5]} + case "translate": + return matrix{1, 0, 0, 1, at(0), at(1)} + case "scale": + sx := at(0) + sy := sx + if len(a) > 1 { + sy = a[1] + } + return matrix{sx, 0, 0, sy, 0, 0} + case "rotate": + r := at(0) * math.Pi / 180 + // Only the two-argument form's centre is ignored, and cairo never writes + // it; a rotation about a point would shift a rule, so it is safer to + // compose the pure rotation than to guess. + return matrix{math.Cos(r), math.Sin(r), -math.Sin(r), math.Cos(r), 0, 0} + default: + return identity + } +} + +// refID reads the id out of a url(#id) attribute value. +func refID(v string) (string, bool) { + v = strings.TrimSpace(v) + if !strings.HasPrefix(v, "url(#") || !strings.HasSuffix(v, ")") { + return "", false + } + return v[len("url(#") : len(v)-1], true +} + +// subpaths splits a path's d attribute into subpaths of points, with every +// curve flattened to its endpoint. +// +// Flattening is right rather than lazy here: a rule is a straight line, so a +// curve's interior cannot be part of one, and its endpoints are exactly what +// closes the rounded corner of a table border. What it costs is that a filled +// shape's bounding box is computed from endpoints alone, which is why +// [ruleWalker.filled] refuses subpaths of more than six points. +func subpaths(d string) [][]point { + toks := tokenizePath(d) + var out [][]point + var cur []point + var pt, start point + cmd := byte('M') + + for i := 0; i < len(toks); { + if toks[i].isCmd { + cmd = toks[i].cmd + i++ + if cmd == 'Z' || cmd == 'z' { + if len(cur) > 0 { + cur = append(cur, start) + out = append(out, cur) + cur = nil + } + pt = start + } + continue + } + var nums []float64 + for i < len(toks) && !toks[i].isCmd { + nums = append(nums, toks[i].num) + i++ + } + upper := cmd &^ 0x20 + k := commandArity(upper) + rel := cmd >= 'a' + for j := 0; j+k <= len(nums); j += k { + a := nums[j : j+k] + switch upper { + case 'H': + if rel { + pt = point{pt.x + a[0], pt.y} + } else { + pt = point{a[0], pt.y} + } + case 'V': + if rel { + pt = point{pt.x, pt.y + a[0]} + } else { + pt = point{pt.x, a[0]} + } + default: + nx, ny := a[k-2], a[k-1] + if rel { + pt = point{pt.x + nx, pt.y + ny} + } else { + pt = point{nx, ny} + } + } + // Only the first pair of a moveto starts a subpath. The rest are + // implicit linetos, which is the SVG rule and not a shortcut: cairo + // writes a table border as one M followed by several coordinate pairs. + if upper == 'M' && j == 0 { + if len(cur) > 0 { + out = append(out, cur) + } + cur = []point{pt} + start = pt + continue + } + cur = append(cur, pt) + } + } + if len(cur) > 0 { + out = append(out, cur) + } + return out +} + +func commandArity(upper byte) int { + switch upper { + case 'C': + return 6 + case 'A': + return 7 + case 'S', 'Q': + return 4 + case 'H', 'V': + return 1 + default: + return 2 + } +} + +// pathToken is either a command letter or a number. +type pathToken struct { + cmd byte + num float64 + isCmd bool +} + +// tokenizePath scans a d attribute. Hand-written rather than a regexp because +// path data is nearly all of a 30 MB page, and this is the inner loop over it. +func tokenizePath(d string) []pathToken { + out := make([]pathToken, 0, len(d)/6) + for i := 0; i < len(d); { + c := d[i] + switch { + case c == ' ' || c == ',' || c == '\t' || c == '\n' || c == '\r': + i++ + case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'): + out = append(out, pathToken{cmd: c, isCmd: true}) + i++ + default: + v, n := scanFloat(d[i:]) + if n == 0 { + // Not a number and not a letter: skip it rather than stall. + i++ + continue + } + out = append(out, pathToken{num: v}) + i += n + } + } + return out +} + +// scanFloat reads one number from the front of s. used is how many bytes it +// consumed, and 0 means there was no number there. +func scanFloat(s string) (value float64, used int) { + i := 0 + if i < len(s) && (s[i] == '-' || s[i] == '+') { + i++ + } + digits := false + for i < len(s) && s[i] >= '0' && s[i] <= '9' { + i, digits = i+1, true + } + if i < len(s) && s[i] == '.' { + i++ + for i < len(s) && s[i] >= '0' && s[i] <= '9' { + i, digits = i+1, true + } + } + if !digits { + return 0, 0 + } + if i < len(s) && (s[i] == 'e' || s[i] == 'E') { + j := i + 1 + if j < len(s) && (s[j] == '-' || s[j] == '+') { + j++ + } + expDigits := j + for j < len(s) && s[j] >= '0' && s[j] <= '9' { + j++ + } + if j > expDigits { + i = j + } + } + v, err := strconv.ParseFloat(s[:i], 64) + if err != nil { + return 0, 0 + } + return v, i +} + +// scanFloats reads every number in s, ignoring whatever separates them. +func scanFloats(s string) []float64 { + var out []float64 + for i := 0; i < len(s); { + v, n := scanFloat(s[i:]) + if n == 0 { + i++ + continue + } + out = append(out, v) + i += n + } + return out +} + +// parseFloat reads an SVG length, ignoring a unit suffix. A malformed one reads +// as 0, which is what an absent attribute means too — a rect with no width draws +// nothing either way. +func parseFloat(s string) float64 { + v, _ := scanFloat(strings.TrimSpace(s)) + return v +} diff --git a/internal/doc/rules_fixture_test.go b/internal/doc/rules_fixture_test.go new file mode 100644 index 0000000..f965d57 --- /dev/null +++ b/internal/doc/rules_fixture_test.go @@ -0,0 +1,370 @@ +package doc_test + +import ( + "context" + "os" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" +) + +// These drive the ruled-line reader against both real manuals, and the numbers +// they assert are the ones a human counted off a render — not the ones the code +// happened to produce. docs/design/conversion.md records the count and the misses +// for each page; where a count is short of what is printed, the shortfall is +// asserted too, so that a change which silently loses a different set of cells +// cannot pass by arriving at the same total. +// +// Both manuals are needed and neither is enough. The columns manual is where the +// hard cases are — tables with no outer border, tables drawn in a blend group, +// pages of framed illustrations ruled exactly like tables. The sequential manual +// is where the volume is, and its 34 translations of the same 5 table pages give +// the one arithmetic check available on the whole pipeline: 34 times 5 is 170. + +// rulesFixture loads a fixture and the tools this file needs. +func rulesFixture(t *testing.T, name string) (path string, pages []doc.PageRuns) { + t.Helper() + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixture and run the real-document tests", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFToHTML, extern.PDFToCairo} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + m, err := fixture.Load(fixturesDir, name) + if err != nil { + t.Fatalf("load manifest: %v", err) + } + cached, err := m.Fetch(context.Background()) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + runs, err := doc.ExtractRuns(context.Background(), cached) + if err != nil { + t.Fatalf("ExtractRuns: %v", err) + } + return cached, runs +} + +// pageOf finds one page's runs by its printed number. +func pageOf(t *testing.T, pages []doc.PageRuns, no int) *doc.PageRuns { + t.Helper() + for i := range pages { + if pages[i].No == no { + return &pages[i] + } + } + t.Fatalf("the document has no page %d", no) + return nil +} + +// tablesOf reads one page's tables. +func tablesOf(t *testing.T, path string, pages []doc.PageRuns, no int) []doc.RuledTable { + t.Helper() + tables, err := doc.PageTables(context.Background(), path, pageOf(t, pages, no)) + if err != nil { + t.Fatalf("PageTables page %d: %v", no, err) + } + return tables +} + +func cellCount(tables []doc.RuledTable) int { + var n int + for i := range tables { + n += len(tables[i].Cells) + } + return n +} + +// TestRecoveredCellsMatchTheCountedPages is the ground truth. Every expected +// number here was arrived at by drawing the cells on a `pdftoppm -r 108` render +// and comparing them with the printed table, and the two pages that come back +// short are short for a reason that is understood and recorded. +func TestRecoveredCellsMatchTheCountedPages(t *testing.T) { + for _, f := range []struct { + fixture string + pages []struct { + no, printed, want int + why string + } + }{ + {"thomas-drybox-amfibia", []struct { + no, printed, want int + why string + }{ + {57, 29, 25, "the misses are two header rows whose top border is not drawn"}, + }}, + {"dreame-l40-ultra", []struct { + no, printed, want int + why string + }{ + {20, 12, 12, ""}, + {100, 16, 16, ""}, + {21, 32, 32, ""}, + {15, 47, 37, "the misses are exactly the vertically merged cells"}, + }}, + } { + path, pages := rulesFixture(t, f.fixture) + for _, p := range f.pages { + tables := tablesOf(t, path, pages, p.no) + got := cellCount(tables) + if got != p.want { + t.Errorf("%s page %d: recovered %d cells, want %d of the %d printed (%s)", + f.fixture, p.no, got, p.want, p.printed, p.why) + continue + } + if len(tables) == 0 { + t.Errorf("%s page %d: %d cells but no table", f.fixture, p.no, got) + } + // Every cell of every one of these tables holds text. That is a + // stronger statement than the guard's, and it is what makes the counts + // mean "cells you could read something out of". + for i := range tables { + tb := &tables[i] + if tb.CellsWithText() != len(tb.Cells) { + t.Errorf("%s page %d table %d: %d of %d cells hold no text", + f.fixture, p.no, i, len(tb.Cells)-tb.CellsWithText(), len(tb.Cells)) + } + } + } + } +} + +// TestPage57IsTwoTablesWithNoOuterVerticals pins the shape of the hardest page, +// not just its total. It is the page whose tables are drawn in a blend group +// hoisted into , whose row rules stop at x=29.7 and x=428.1 with no +// vertical drawn there at all, and whose full-width section rows fragment one +// printed table into three components. +func TestPage57IsTwoTablesWithNoOuterVerticals(t *testing.T) { + path, pages := rulesFixture(t, "thomas-drybox-amfibia") + tables := tablesOf(t, path, pages, 57) + + if len(tables) != 2 { + t.Fatalf("got %d tables, want 2 side by side: %+v", len(tables), tables) + } + left, right := &tables[0], &tables[1] + if len(left.Cells) != 12 || len(right.Cells) != 13 { + t.Errorf("tables hold %d and %d cells, want 12 and 13", + len(left.Cells), len(right.Cells)) + } + // The implied outer edges. Nothing draws a vertical at either, and the whole + // table depends on believing the x where every row rule stops. + if !near(left.Box.X0, 29.7, 0.5) || !near(left.Box.X1, 428.1, 0.5) { + t.Errorf("left table spans x=%.1f..%.1f, want the implied 29.7..428.1", + left.Box.X0, left.Box.X1) + } + if left.Box.X1 > right.Box.X0 { + t.Errorf("the two tables overlap: %.1f..%.1f and %.1f..%.1f", + left.Box.X0, left.Box.X1, right.Box.X0, right.Box.X1) + } + // The rules were found at all, which is what fails if is skipped: that + // reading returns 18 rules for this page, every one of them a footer crop mark. + var rules int + for i := range tables { + rules += len(tables[i].Rules) + } + if rules < 40 { + t.Errorf("the two tables were built from %d rules; skipping leaves 18 "+ + "crop marks and no table rule at all", rules) + } + // A spanning cell exists, because a section row interrupts the column rule. + var spanning int + for i := range tables { + for j := range tables[i].Cells { + if tables[i].Cells[j].ColSpan > 1 { + spanning++ + } + } + } + if spanning == 0 { + t.Error("no cell spans a column, but this page's section rows run the full width") + } +} + +// TestFramedIllustrationsAreNotTables is the text guard, on the pages that need +// it. They are grids of boxed pictures: ruled in rows and columns, aligned, and +// empty. Geometry passes them and only the words reject them. +func TestFramedIllustrationsAreNotTables(t *testing.T) { + path, pages := rulesFixture(t, "thomas-drybox-amfibia") + // 38 was the third of these until the clip was read. Its frame's left edge is + // drawn 30 units past where it is painted, and that over-long rule was what + // closed a cell; clipped, the page does not pass the shape guard, so it can no + // longer test what happens after it. It still comes back as no table, which is + // asserted below for all three. + for _, no := range []int{22, 44} { + page := pageOf(t, pages, no) + rules, err := doc.ExtractRules(context.Background(), path, no) + if err != nil { + t.Fatalf("ExtractRules page %d: %v", no, err) + } + if got := doc.FindRuledTables(rules, page); len(got) != 0 { + t.Errorf("page %d is a grid of framed illustrations but came back as %d "+ + "table(s) with %d cells", no, len(got), cellCount(got)) + } + // And the reason it must be the text guard that rejects them: the shape + // guard passes. If this stops being true the guard above has stopped being + // tested by this page, even though the page still comes out right. + shaped := doc.FindRuledTables(rules, nil) + if len(shaped) == 0 { + t.Errorf("page %d no longer passes the shape guard, so it no longer "+ + "exercises the text guard", no) + } + } + + // Page 38 keeps its half of the claim: still a grid of framed illustrations, + // still no table, now rejected by the shape guard instead. + page := pageOf(t, pages, 38) + rules, err := doc.ExtractRules(context.Background(), path, 38) + if err != nil { + t.Fatalf("ExtractRules page 38: %v", err) + } + if got := doc.FindRuledTables(rules, page); len(got) != 0 { + t.Errorf("page 38 came back as %d table(s) with %d cells", len(got), cellCount(got)) + } +} + +// TestPagesThatMustNotBeTables covers the two other ways a page can look like +// one: a contents page ruled with leader lines and a page of three parallel +// language columns, which is the case layouts.md records that geometry cannot +// distinguish from a table. Plus a page with no vector rules at all, which must +// come back empty rather than fail. +func TestPagesThatMustNotBeTables(t *testing.T) { + colPath, colPages := rulesFixture(t, "thomas-drybox-amfibia") + for _, no := range []int{2, 13} { + if got := tablesOf(t, colPath, colPages, no); len(got) != 0 { + t.Errorf("columns manual page %d came back as %d table(s) with %d cells; "+ + "it is a contents page or parallel language columns, not a table", + no, len(got), cellCount(got)) + } + } + + seqPath, seqPages := rulesFixture(t, "dreame-l40-ultra") + rules, err := doc.ExtractRules(context.Background(), seqPath, 8) + if err != nil { + t.Fatalf("ExtractRules: %v", err) + } + if len(rules) != 0 { + t.Errorf("sequential manual page 8 draws %d rules, and it draws none", len(rules)) + } + if got := tablesOf(t, seqPath, seqPages, 8); len(got) != 0 { + t.Errorf("a page with no rules produced %d tables", len(got)) + } +} + +// TestBothGuardsAreNeededOverTheWholeDocument is the discrimination measurement, +// asserted rather than only recorded. It is the test that fails if either guard +// is dropped, and the numbers are what docs/design/conversion.md argues from. +// +// It reads both documents page by page, so it is the slow one here: about 6 +// seconds for 68 pages and 37 for 560. +func TestBothGuardsAreNeededOverTheWholeDocument(t *testing.T) { + for _, want := range []struct { + fixture string + pages int + anyRule, shape, both int + note string + }{ + // Every page carries footer crop marks, so "has a ruled line" is the whole + // document and separates nothing. + // 12 pass the shape guard where 13 did before the clip was read: page 38's + // frame has a left edge drawn 30 units longer than it is painted, and that + // over-long rule was closing a cell. Clipped to what the page prints, the + // page no longer looks like a table at all — checked against a 432 dpi + // render of x=85-145, y=150-290, where the stroke ends at y=238 and the + // unclipped extent ran to 268.7. The page's answer is unchanged either way: + // it is a grid of framed illustrations and produces no table. + {"thomas-drybox-amfibia", 68, 68, 12, 10, + "the 2 pages between the shape guard and the text guard are 22 and 44"}, + // 170 is 34 languages times 5 table pages, exactly. + {"dreame-l40-ultra", 560, 226, 171, 170, "170 = 34 languages x 5 table pages"}, + } { + path, pages := rulesFixture(t, want.fixture) + var anyRule, shape, both int + var tablePages []int + for i := range pages { + page := &pages[i] + rules, err := doc.ExtractRules(context.Background(), path, page.No) + if err != nil { + t.Fatalf("ExtractRules page %d: %v", page.No, err) + } + if len(rules) > 0 { + anyRule++ + } + if len(doc.FindRuledTables(rules, nil)) > 0 { + shape++ + } + if len(doc.FindRuledTables(rules, page)) > 0 { + both++ + tablePages = append(tablePages, page.No) + } + } + if len(pages) != want.pages { + t.Fatalf("%s: %d pages, want %d", want.fixture, len(pages), want.pages) + } + if anyRule != want.anyRule || shape != want.shape || both != want.both { + t.Errorf("%s: %d pages draw a rule, %d pass the shape guard, %d pass both; "+ + "want %d, %d, %d (%s)", want.fixture, anyRule, shape, both, + want.anyRule, want.shape, want.both, want.note) + } + if want.fixture == "thomas-drybox-amfibia" { + // Its tables are pages 52 to 61, which conversion.md corrects from the + // 57-61 the manifest recorded. + for i, no := range tablePages { + if no != 52+i { + t.Errorf("table pages are %v, want 52 to 61 contiguous", tablePages) + break + } + } + } + } +} + +// TestRuleCoordinateSpaceMatchesTheRuns is the check every count above depends +// on: cairo writes PDF points and everything else in this package works in +// poppler's 1.5-scaled space, so if the ratio were wrong every cell would be +// two thirds of its size and no text would fall inside one. +func TestRuleCoordinateSpaceMatchesTheRuns(t *testing.T) { + for _, f := range []struct { + name string + page int + }{ + {"thomas-drybox-amfibia", 57}, + {"dreame-l40-ultra", 20}, + } { + path, pages := rulesFixture(t, f.name) + page := pageOf(t, pages, f.page) + rules, err := doc.ExtractRules(context.Background(), path, f.page) + if err != nil { + t.Fatalf("ExtractRules: %v", err) + } + if len(rules) == 0 { + t.Fatalf("%s page %d draws no rules", f.name, f.page) + } + // Every rule must land on the page poppler reports, within a small + // fraction of it. The slack is not politeness: the columns manual is a + // print PDF and draws its footer crop marks on the trim, measured at + // y=852.1 to 858.3 on a page poppler calls 850 tall, because cairo's own + // media box is 566.929pt — 850.39 — and the marks sit outside even that. + // A wrong scale is off by a third, not by 1%, so this still catches one. + for i := range rules { + r := &rules[i] + along, across := page.Width, page.Height + if r.Dir == doc.Vertical { + along, across = page.Height, page.Width + } + slack := 0.02 * across + if r.At < -slack || r.At > across+slack || + r.Start < -0.02*along || r.End > along+0.02*along { + t.Errorf("%s page %d: %s rule at %.1f spanning %.1f..%.1f is off a "+ + "%.0fx%.0f page — the coordinate scale is wrong", + f.name, f.page, r.Dir, r.At, r.Start, r.End, page.Width, page.Height) + break + } + } + } +} + +func near(a, b, tol float64) bool { return a-b < tol && b-a < tol } diff --git a/internal/doc/rules_internal_test.go b/internal/doc/rules_internal_test.go new file mode 100644 index 0000000..bcb3649 --- /dev/null +++ b/internal/doc/rules_internal_test.go @@ -0,0 +1,495 @@ +package doc + +import ( + "fmt" + "math" + "strings" + "testing" +) + +// Unit tests for the SVG rule reader and the cell walk. No poppler and no PDF: +// rules_fixture_test.go drives the real tool against the real manuals. +// +// compositingSVG reproduces the shapes cairo actually emits, and every element +// in it is there because a real page has one. The attribute spellings are copied +// from the columns fixture's page 57 rather than invented — `stroke="rgb(100%, +// 100%, 100%)"`, a `matrix` with a negative y scale, a filled path whose last +// subpath is a bare moveto — because each of those broke an earlier reading of +// it. +// +// The table is drawn inside a hoisted blend group, which is the trap this file's +// header describes: `` +// lives in and is pulled back by ``. The two translations cancel, so a correct +// reading returns the coordinates as written and either mistake is visible in +// the numbers rather than in a count: skipping loses the table entirely, +// and entering it without composing the use site's matrix shifts every +// coordinate by (30, 15) output units. +const compositingSVG = ` + + + + + + + + + + + + + + + + + + + + + + + + + + +` + +// compositingPage is text sitting in all four cells of compositingSVG's table, +// in the output coordinate space, so the text guard has something to read. +func compositingPage() *PageRuns { + page := &PageRuns{No: 1, Width: 300, Height: 150} + for _, cell := range [][2]float64{{40, 35}, {115, 35}, {40, 65}, {115, 65}} { + page.Runs = append(page.Runs, TextRun{ + X: cell[0], Y: cell[1], Width: 50, Height: 10, Text: "content", + }) + } + return page +} + +func TestRulesInsideACompositingGroupKeepTheirCoordinates(t *testing.T) { + rules, err := parseRules([]byte(compositingSVG)) + if err != nil { + t.Fatalf("parseRules: %v", err) + } + + // The table's six rules, at the coordinates written in the group, times 1.5. + // If were skipped these would all be missing; if the use site's + // transform were not composed every one would be 30 further right and 15 + // further down. + want := []Rule{ + {Dir: Horizontal, At: 30, Start: 30, End: 180}, + {Dir: Horizontal, At: 60, Start: 30, End: 180}, + {Dir: Horizontal, At: 90, Start: 30, End: 180}, + {Dir: Vertical, At: 30, Start: 30, End: 90}, + {Dir: Vertical, At: 105, Start: 30, End: 90}, + {Dir: Vertical, At: 180, Start: 30, End: 90}, + } + for _, w := range want { + if !hasRule(rules, w) { + t.Errorf("missing %s rule at %.1f spanning %.1f..%.1f\ngot %s", + w.Dir, w.At, w.Start, w.End, formatRules(rules)) + } + } + + // The body's own two rules: one stroked, one a filled sliver. Both are real + // forms in both fixtures — 800 of the columns manual's 4,360 rules are filled — + // and this is the only test that fails if the filled branch is removed, since + // no table in either fixture depends on one. See [ruleWalker.filled]. + if !hasRule(rules, Rule{Dir: Horizontal, At: 142.5, Start: 15, End: 120}) { + t.Errorf("the body's stroked rule is missing\ngot %s", formatRules(rules)) + } + var filled *Rule + for i := range rules { + if rules[i].Filled { + filled = &rules[i] + } + } + if filled == nil { + t.Fatalf("the filled sliver was not read as a rule\ngot %s", formatRules(rules)) + } + if filled.Dir != Horizontal || math.Abs(filled.At-127.875) > 0.01 || + math.Abs(filled.Start-15) > 0.01 || math.Abs(filled.End-120) > 0.01 { + t.Errorf("filled sliver is %s at %.3f spanning %.2f..%.2f, want horizontal at "+ + "127.875 spanning 15..120", filled.Dir, filled.At, filled.Start, filled.End) + } + + // A glyph outline is a filled path of exactly the shape a hairline rule has, + // which is why it is excluded structurally. This one is a 0.5 by 80 sliver at + // x=190pt, so if the were walked — directly or through the + // that references it — there would be a vertical rule at x=285. + for i := range rules { + if rules[i].Dir == Vertical && math.Abs(rules[i].At-285) < 1 { + t.Errorf("a glyph outline was read as a rule at x=%.1f", rules[i].At) + } + } + if len(rules) != 8 { + t.Errorf("got %d rules, want 8 (six table, one stroked, one filled)\n%s", + len(rules), formatRules(rules)) + } +} + +func TestCompositingGroupYieldsATable(t *testing.T) { + rules, err := parseRules([]byte(compositingSVG)) + if err != nil { + t.Fatalf("parseRules: %v", err) + } + tables := FindRuledTables(rules, compositingPage()) + if len(tables) != 1 { + t.Fatalf("got %d tables, want 1: %+v", len(tables), tables) + } + got := &tables[0] + if got.Rows != 2 || got.Cols != 2 || len(got.Cells) != 4 { + t.Errorf("got %dx%d with %d cells, want 2x2 with 4", + got.Rows, got.Cols, len(got.Cells)) + } + want := []CellRect{ + {X0: 30, Y0: 30, X1: 105, Y1: 60}, {X0: 105, Y0: 30, X1: 180, Y1: 60}, + {X0: 30, Y0: 60, X1: 105, Y1: 90}, {X0: 105, Y0: 60, X1: 180, Y1: 90}, + } + for i, w := range want { + if i >= len(got.Cells) { + break + } + if !sameRect(got.Cells[i].Rect, w) { + t.Errorf("cell %d is %+v, want %+v", i, got.Cells[i].Rect, w) + } + if got.Cells[i].Chars == 0 { + t.Errorf("cell %d holds no text, but a run was placed in it", i) + } + } + if got.Box != (CellRect{X0: 30, Y0: 30, X1: 180, Y1: 90}) { + t.Errorf("box is %+v, want 30,30-180,90", got.Box) + } +} + +// TestTheTextGuardRejectsAGridOfFrames is the hermetic form of what pages 22, 38 +// and 44 of the columns fixture are: the same rules, the same grid, and no words +// in the cells. Geometry cannot tell the two apart and the text is the only +// difference, so the same SVG is used for both and only the runs change. +func TestTheTextGuardRejectsAGridOfFrames(t *testing.T) { + rules, err := parseRules([]byte(compositingSVG)) + if err != nil { + t.Fatalf("parseRules: %v", err) + } + + // One cell of the four holds a caption; the other three are pictures. That is + // 25% and the guard wants half. + framed := &PageRuns{No: 1, Width: 300, Height: 150, Runs: []TextRun{ + {X: 40, Y: 35, Width: 50, Height: 10, Text: "Fig. 1"}, + }} + if got := FindRuledTables(rules, framed); len(got) != 0 { + t.Errorf("a grid with text in 1 of 4 cells was read as a table: %+v", got) + } + + // Two of four is exactly half, which passes: a real table often has an empty + // cell, and page 15 of the sequential fixture has three. + half := &PageRuns{No: 1, Width: 300, Height: 150, Runs: []TextRun{ + {X: 40, Y: 35, Width: 50, Height: 10, Text: "left"}, + {X: 115, Y: 65, Width: 50, Height: 10, Text: "right"}, + }} + if got := FindRuledTables(rules, half); len(got) != 1 { + t.Errorf("a grid with text in 2 of 4 cells was rejected, want one table: %+v", got) + } + + // And with no page at all the shape guard stands alone, which is what the + // discrimination measurement in docs/design/conversion.md was taken with. + if got := FindRuledTables(rules, nil); len(got) != 1 { + t.Errorf("the shape guard alone found %d tables, want 1", len(got)) + } +} + +// TestATableWithNoOuterVerticals is the columns fixture's own table shape, which +// draws no left or right border at all: the row rules simply stop together, and +// the page reads as bordered because of it. Its section rows also interrupt the +// column rule, which fragments one printed table into three connected components +// and must be rejoined. +func TestATableWithNoOuterVerticals(t *testing.T) { + // Three row bands. The middle one is a full-width section heading, so the + // column rule is drawn only above and below it. + var rules []Rule + for _, y := range []float64{100, 130, 160, 190} { + rules = append(rules, Rule{Dir: Horizontal, At: y, Start: 30, End: 430, Thickness: 0.75}) + } + rules = append(rules, + Rule{Dir: Vertical, At: 170, Start: 100, End: 130, Thickness: 0.75}, + Rule{Dir: Vertical, At: 170, Start: 160, End: 190, Thickness: 0.75}, + ) + + page := &PageRuns{No: 1, Width: 892, Height: 850} + for _, r := range [][2]float64{{40, 105}, {200, 105}, {40, 135}, {40, 165}, {200, 165}} { + page.Runs = append(page.Runs, TextRun{ + X: r[0], Y: r[1], Width: 60, Height: 17, Text: "cell text", + }) + } + + tables := FindRuledTables(rules, page) + if len(tables) != 1 { + t.Fatalf("got %d tables, want 1 — the fragments were not rejoined: %+v", + len(tables), tables) + } + got := &tables[0] + if len(got.Cells) != 5 { + t.Fatalf("got %d cells, want 5 (two, one spanning, two): %+v", + len(got.Cells), got.Cells) + } + // The middle row is one cell across the whole width, which is what walking a + // row at a time expresses and a grid-rectangle walk cannot. + middle := got.Cells[2] + if middle.Rect.X0 != 30 || middle.Rect.X1 != 430 { + t.Errorf("the section row is %+v, want one cell from 30 to 430", middle.Rect) + } + if middle.ColSpan != 2 { + t.Errorf("the section row spans %d columns, want 2", middle.ColSpan) + } + // The outer edges at x=30 and x=430 are drawn by nothing; they are believed + // because every row rule stops there. + if got.Box.X0 != 30 || got.Box.X1 != 430 { + t.Errorf("box is %+v, want the implied outer edges 30 and 430", got.Box) + } +} + +// TestTheShapeGuardNeedsMoreThanALine covers what "has a ruled line" is worth on +// its own: nothing. Every page of the columns fixture draws footer crop marks, +// so the guard has to ask for a grid. +func TestTheShapeGuardNeedsMoreThanALine(t *testing.T) { + cropMarks := []Rule{ + {Dir: Horizontal, At: 825, Start: 30, End: 188, Thickness: 0.5}, + {Dir: Vertical, At: 30, Start: 821, End: 828, Thickness: 0.5}, + {Dir: Vertical, At: 188, Start: 821, End: 828, Thickness: 0.5}, + } + if got := FindRuledTables(cropMarks, nil); len(got) != 0 { + t.Errorf("footer crop marks were read as a table: %+v", got) + } + + // A grid of four cells too narrow to hold a word is the other half of it: + // the exploded parts diagrams of five of the columns fixture's pages enclose + // grids of 9-to-20-unit slivers exactly like this one. + var slivers []Rule + for _, y := range []float64{100, 120, 140} { + slivers = append(slivers, Rule{Dir: Horizontal, At: y, Start: 100, End: 140}) + } + for _, x := range []float64{100, 120, 140} { + slivers = append(slivers, Rule{Dir: Vertical, At: x, Start: 100, End: 140}) + } + if got := FindRuledTables(slivers, nil); len(got) != 0 { + t.Errorf("a 20-unit grid of slivers was read as a table: %+v", got) + } +} + +// TestTwoTablesSideBySideStaySeparate is the fragmenting case from page 57: two +// independent tables that share a row position. Joining collinear rules without +// asking whether they touch welds them into one grid. +func TestTwoTablesSideBySideStaySeparate(t *testing.T) { + var rules []Rule + for _, x0 := range []float64{30, 450} { + for _, y := range []float64{100, 140, 180} { + rules = append(rules, Rule{Dir: Horizontal, At: y, Start: x0, End: x0 + 380}) + } + for _, dx := range []float64{0, 190, 380} { + rules = append(rules, Rule{Dir: Vertical, At: x0 + dx, Start: 100, End: 180}) + } + } + got := FindRuledTables(rules, nil) + if len(got) != 2 { + t.Fatalf("got %d tables, want 2 — a shared row position welded them: %+v", + len(got), got) + } + if got[0].Box.X1 > got[1].Box.X0 { + t.Errorf("the two tables overlap: %+v and %+v", got[0].Box, got[1].Box) + } + for i := range got { + if len(got[i].Cells) != 4 { + t.Errorf("table %d has %d cells, want 4", i, len(got[i].Cells)) + } + } +} + +func TestParseTransformComposesInWritingOrder(t *testing.T) { + // A translate then a scale: the scale applies inside the translate, so the + // point (1,1) lands at 10+2, 20+3 rather than at (10+1)*2. + m := parseTransform("translate(10, 20) scale(2, 3)") + if x, y := m.apply(1, 1); x != 12 || y != 23 { + t.Errorf("translate then scale put (1,1) at (%g,%g), want (12,23)", x, y) + } + // The form cairo actually writes for a flipped page. + m = parseTransform("matrix(0.998785, 0, 0, -0.998785, 19.771253, 68.984759)") + x, y := m.apply(0, 0) + if math.Abs(x-19.771253) > 1e-9 || math.Abs(y-68.984759) > 1e-9 { + t.Errorf("matrix put the origin at (%g,%g), want its translation", x, y) + } + // A negative determinant must not make the stroke width negative. + if s := m.scale(); math.Abs(s-0.998785) > 1e-9 { + t.Errorf("scale of a y-flipped matrix is %g, want 0.998785", s) + } + if got := parseTransform(""); got != identity { + t.Errorf("an absent transform is %v, want the identity", got) + } + if got := parseTransform("skewX(30)"); got != identity { + t.Errorf("an unhandled transform is %v, want the identity", got) + } +} + +func TestSubpathsFlattenCurvesAndSplitOnMoveto(t *testing.T) { + // A moveto's extra coordinate pairs are implicit linetos, which is how cairo + // writes a table border, and a curve contributes only its endpoint. + got := subpaths("M 10 10 20 10 L 30 10 C 40 10 50 10 60 10 Z M 100 100 L 110 100") + if len(got) != 2 { + t.Fatalf("got %d subpaths, want 2: %v", len(got), got) + } + // M(10,10) 20,10 30,10 60,10 and the close back to the start. + want := []point{{10, 10}, {20, 10}, {30, 10}, {60, 10}, {10, 10}} + if len(got[0]) != len(want) { + t.Fatalf("first subpath is %v, want %v", got[0], want) + } + for i := range want { + if got[0][i] != want[i] { + t.Errorf("first subpath point %d is %v, want %v", i, got[0][i], want[i]) + } + } + // Relative commands, and the horizontal and vertical shorthands. + got = subpaths("m 10 10 h 20 v 5 l -20 0 z") + if len(got) != 1 { + t.Fatalf("got %d subpaths, want 1: %v", len(got), got) + } + if last := got[0][3]; last != (point{10, 15}) { + t.Errorf("relative path reached %v, want 10,15", last) + } + if got := subpaths(""); got != nil { + t.Errorf("an empty path is %v, want nothing", got) + } +} + +func TestScanFloatReadsTheFormsCairoWrites(t *testing.T) { + for _, tc := range []struct { + in string + want float64 + used int + }{ + {"0.691406", 0.691406, 8}, + {"-0.000986825 ", -0.000986825, 12}, + {"13.729858%", 13.729858, 9}, + {"1e-5", 1e-5, 4}, + {"1.5E+2x", 150, 6}, + {".5", 0.5, 2}, + {"5.", 5, 2}, + {"-", 0, 0}, + {"none", 0, 0}, + {"", 0, 0}, + // An exponent with no digits is not part of the number: "2e" is 2. + {"2e", 2, 1}, + } { + got, used := scanFloat(tc.in) + if got != tc.want || used != tc.used { + t.Errorf("scanFloat(%q) = %g after %d bytes, want %g after %d", + tc.in, got, used, tc.want, tc.used) + } + } +} + +// TestParseSVGRejectsNothingItCannotRead checks the degradation, since the input +// is derived from an untrusted PDF: malformed XML is an error, but an SVG with +// nothing recognisable in it is simply a page with no rules. +func TestParseSVGDegradesRatherThanPanics(t *testing.T) { + if _, err := parseRules([]byte("")); err == nil { + t.Error("truncated XML parsed without error") + } + for _, in := range []string{ + ``, + ``, + ``, + ``, + ``, + // A reference cycle, which the visited set has to break. + ``, + } { + got, err := parseRules([]byte(in)) + if err != nil { + t.Errorf("parseRules(%q): %v", in, err) + } + if len(got) != 0 { + t.Errorf("parseRules(%q) found %d rules, want none", in, len(got)) + } + } +} + +func TestMergeSpansUnionsTheSegmentsOfOneRule(t *testing.T) { + // The measured shape: a row rule crossing a column divider arrives as two + // segments meeting at it. Unmerged, that meeting point reads as a terminus. + got := mergeSpans([]ruleSpan{{173.3, 428.1}, {29.7, 173.3}}) + if len(got) != 1 || got[0] != (ruleSpan{29.7, 428.1}) { + t.Errorf("got %v, want one span 29.7..428.1", got) + } + // Two rules 22 units apart are two lines, which is what keeps page 57's + // side-by-side tables separate. + got = mergeSpans([]ruleSpan{{29.7, 428.1}, {450.2, 848.7}}) + if len(got) != 2 { + t.Errorf("got %v, want two spans", got) + } +} + +func TestCoveredFraction(t *testing.T) { + for _, tc := range []struct { + spans []ruleSpan + lo, hi float64 + want float64 + }{ + {[]ruleSpan{{0, 10}}, 0, 10, 1}, + {[]ruleSpan{{0, 5}}, 0, 10, 0.5}, + {[]ruleSpan{{0, 6}, {4, 10}}, 0, 10, 1}, + {nil, 0, 10, 0}, + {[]ruleSpan{{0, 10}}, 10, 10, 0}, + {[]ruleSpan{{-100, 100}}, 0, 10, 1}, + } { + if got := coveredFraction(tc.spans, tc.lo, tc.hi); math.Abs(got-tc.want) > 1e-9 { + t.Errorf("coveredFraction(%v, %g, %g) = %g, want %g", + tc.spans, tc.lo, tc.hi, got, tc.want) + } + } +} + +func TestClusterPositionsChains(t *testing.T) { + // One printed rule drawn twice for a blend comes back a fraction apart and + // must be one line. Clustering chains, so a run of near neighbours is one + // line rather than splitting at the first pair further than the tolerance. + // 100, 102 and 104 are one line although the ends are 4 apart, because each + // is within the tolerance of its neighbour; 143.2 is a line of its own. + got := clusterPositions([]float64{104, 143.2, 100, 102}) + if len(got) != 2 { + t.Fatalf("got %v, want two lines", got) + } + if got[0] != 102 || got[1] != 143.2 { + t.Errorf("got %v, want the mean of the chain and 143.2", got) + } + if got := clusterPositions(nil); got != nil { + t.Errorf("got %v, want nothing", got) + } +} + +func hasRule(rules []Rule, want Rule) bool { + for i := range rules { + r := &rules[i] + if r.Dir == want.Dir && math.Abs(r.At-want.At) < 0.01 && + math.Abs(r.Start-want.Start) < 0.01 && math.Abs(r.End-want.End) < 0.01 { + return true + } + } + return false +} + +func sameRect(a, b CellRect) bool { + return math.Abs(a.X0-b.X0) < 0.01 && math.Abs(a.Y0-b.Y0) < 0.01 && + math.Abs(a.X1-b.X1) < 0.01 && math.Abs(a.Y1-b.Y1) < 0.01 +} + +func formatRules(rules []Rule) string { + var b strings.Builder + for i := range rules { + r := &rules[i] + fmt.Fprintf(&b, "\n %s at %.3f from %.3f to %.3f thick %.3f", + r.Dir, r.At, r.Start, r.End, r.Thickness) + if r.Filled { + b.WriteString(" filled") + } + } + return b.String() +} diff --git a/internal/doc/runs.go b/internal/doc/runs.go new file mode 100644 index 0000000..e64a15c --- /dev/null +++ b/internal/doc/runs.go @@ -0,0 +1,548 @@ +package doc + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "os/exec" + "strings" + + "github.com/gordon2/manualbox/internal/extern" +) + +// Positioned text is read with pdftohtml, not pdftotext, because a column is a +// geometric fact and pdftotext reports no coordinates at all. It is the input +// [DetectColumns] and [ColumnLanguages] were written against, and until this +// existed nothing could supply it: both were only ever called with coordinates +// written by hand in a test, so a disagreement between what poppler reports and +// what those tests assumed could not be caught. +// +// The cost is measured, whole-document, poppler 26.07.0: +// +// 560-page, 15 MB manual 1.79 s 3.8 MB of XML 34,413 runs +// 68-page, 9 MB manual 3.18 s 920 KB of XML 7,493 runs +// +// So it is the same order as the pdftotext pass it runs beside (1.8 s on the +// first document) and stays inside the free stages. One invocation over the whole +// document rather than one per page, for the reason [ExtractText] gives: process +// startup dominates everything else. +// +// Coordinates come back at 1.5 times the PDF's own points — 108 dpi against 72 — +// which is not a detail to work around but the property that makes the geometry +// checkable: a `pdftoppm -r 108` raster matches this space 1:1, so a detected +// column can be drawn on the rendered page and looked at. Measured on both +// fixtures: 918/612.283 and 892/595.276, 850/566.929. + +// PageRuns is one page's positioned text and the page box it sits in. +type PageRuns struct { + // No is the 1-based page number in the original PDF. + No int + // Width and Height are the page box as poppler reports it — the PDF's own + // size scaled by 1.5. Carried per page rather than per document because + // nothing guarantees a manual's pages are one size, and every threshold in + // [DetectColumns] is a fraction of the page it is judging. + Width, Height float64 + // Runs is the page's text, in the order the tool emitted it. + Runs []TextRun +} + +// HasText reports whether any text was found on the page. +func (p *PageRuns) HasText() bool { return len(p.Runs) > 0 } + +// Font is the typeface a run is set in, as far as poppler's XML reveals it. +// +// It exists because size alone cannot find a heading. Measured over both +// fixtures, whole-document, in characters — TestFontDistributionOfBothManuals +// prints all of this and more: +// +// sequential manual parallel-columns manual +// the body size 11: 64.1% of chars 14: 84.0% of chars +// the next size up 17: 14.7% 17: 5.5% +// ...which is set in MiSans, regular FuturaCon-Lig, the body face +// heavier at the body size MiSans-Medium 5.1% FuturaCon-Med 17.2% +// +// Row three is the trap [extern.PDFToHTML] records: the larger face is not a +// heading face, it is what the safety text is set in, so "larger than body means +// heading" promotes prose — and it is 14.7% of a 560-page manual. Row four is the +// other half of it, and it is worse on the columns manual, where 84.0% of the +// characters are one size and so size discriminates almost nothing: what +// separates that document's emphasis from its body is only the weight of the +// face at the same size. +// +// Weight and slope are therefore carried from two independent signals rather +// than one, because poppler reports them two ways and the two disagree in both +// directions. See [Weight] for the disagreement and its numbers. +type Font struct { + // Size is the size poppler declares for the run's fontspec — in the same + // 1.5-scaled space as the coordinates and [PageRuns.Height], not in the PDF's + // own points. Measured by writing a known PDF: 11pt and 17pt text come back + // as 17 and 26. So it is directly comparable to a run's Height and to the + // page box, and a caller must not print it as a point size. + // + // Read as a float although poppler currently prints integers, for the reason + // [xmlText] gives about coordinates. + Size float64 `json:"size,omitempty"` + // Family is the family verbatim, subset prefix and all — "HUTKLI+FuturaCon-Lig", + // not "FuturaCon-Lig". Kept raw on purpose: [Weight] and Oblique below are + // conclusions drawn from this string by a name heuristic, and a caller that + // distrusts one of them must be able to see what it was concluded from. + // The prefix is not noise either; it identifies the embedded subset, and two + // subsets sharing a base name can differ in real weight (measured below). + Family string `json:"family,omitempty"` + // Weight is what the family name says the weight is. WeightUnknown when the + // name says nothing, which is the zero value and the honest reading. + Weight Weight `json:"weight,omitempty"` + // Oblique is whether the family name says italic or oblique. + Oblique bool `json:"oblique,omitempty"` + // MarkedBold and MarkedItalic are poppler's own verdict: it wrapped the + // run's text in or . That is a different signal from the two above, + // not a restatement of them — poppler reads the font descriptor, so it + // catches emphasis no name declares and misses emphasis every name declares. + MarkedBold bool `json:"markedBold,omitempty"` + MarkedItalic bool `json:"markedItalic,omitempty"` +} + +// Weight is a font weight as the embedded font's own name declares it. +// +// It is graded rather than a bold/not-bold pair, and that is the measured shape +// of the problem rather than a preference. Poppler's markup is effectively +// boolean and it draws the line above Medium: counted per family across all its +// sizes, it wraps every run of a name that says Bold, Demibold, SemiBold or +// Xbold — FuturaCon-Bol 157 of 159 runs, Function-Xbold 81/81, MiSans-Demibold +// 2861/2861, MiSansLatin-Demibold 607/607, Arimo-SemiBold 89/89, +// Sarabun-SemiBold 105/105, Alibaba-PuHuiTi-B 23/23 — and not one run of a name +// that says Medium: FuturaCon-Med 0 of 1,241, MiSans-Medium 0 of 2,887, +// Arimo-Medium 0/248, Sarabun-Medium 0/157. On the columns manual that discards +// the single most useful distinction in the document, since FuturaCon-Med is +// 17.2% of its characters at the same size as its body text. +// +// The reverse disagreement is just as real, which is why the markup is kept too +// and neither signal is folded into the other. Poppler marks bold where no name +// admits it: FuturaBQ 78 of 78 runs, FuturaStd 11/11, Calibri 12/12, and 498 of +// the 16,426 runs whose family reads plainly "MiSans" — same base name, +// different embedded subsets, different actual weight. Slope disagrees both ways +// as well: appears only on oblique-named families (Futura-BooObl 12/12, +// FuturaCon-MedObl 9/9, FuturaCon-BolObl 5/5), yet FuturaCon-BooObl gets 66 runs +// and not one . +// +// What settles it is that the two manuals rely on opposite signals. On the +// columns manual the names carry the document — 93.4% of its characters are in a +// face whose name states a weight, and poppler marks only 1.5% of them bold. On +// the sequential manual it is the reverse: 73.2% of its characters are in a face +// whose name states nothing at all (MiSans, MiSansLatin, Sarabun, Arimo, +// HarmonyOS_Sans_Naskh_Arabic), and poppler's markup is the only weight there is. +// Either signal alone reads one of these two documents and is close to blind on +// the other, and there are two documents. +// +// So a name heuristic is exactly the sort of thing that misbehaves on the next +// document, and it is used here only because dropping it would discard the +// weight of 93% of one manual. It is reported beside Family and beside poppler's +// verdict, never instead of them, and no rule here decides what a heading is — +// that needs this data first. +type Weight int8 + +// The scale is the one type designers use, ordered so that comparing two weights +// is meaningful. WeightUnknown is deliberately below every named weight and is +// the zero value: a run whose font did not resolve, or whose name says nothing, +// must not compare as body weight. +const ( + WeightUnknown Weight = iota + WeightLight + WeightRegular + WeightMedium + WeightSemibold + WeightBold + WeightHeavy +) + +func (w Weight) String() string { + switch w { + case WeightLight: + return "light" + case WeightRegular: + return "regular" + case WeightMedium: + return "medium" + case WeightSemibold: + return "semibold" + case WeightBold: + return "bold" + case WeightHeavy: + return "heavy" + default: + return "unknown" + } +} + +// obliqueTails are the name fragments that mean a slanted face. Matched as a +// tail rather than a whole token because the two are glued in real names: +// "FuturaCon-BooObl" is Book plus oblique in one token, and stripping the tail +// is what leaves "Boo" behind to be read as a weight. +var obliqueTails = []string{"oblique", "italic", "ital", "obl"} + +// weightTails maps a name fragment to a weight, longest and most specific +// first, because these are matched as tails of a token and the shorter ones are +// suffixes of the longer: read in the wrong order "Xbold" is bold and +// "Demibold" is bold. The abbreviations are all measured in the two fixtures — +// "Bol", "Lig", "Med", "Boo" — and "Boo" being one letter from "Bol" while +// meaning the opposite is why nothing here matches a prefix. +var weightTails = []struct { + tail string + weight Weight +}{ + {"ultrabold", WeightHeavy}, + {"extrabold", WeightHeavy}, + {"semibold", WeightSemibold}, + {"demibold", WeightSemibold}, + {"xbold", WeightHeavy}, + {"black", WeightHeavy}, + {"heavy", WeightHeavy}, + {"bold", WeightBold}, + {"regular", WeightRegular}, + {"normal", WeightRegular}, + {"medium", WeightMedium}, + {"light", WeightLight}, + {"roman", WeightRegular}, + {"thin", WeightLight}, + {"book", WeightRegular}, + {"demi", WeightSemibold}, + {"semi", WeightSemibold}, + {"bol", WeightBold}, + {"boo", WeightRegular}, + {"lig", WeightLight}, + {"med", WeightMedium}, +} + +// weightLetters are the single-letter weight codes, matched only as a whole +// token. "Alibaba-PuHuiTi-B" and "-M" and "-R" are all in the sequential +// manual. They are excluded from tail matching because one letter at the end of +// a word means nothing: "FZYOUH_508R" is not regular. +var weightLetters = map[string]Weight{ + "b": WeightBold, + "sb": WeightSemibold, + "db": WeightSemibold, + "m": WeightMedium, + "r": WeightRegular, +} + +// parseFamily reads what a font's own name admits about its weight and slope. +// +// The subset prefix is dropped for matching only. Tokens are read right to left +// because the weight is the last thing in every name measured here, and the +// first match wins so that "MiSansLatin-Demibold" is semibold rather than +// stopping at the family. +func parseFamily(family string) (w Weight, oblique bool) { + name := family + if plus := strings.IndexByte(name, '+'); plus >= 0 { + name = name[plus+1:] + } + tokens := strings.FieldsFunc(name, func(r rune) bool { + return r == '-' || r == '_' || r == ' ' + }) + + for i := len(tokens) - 1; i >= 0; i-- { + tok := strings.ToLower(tokens[i]) + + for _, tail := range obliqueTails { + if strings.HasSuffix(tok, tail) { + oblique = true + tok = strings.TrimSuffix(tok, tail) + break + } + } + if tok == "" { + continue + } + if w != WeightUnknown { + continue + } + if letter, ok := weightLetters[tok]; ok { + w = letter + continue + } + for _, wt := range weightTails { + if strings.HasSuffix(tok, wt.tail) { + w = wt.weight + break + } + } + } + return w, oblique +} + +// ExtractRuns reads every page's positioned text with pdftohtml. +// +// It never mutates the file and calls nothing remote, so like [ExtractText] it is +// a pure function of the bytes and safe to re-run — which is what lets the probe +// job be idempotent. +// +// pdftohtml is optional at runtime. A caller that cannot get runs must still be +// able to probe a document, so the error from a missing tool is returned plainly +// for the caller to degrade on rather than treated as a failed document. +func ExtractRuns(ctx context.Context, path string) ([]PageRuns, error) { + bin, err := extern.Require(extern.PDFToHTML) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, extractTimeout) + defer cancel() + + // -xml for the coordinate form, -i to skip images (nothing here reads them, + // and writing them would put files beside the blob store), -stdout to keep + // the output in memory. Deliberately no -hidden: the flags are exactly the + // ones the fixture's run counts and the detector's thresholds were measured + // with, and -hidden would add text those measurements never saw. + // #nosec G204 -- see ProbeInfo: bin comes from extern's own tool table and + // path is a blob-store path derived from a validated SHA-256 digest. + cmd := exec.CommandContext(ctx, bin, "-xml", "-i", "-enc", "UTF-8", "-stdout", path) + out := &limitedBuffer{limit: maxExtractedBytes} + var errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = out, &errOut + if err := cmd.Run(); err != nil { + if errors.Is(err, errOutputTooLarge) { + return nil, fmt.Errorf("%w (limit %d bytes)", errOutputTooLarge, maxExtractedBytes) + } + return nil, fmt.Errorf("doc: pdftohtml failed: %w: %s", + err, redact(strings.TrimSpace(errOut.String()), path)) + } + + pages, err := parsePDFXML(out.buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("doc: reading pdftohtml output for %s: %w", redact(path, path), err) + } + return pages, nil +} + +// parsePDFXML reads poppler's pdf2xml form into pages of runs. +func parsePDFXML(data []byte) ([]PageRuns, error) { + // Unmarshalling rather than a hand-rolled scan, because the payload is real + // XML: the measured manual carries 57 escaped entities in body text, and a + // regex over the raw bytes reports "GmbH & Co. KG" as its own text. + // + // The DOCTYPE poppler emits names an external DTD. Go's decoder never + // resolves one, so it is inert here — worth knowing rather than worth + // stripping. + var doc pdfXML + if err := xml.Unmarshal(data, &doc); err != nil { + return nil, err + } + + // One font table for the whole document, filled as the pages are walked. + // + // This is the opposite of what the format's per-page elements + // suggest, and it is measured rather than assumed. Poppler 26.07.0 allocates + // ids once per document and declares each one on the page that first uses it: + // the columns manual declares 83 ids, 0-82 contiguous, spread over 29 of its + // 68 pages, and the sequential manual 309 ids, 0-308, over 167 of its 560 — + // with not one id redeclared anywhere in either document. Every later page + // refers back to ids declared before it, so a table scoped to a single page + // resolves nothing for most of the document: it leaves 6,201 of the columns + // manual's 7,493 runs and 32,858 of the sequential manual's 34,413 runs with + // no font at all, 83% and 95%. Carried forward, both resolve completely. + // + // A page that does redeclare an id still wins for its own runs and every page + // after it, because its declaration overwrites the entry before its text is + // read — measured, no page in either document does, but that is the ordering + // the format implies and it costs nothing to honour. Poppler emits a page's + // fontspecs before any of its text, checked on both documents, so reading them + // first here is faithful to the stream rather than a reordering of it. + fonts := make(map[int]Font, 16) + + pages := make([]PageRuns, 0, len(doc.Pages)) + for i := range doc.Pages { + p := &doc.Pages[i] + for j := range p.Fonts { + spec := &p.Fonts[j] + weight, oblique := parseFamily(spec.Family) + fonts[spec.ID] = Font{ + Size: spec.Size, + Family: spec.Family, + Weight: weight, + Oblique: oblique, + } + } + if p.Number < 1 { + // Page numbers reach the language map and the database, where a wrong + // one mislabels a whole section. A tool that stops emitting them should + // say so here rather than have positions guessed for it. + return nil, fmt.Errorf("page %d of %d carries no page number", i+1, len(doc.Pages)) + } + out := PageRuns{ + No: p.Number, + Width: p.Width, + Height: p.Height, + Runs: make([]TextRun, 0, len(p.Texts)), + } + for j := range p.Texts { + t := &p.Texts[j] + // An unresolvable id leaves the size and family zero rather than + // failing the document: a font is an enrichment, and a page of text + // with no fontspec still has to reach the language map. The markup + // verdict is attached either way — it is poppler's, not the table's. + font := fonts[t.Font] + font.MarkedBold, font.MarkedItalic = t.MarkedBold, t.MarkedItalic + out.Runs = append(out.Runs, TextRun{ + X: t.Left, Y: t.Top, Width: t.Width, Height: t.Height, Text: t.Text, + Font: font, + }) + } + pages = append(pages, out) + } + return pages, nil +} + +// pdfXML mirrors the part of poppler's pdf2xml output this code reads. Images +// and the producer are ignored; the geometry, the characters and the font are +// what the pipeline reads. +type pdfXML struct { + Pages []xmlPage `xml:"page"` +} + +type xmlPage struct { + Number int `xml:"number,attr"` + Width float64 `xml:"width,attr"` + Height float64 `xml:"height,attr"` + Fonts []xmlFontSpec `xml:"fontspec"` + Texts []xmlText `xml:"text"` +} + +// xmlFontSpec is one declaration: the table a run's font attribute +// indexes into. See parsePDFXML for the measured scope of the id. +// +// The color attribute is deliberately not read. It would separate white label +// text over a diagram from body copy, which is a real distinction, but nothing +// needs it yet and an unused field invites a caller to trust it untested. +type xmlFontSpec struct { + ID int `xml:"id,attr"` + Size float64 `xml:"size,attr"` + Family string `xml:"family,attr"` +} + +// xmlText is one element: a positioned run of characters. +// +// Coordinates are read as floats although poppler currently prints integers, +// since nothing in the format promises that and a truncated coordinate would +// move a column boundary. +type xmlText struct { + Top float64 + Left float64 + Width float64 + Height float64 + Text string + // Font is the fontspec id from the font attribute, or -1 where the attribute + // is absent. Absent must not read as 0, because poppler numbers fontspec ids + // from 0 and that is a real font. Measured: every one of the columns manual's + // 7,493 runs and the sequential manual's 34,413 carries the attribute. + Font int + // MarkedBold and MarkedItalic record that the whole run was wrapped in + // or . See UnmarshalXML for what "whole" is doing there. + MarkedBold bool + MarkedItalic bool +} + +// UnmarshalXML collects the element's characters including those inside child +// elements. +// +// This is the whole reason for a custom unmarshaller, and it is not cosmetic. +// Go's `,chardata` skips the content of child elements, and poppler wraps a +// styled run's text in or — measured on the column fixture, 355 runs are +// wrapped and every one of them is wrapped whole, so `,chardata` returns nothing +// at all for them. Among those 355 are the printed language tabs D, PL and UA. +// +// The effect was measured both ways over that document's 169 columns rather than +// argued from the tabs' importance, and the first guess was wrong: +// +// naive correct +// columns named 166 167 +// named by printed tag 0 53 +// named by its alphabet 166 114 +// tag/alphabet conflicts 0 1 +// +// So the count barely moves. What collapses is attribution: 53 columns stop being +// named by the document's own printed tab and are named by their letters instead, +// and the one place where the two disagree stops being detectable. The count +// survives only because this manual's five languages have distinguishable +// alphabets — and a manual whose languages share one is precisely the case the +// printed tag outranks every other signal for, where nothing would be left. +// +// `,innerxml` fails the other way, keeping the tags and the raw entities. +// +// The same walk now also notes which element did the wrapping, since that is +// poppler's own verdict on the run's weight and slope — see [Weight] for why it +// is kept alongside the family name rather than instead of it. Nesting is real: +// 5 runs of the columns manual are ..., so both are recorded, at +// any depth, rather than only the outermost. +// +// A style is recorded only when the wrapper encloses the entire run. That is the +// measured shape — all 355 styled runs of the columns manual and all 4,336 of the +// sequential manual are wrapped whole, none partially — and it keeps the mixed +// case honest: a run reading `plain word plain` is not an italic run, and +// calling it one would label a whole line by one word of it. +func (t *xmlText) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + t.Font = -1 + for _, attr := range start.Attr { + if attr.Name.Local == "font" { + if _, err := fmt.Sscanf(attr.Value, "%d", &t.Font); err != nil { + return fmt.Errorf("text font=%q: %w", attr.Value, err) + } + continue + } + target := map[string]*float64{ + "top": &t.Top, + "left": &t.Left, + "width": &t.Width, + "height": &t.Height, + }[attr.Name.Local] + if target == nil { + continue + } + if _, err := fmt.Sscanf(attr.Value, "%g", target); err != nil { + return fmt.Errorf("text %s=%q: %w", attr.Name.Local, attr.Value, err) + } + } + + var text, outside strings.Builder + var bold, italic bool + depth := 0 + for { + tok, err := d.Token() + if err != nil { + // A truncated element is a broken document, not an empty one: io.EOF + // here means the tool's output was cut off mid-run. + if errors.Is(err, io.EOF) { + return fmt.Errorf("text element is unterminated") + } + return err + } + switch v := tok.(type) { + case xml.CharData: + text.Write(v) + if depth == 0 { + // Characters outside every wrapper mean the run is only partly + // styled, so no wrapper speaks for the whole of it. + outside.Write(v) + } + case xml.StartElement: + switch v.Name.Local { + case "b": + bold = true + case "i": + italic = true + } + depth++ + case xml.EndElement: + if depth == 0 { + t.Text = text.String() + if strings.TrimSpace(outside.String()) == "" { + t.MarkedBold, t.MarkedItalic = bold, italic + } + return nil + } + depth-- + } + } +} diff --git a/internal/doc/runs_internal_test.go b/internal/doc/runs_internal_test.go new file mode 100644 index 0000000..21d224d --- /dev/null +++ b/internal/doc/runs_internal_test.go @@ -0,0 +1,302 @@ +package doc + +import ( + "strings" + "testing" +) + +// Unit tests for the pdf2xml reader, against the shapes actually measured in +// poppler's output. No poppler and no PDF: runs_test.go drives the real tool. + +// realShapeXML reproduces every form the column fixture's 7,493 runs take, +// with neutral text. Each one is there because it was measured, and the counts +// are quoted where they matter. +// The fontspec ids and families are the real ones: 5 is the body face, 11 the +// face poppler marks bold although its name does not say so, 73 a face whose +// name does, 75 an oblique, 9 the heaviest in the document, and 55 the Medium +// that poppler never marks at all. See Weight in runs.go for the counts each of +// those cases was measured at. +const realShapeXML = ` + + + + + + + + + + + D + Ordinary body text. + Widgets & Sprockets GmbH + Parked above the page. + mixed styling inside + rotated + + Wholly slanted + Nested emphasis + Emphasis by weight alone + + + + +` + +func TestParsePDFXMLReadsTheShapesPopplerEmits(t *testing.T) { + pages, err := parsePDFXML([]byte(realShapeXML)) + if err != nil { + t.Fatalf("parsePDFXML: %v", err) + } + if len(pages) != 2 { + t.Fatalf("got %d pages, want 2", len(pages)) + } + + p := &pages[0] + if p.No != 1 { + t.Errorf("page number = %d, want 1", p.No) + } + if p.Width != 892 || p.Height != 850 { + t.Errorf("page box = %gx%g, want 892x850", p.Width, p.Height) + } + if len(p.Runs) != 10 { + t.Fatalf("got %d runs, want 10 — every element is a run, including "+ + "the empty and the off-page ones, which the detector counts and reports", + len(p.Runs)) + } + + // The styled-tag case is the one that matters most. 355 of the fixture's runs + // wrap their text in or , every one of them whole, and the printed + // language tabs D, PL and UA are among them. Go's `,chardata` returns "" for + // exactly these; runs.go records what that measurably costs on the real + // document, and columns_fixture_test.go pins the split it costs it in. + if got := p.Runs[0].Text; got != "D" { + t.Errorf("run wrapped in read as %q, want %q — the printed language "+ + "tabs are styled runs, and losing them loses the page-tag signal", got, "D") + } + if got := p.Runs[2].Text; got != "Widgets & Sprockets GmbH" { + t.Errorf("entity read as %q, want the unescaped form", got) + } + if got := p.Runs[4].Text; got != "mixed styling inside" { + t.Errorf("run with interior markup read as %q, want %q", got, "mixed styling inside") + } + + // Coordinates are passed through as poppler gives them, negatives included. + // 218 of the fixture's 769 runs on one page are parked above the top edge, and + // dropping them here would hide the fact from DetectColumns, which counts them + // as off-page and reports the count — that is what stops them merging two of + // that page's three columns. + if got := p.Runs[3].Y; got != -38 { + t.Errorf("off-page run y = %g, want -38 kept as measured", got) + } + // Poppler reports rotated text with width 0. The detector drops it and says so; + // the reader must not silently discard it first. + if got := p.Runs[5].Width; got != 0 { + t.Errorf("rotated run width = %g, want 0 kept as measured", got) + } + + if pages[1].HasText() { + t.Error("a page with no elements should report no text") + } + if pages[1].Width != 892 { + t.Errorf("a page with no text still has a box; got width %g", pages[1].Width) + } +} + +// TestParsePDFXMLReadsTheFontOfEachRun checks the font a run resolves to against +// the six real fontspec shapes, and specifically the cases where poppler's / +// markup and the family name disagree — which is why both are carried. +func TestParsePDFXMLReadsTheFontOfEachRun(t *testing.T) { + pages, err := parsePDFXML([]byte(realShapeXML)) + if err != nil { + t.Fatalf("parsePDFXML: %v", err) + } + runs := pages[0].Runs + + for _, tc := range []struct { + run int + why string + want Font + }{ + {1, "the body face: light by name, marked nothing", Font{ + Size: 14, Family: "HUTKLI+FuturaCon-Lig", Weight: WeightLight}}, + {0, "poppler marks the printed tab bold; FuturaBQ's name does not say so, " + + "and 78 of the fixture's 78 FuturaBQ runs are this case", Font{ + Size: 15, Family: "HUTKLI+FuturaBQ", Weight: WeightUnknown, MarkedBold: true}}, + {2, "a name that does say bold — Bol, one letter from Boo, which is Book", Font{ + Size: 18, Family: "GMJEXK+FuturaCon-Bol", Weight: WeightBold}}, + {7, "wholly wrapped in , and BooObl is Book plus oblique glued together", Font{ + Size: 12, Family: "HUTKLI+Futura-BooObl", Weight: WeightRegular, + Oblique: true, MarkedItalic: true}}, + {8, " nests on 5 runs of the fixture, so both must be recorded", Font{ + Size: 19, Family: "HUTKLI+Function-Xbold", Weight: WeightHeavy, + MarkedBold: true, MarkedItalic: true}}, + {9, "the case that makes the markup insufficient on its own: Medium is " + + "17.2% of the columns manual's characters at its body size, and poppler " + + "marks not one of its 1,241 runs", Font{ + Size: 14, Family: "HUTKLI+FuturaCon-Med", Weight: WeightMedium}}, + } { + if got := runs[tc.run].Font; got != tc.want { + t.Errorf("run %d font = %+v, want %+v — %s", tc.run, got, tc.want, tc.why) + } + } + + // A run only partly wrapped is not a styled run. Labelling the whole of + // `mixed styling inside` italic would name a line after one word of it. + if f := runs[4].Font; f.MarkedItalic || f.MarkedBold { + t.Errorf("run with interior markup reported as styled: %+v — the wrapper "+ + "speaks for the whole run or for none of it", f) + } + // And the text still has to survive, which is the older measurement this + // walk exists for. + if got := runs[4].Text; got != "mixed styling inside" { + t.Errorf("noticing the markup cost the text: %q", got) + } +} + +// fontScopeXML gives page 2 a run whose font was declared on page 1, and page 3 a +// redeclaration of that same id. +const fontScopeXML = ` + + + Declared here. + + + Declared on page 1. + + + + Redeclared here. + + + Never declared. + No font attribute. + + +` + +// TestFontSpecCarriesForwardAcrossPages is the trap, and it is the opposite of +// the one the format's per-page elements suggest. +// +// Poppler allocates ids once per document and declares each on the page that +// first uses it, so most pages declare none: 29 of the columns manual's 68 pages +// and 167 of the sequential manual's 560. A table scoped to one page therefore +// leaves 83% and 95% of their runs with no font, silently — every one of them +// still has coordinates and text, so nothing else fails. Reverted to a per-page +// table, this test reports the size as 0 on page 2. +func TestFontSpecCarriesForwardAcrossPages(t *testing.T) { + pages, err := parsePDFXML([]byte(fontScopeXML)) + if err != nil { + t.Fatalf("parsePDFXML: %v", err) + } + if len(pages) != 4 { + t.Fatalf("got %d pages, want 4", len(pages)) + } + + if got := pages[1].Runs[0].Font; got.Size != 11 || got.Family != "RJYQHP+MiSansLatin" { + t.Errorf("page 2 run font = %+v, want the id 0 declared on page 1 (11pt "+ + "MiSansLatin) — a page-scoped table resolves nothing on the pages that "+ + "declare no fontspec, which is most of a real manual", got) + } + + // A page that does redeclare an id wins for its own runs. No page of either + // fixture does, but that is what a per-page declaration means, and the cost + // of honouring it is that the table is written before the page's text is read. + if got := pages[2].Runs[0].Font; got.Size != 21 || got.Weight != WeightSemibold { + t.Errorf("page 3 run font = %+v, want its own redeclaration of id 0 "+ + "(21pt, semibold), not the one inherited from page 1", got) + } + + // An id that was never declared leaves the font empty rather than failing the + // document: a font is an enrichment, and the page still owes the language map + // its text. + if got := pages[3].Runs[0]; got.Font != (Font{}) || got.Text != "Never declared." { + t.Errorf("run with an unknown font id = %+v, want empty font and intact text", got) + } + + // And an absent font attribute must not read as id 0, which is a real font: + // poppler numbers fontspec ids from 0, so the zero value of the parsed field + // cannot double as "no font". Every run of both fixtures does carry the + // attribute, which is exactly why this would go unnoticed. + if got := pages[3].Runs[1].Font; got != (Font{}) { + t.Errorf("run with no font attribute resolved to %+v, want no font — id 0 "+ + "is the first font poppler declares, not a marker for its absence", got) + } +} + +func TestParseFamilyReadsTheWeightAFontNameAdmits(t *testing.T) { + // Every input is a family measured in one of the two fixtures. + for _, tc := range []struct { + family string + weight Weight + oblique bool + }{ + {"HUTKLI+FuturaCon-Lig", WeightLight, false}, + {"MyriadPro-Light", WeightLight, false}, + {"AlibabaSans-Light", WeightLight, false}, + {"HUTKLI+Futura-Boo", WeightRegular, false}, + {"HUTKLI+FuturaPT-Book", WeightRegular, false}, + {"ZILBKZ+MiSans-Normal", WeightRegular, false}, + {"NQXXDX+Alibaba-PuHuiTi-R", WeightRegular, false}, + {"HUTKLI+FuturaCon-Med", WeightMedium, false}, + {"GGMMEH+MiSans-Medium", WeightMedium, false}, + {"PMJWFT+Alibaba-PuHuiTi-M", WeightMedium, false}, + {"AlibabaPuHuiTi_2_65_Medium", WeightMedium, false}, + {"SJEGLN+MiSansLatin-Demibold", WeightSemibold, false}, + {"Arimo-SemiBold", WeightSemibold, false}, + {"GMJEXK+FuturaCon-Bol", WeightBold, false}, + {"AAACTX+HarmonyOS_Sans_SC_Bold", WeightBold, false}, + {"RanyBold", WeightBold, false}, + {"Alibaba-PuHuiTi-B", WeightBold, false}, + {"HUTKLI+Function-Xbold", WeightHeavy, false}, + + // Slope, including the three names that glue it to the weight. + {"HUTKLI+Futura-BooObl", WeightRegular, true}, + {"GMJEXK+FuturaCon-BolObl", WeightBold, true}, + {"GMJEXK+FuturaCon-MedObl", WeightMedium, true}, + + // Names that admit nothing. Claiming regular for these would be the + // heuristic overreaching: poppler marks FuturaBQ, FuturaStd and Calibri + // bold, and 498 of 16,426 runs of plain "MiSans", so "the name is silent" + // and "the font is regular" are different facts. + {"HUTKLI+FuturaBQ", WeightUnknown, false}, + {"HUTKLI+FuturaStd", WeightUnknown, false}, + {"BZPIHV+Calibri", WeightUnknown, false}, + {"WVCUZF+MiSans", WeightUnknown, false}, + {"VPWOLH+Arimo", WeightUnknown, false}, + {"QIGZJR+MiSansVF", WeightUnknown, false}, + {"HUTKLI+Helvetica", WeightUnknown, false}, + {"GMJEXK+ZapfDingbatsITC", WeightUnknown, false}, + {"HUTKLI+EuropeanPiStd-1", WeightUnknown, false}, + // A trailing letter inside a longer token is not a weight code. This one + // is a real family from the sequential manual, and reading its "508R" as + // regular is what restricting single letters to whole tokens prevents. + {"FZYOUH_508R--GB1-4", WeightUnknown, false}, + {"", WeightUnknown, false}, + } { + w, obl := parseFamily(tc.family) + if w != tc.weight || obl != tc.oblique { + t.Errorf("parseFamily(%q) = %s/oblique=%t, want %s/oblique=%t", + tc.family, w, obl, tc.weight, tc.oblique) + } + } +} + +func TestParsePDFXMLRejectsAPageWithNoNumber(t *testing.T) { + // A page number reaches the language map and the database, where a wrong one + // mislabels a whole section. Guessing from position would be silent. + const noNumber = ` + x` + + if _, err := parsePDFXML([]byte(noNumber)); err == nil { + t.Fatal("a page with no number was accepted") + } else if !strings.Contains(err.Error(), "page number") { + t.Errorf("error does not say what was missing: %v", err) + } +} + +func TestParsePDFXMLRejectsMalformedOutput(t *testing.T) { + if _, err := parsePDFXML([]byte(` verdict, +// and the family-name weight is unit-tested against the real fixture families in +// runs_internal_test.go instead. +func TestExtractRunsReadsTheFontOfEachRun(t *testing.T) { + requirePDFToHTML(t) + + const headingText = "A heading in the heavier face" + const bodyText = "Ordinary body text on this page." + pages := extract(t, testpdf.Doc{Pages: []testpdf.Page{{ + Headings: []string{headingText}, + Lines: []string{bodyText}, + }}}) + + byText := make(map[string]doc.Font, 2) + for i := range pages[0].Runs { + r := &pages[0].Runs[i] + byText[strings.TrimSpace(r.Text)] = r.Font + } + heading, ok := byText[headingText] + if !ok { + t.Fatalf("the heading did not come back as a run of its own; got %q", runTexts(pages[0].Runs)) + } + body := byText[bodyText] + + // Exact, because these are the sizes written scaled by the same 1.5 the page + // box is scaled by — 17pt and 11pt. That the size shares the coordinate space + // rather than being the PDF's point size is a property of the format, and a + // caller comparing a size against a run height depends on it. + if heading.Size != 26 || body.Size != 17 { + t.Errorf("heading size %g and body size %g, want 26 and 17 — poppler scales "+ + "the fontspec size by 1.5 exactly as it scales the coordinates", + heading.Size, body.Size) + } + if heading.Family == "" || body.Family == "" { + t.Errorf("family missing: heading %q, body %q", heading.Family, body.Family) + } + + // The weight, which is the whole reason the font is read at all. Poppler names + // both faces "Helvetica", so its own verdict is the only signal here — and that + // is exactly the case measured on the real manual as FuturaBQ, 78 runs of 78 + // marked bold with nothing in the name to say so. + if !heading.MarkedBold { + t.Errorf("the heading was not marked bold: %+v — without a weight, a larger "+ + "size cannot tell a heading from the larger regular face that real "+ + "manuals set safety text in", heading) + } + if body.MarkedBold { + t.Errorf("body text was marked bold: %+v", body) + } + if heading.MarkedItalic || body.MarkedItalic { + t.Error("nothing on this page is italic") + } +} + +func TestExtractRunsRejectsAMissingFile(t *testing.T) { + requirePDFToHTML(t) + + _, err := doc.ExtractRuns(context.Background(), filepath.Join(t.TempDir(), "absent.pdf")) + if err == nil { + t.Fatal("extracting a file that does not exist succeeded") + } + // The message reaches documents.last_error and the log, so it must not carry + // the directory: a blob path sits under the data directory and so under a home + // directory, which names the operating-system user. See privacy.md threat 2. + if strings.Contains(err.Error(), os.TempDir()) { + t.Errorf("error leaks the containing directory: %v", err) + } +} + +func runTexts(runs []doc.TextRun) []string { + out := make([]string, 0, len(runs)) + for i := range runs { + out = append(out, runs[i].Text) + } + return out +} diff --git a/internal/doc/script.go b/internal/doc/script.go new file mode 100644 index 0000000..d37941f --- /dev/null +++ b/internal/doc/script.go @@ -0,0 +1,227 @@ +package doc + +import ( + "sort" + "unicode" +) + +// Script names reported by [DominantScript]. These are Unicode script names +// except for Kana, which is a deliberate composite — see below. +const ( + ScriptLatin = "Latin" + ScriptCyrillic = "Cyrillic" + ScriptGreek = "Greek" + ScriptHebrew = "Hebrew" + ScriptArabic = "Arabic" + ScriptThai = "Thai" + ScriptHan = "Han" + ScriptKana = "Kana" + ScriptHangul = "Hangul" + ScriptDevanagari = "Devanagari" + ScriptArmenian = "Armenian" + ScriptGeorgian = "Georgian" +) + +// scriptTables maps a reported script name to its Unicode range table. +// +// Hiragana and Katakana are counted together as Kana rather than separately, +// because the distinction between them is orthographic rather than linguistic: +// both mean Japanese. +var scriptTables = []struct { + name string + table *unicode.RangeTable +}{ + {ScriptLatin, unicode.Latin}, + {ScriptCyrillic, unicode.Cyrillic}, + {ScriptGreek, unicode.Greek}, + {ScriptHebrew, unicode.Hebrew}, + {ScriptArabic, unicode.Arabic}, + {ScriptThai, unicode.Thai}, + {ScriptHan, unicode.Han}, + {ScriptKana, unicode.Hiragana}, + {ScriptKana, unicode.Katakana}, + {ScriptHangul, unicode.Hangul}, + {ScriptDevanagari, unicode.Devanagari}, + {ScriptArmenian, unicode.Armenian}, + {ScriptGeorgian, unicode.Georgian}, +} + +// kanaEvidenceRunes is how many kana are enough to call a Han-dominant page +// Japanese. A handful of stray kana in a Chinese document is possible; a page of +// Japanese prose always carries many, because grammatical particles are kana. +const kanaEvidenceRunes = 5 + +// ScriptCounts returns the number of letters per script in s. +func ScriptCounts(s string) map[string]int { + counts := make(map[string]int, 4) + for _, r := range s { + if !unicode.IsLetter(r) { + continue + } + for _, st := range scriptTables { + if unicode.Is(st.table, r) { + counts[st.name]++ + break + } + } + } + return counts +} + +// DominantScript reports which script a page is written in, or "" when there are +// no letters to judge. +// +// This is the cheapest language signal there is: it needs no models, no network +// and no dependency, and it settles the non-Latin scripts outright. Measured on a +// 34-language manual it resolved 27% of pages to a single language and narrowed +// the Cyrillic pages to three candidates. What it cannot do is separate the 25 +// languages that share the Latin alphabet, which is the residue that needs a +// statistical detector. See docs/design/language-detection.md. +func DominantScript(s string) string { + counts := ScriptCounts(s) + if len(counts) == 0 { + return "" + } + + // Japanese mixes kanji with kana, and kanji usually outnumber kana on a page, + // so a plain maximum would report Han and lose the distinction from Chinese. + // Any substantial kana presence is decisive. + if counts[ScriptKana] >= kanaEvidenceRunes { + return ScriptKana + } + + best, bestCount := "", 0 + for name, n := range counts { + // Ties resolve by name so the result is deterministic; a tie between two + // scripts on one page is a mixed page and either answer is arbitrary. + if n > bestCount || (n == bestCount && name < best) { + best, bestCount = name, n + } + } + return best +} + +// ScriptLanguages maps a script to the language subtags that use it, most common +// first. It is used to narrow candidates and to sanity-check a printed page tag: +// a page tagged EL that is not Greek script is not really Greek. +// +// The Latin entry is deliberately empty. Listing the dozens of languages that +// use the Latin alphabet would imply a discrimination this signal cannot make, +// and callers must treat an empty result as "this script tells you nothing". +var scriptLanguages = map[string][]string{ + ScriptGreek: {"el"}, + ScriptHebrew: {"he"}, + ScriptArabic: {"ar", "fa", "ur"}, + ScriptThai: {"th"}, + ScriptHan: {"zh"}, + ScriptKana: {"ja"}, + ScriptHangul: {"ko"}, + ScriptDevanagari: {"hi", "mr", "ne"}, + ScriptArmenian: {"hy"}, + ScriptGeorgian: {"ka"}, + ScriptCyrillic: {"ru", "uk", "bg", "sr", "kk", "mk", "be"}, + ScriptLatin: {}, +} + +// ScriptLanguages returns the language subtags that use a script. +func ScriptLanguages(script string) []string { + langs := scriptLanguages[script] + out := make([]string, len(langs)) + copy(out, langs) + return out +} + +// ScriptAllows reports whether a language subtag is plausible for a script. +// +// A Latin-script page allows any language, because the signal cannot narrow it; +// saying otherwise would turn "no information" into a false rejection. An unknown +// script also allows anything, for the same reason. +func ScriptAllows(script, lang string) bool { + if script == "" || script == ScriptLatin { + return true + } + langs, known := scriptLanguages[script] + if !known || len(langs) == 0 { + return true + } + for _, l := range langs { + if l == lang { + return true + } + } + return false +} + +// languageScripts records the scripts a language is actually written in, for the +// languages that are not written in the Latin alphabet. +// +// This is the converse of scriptLanguages and it catches a different error. A +// Latin-script page permits any language as far as [ScriptAllows] is concerned, +// which is correct — Latin cannot narrow a language. But it is still absurd for a +// page of English prose to be labelled Japanese, and that is exactly what happened +// on the measured fixture: the printed index's Japanese entry, being last, claimed +// every page to the end of the document, absorbing an English back cover. +// +// Serbian deliberately lists both Cyrillic and Latin: it is genuinely written in +// both, and the measured fixture uses Latin. +var languageScripts = map[string][]string{ + "ja": {ScriptKana, ScriptHan}, + "zh": {ScriptHan, ScriptKana}, + "ko": {ScriptHangul, ScriptHan}, + "ru": {ScriptCyrillic}, "uk": {ScriptCyrillic}, "bg": {ScriptCyrillic}, + "be": {ScriptCyrillic}, "mk": {ScriptCyrillic}, "kk": {ScriptCyrillic}, + "sr": {ScriptCyrillic, ScriptLatin}, + "el": {ScriptGreek}, + "he": {ScriptHebrew}, + "ar": {ScriptArabic}, "fa": {ScriptArabic}, "ur": {ScriptArabic}, + "th": {ScriptThai}, + "hy": {ScriptArmenian}, "ka": {ScriptGeorgian}, + "hi": {ScriptDevanagari}, "mr": {ScriptDevanagari}, "ne": {ScriptDevanagari}, +} + +// LanguageAllowsScript reports whether a language can be written in a script. +// Languages absent from the table use the Latin alphabet and are unconstrained. +func LanguageAllowsScript(lang, script string) bool { + if script == "" || lang == "" { + return true + } + scripts, constrained := languageScripts[lang] + if !constrained { + return true + } + for _, s := range scripts { + if s == script { + return true + } + } + return false +} + +// ScriptCompatible reports whether a script and a language can coexist on a page, +// checked in both directions: the script must permit the language, and the +// language must be written in that script. Either check alone lets an obvious +// nonsense through. +func ScriptCompatible(script, lang string) bool { + base := BaseLanguage(lang) + if base == "" { + base = lang + } + return ScriptAllows(script, base) && LanguageAllowsScript(base, script) +} + +// SortedScripts returns the scripts present in s, most letters first. Used for +// reporting rather than for decisions. +func SortedScripts(s string) []string { + counts := ScriptCounts(s) + names := make([]string, 0, len(counts)) + for name := range counts { + names = append(names, name) + } + sort.Slice(names, func(i, j int) bool { + if counts[names[i]] != counts[names[j]] { + return counts[names[i]] > counts[names[j]] + } + return names[i] < names[j] + }) + return names +} diff --git a/internal/doc/signals.go b/internal/doc/signals.go new file mode 100644 index 0000000..e41e6dc --- /dev/null +++ b/internal/doc/signals.go @@ -0,0 +1,503 @@ +package doc + +import ( + "fmt" + "sort" + "strconv" + "strings" + "unicode" +) + +// minTagRunPages is how many consecutive pages must carry the same printed tag +// before it is believed. +// +// This guard is not defensive programming, it is a measured necessity. A manual's +// contents pages list every language code in the same corner the per-page tab +// occupies, so they produce spurious single-page runs — on the measured fixture, +// three of them (EN, MS and RO on pages 2, 3 and 4). Requiring two consecutive +// pages removed all three false positives and left exactly the 34 real sections. +const minTagRunPages = 2 + +// tagScriptAgreementFraction is the share of a tag run's pages whose script must +// be compatible with the tagged language. Below this the tag is disbelieved: a +// run tagged EL whose pages are not Greek is not Greek, whatever the tab says. +const tagScriptAgreementFraction = 0.5 + +// TagRuns builds language runs from the code each page prints on itself. +// +// This is the cheapest accurate signal available, and on a manual that prints +// tags it is also the most accurate: measured at 553 of 553 content pages, it +// labelled correctly two sections that statistical detection cannot — Uzbek, +// which lingua-go does not support at all, and Latin-script Serbian, which reads +// as Croatian. See docs/design/language-detection.md. +// +// Not every manual prints tags. An empty result is a normal outcome, not a +// failure. +// EffectiveTags decides each page's printed language tag, given the set of codes +// the document's own contents table declares. +// +// The conservative reading — a code among the first lines of the page — is +// trusted outright. Beyond that, a candidate found anywhere on the page is +// accepted only if the printed index also lists that code, which is what makes +// searching the whole page safe: NO, IT, IS and BE are valid language codes and +// ordinary words, but a manual that does not contain a Norwegian section does not +// list NO in its contents. +// +// This exists because right-to-left pages put their tab well after the heading in +// reading order. Without it the Hebrew and Arabic sections of the measured +// fixture went partly unlabelled, and the gaps were filled by an erroneous claim +// from the index. +// +// It returns a copy of the tags rather than mutating pages, so the raw extraction +// stays separable from the interpretation of it. +func EffectiveTags(pages []Page, knownCodes map[string]bool) []string { + tags := make([]string, len(pages)) + for i := range pages { + p := &pages[i] + // A contents table is not a page of any language section, whatever code it + // happens to print first. The run-length guard alone does not catch this: + // it only helps when the contents page is separated from the section it + // lists, and a contents page sitting immediately before the first section + // is otherwise absorbed straight into it. + if IsContentsPage(p) { + continue + } + if p.Tag != "" { + tags[i] = p.Tag + continue + } + for _, c := range p.TagCandidates { + // The vocabulary check is what makes a single letter usable at all: + // "D" is German on a manual whose contents table lists D, and a list + // marker everywhere else. + if knownCodes[c] { + tags[i] = c + break + } + } + } + return tags +} + +// IsContentsPage reports whether a page is a printed contents table rather than a +// page of content. +// +// The test is structural: several code/title/page triples in reading order. Three +// is enough to tell a real index from a page that merely mentions a language code +// once. +func IsContentsPage(p *Page) bool { + return len(parseIndexPage(p.Text)) >= minIndexEntriesPerPage +} + +// IndexCodes returns the set of language codes the printed index declares. It is +// the vocabulary against which looser tag candidates are checked. +func IndexCodes(indexRuns []Run) map[string]bool { + codes := make(map[string]bool, len(indexRuns)) + for _, r := range indexRuns { + codes[strings.ToUpper(r.Code)] = true + } + return codes +} + +// TagRuns builds language runs from per-page printed tags. tags is parallel to +// pages and normally comes from [EffectiveTags]. +func TagRuns(pages []Page, tags []string) []Run { + // Work on a copy carrying the effective tags, so grouping sees the narrowed + // interpretation rather than the raw first-lines reading. + if len(tags) == len(pages) { + tagged := make([]Page, len(pages)) + copy(tagged, pages) + for i := range tagged { + tagged[i].Tag = tags[i] + } + pages = tagged + } + + var runs []Run + for _, group := range groupBy(pages, func(p *Page) string { return p.Tag }) { + if group.key == "" || group.pages() < minTagRunPages { + continue + } + lang, ok := NormalizeCode(group.key) + if !ok { + // A tag that is not a language code at all: keep the raw code so the + // run is still reportable, but claim no language for it. + runs = append(runs, Run{ + Source: SourcePageTag, Code: group.key, Start: group.start, End: group.end, + Confidence: 0.3, + Note: fmt.Sprintf("printed tag %q is not a recognised language code", group.key), + }) + continue + } + + agree, judged := 0, 0 + for i := group.startIdx; i <= group.endIdx; i++ { + if pages[i].Script == "" { + continue + } + judged++ + // Agreement is checked in both directions, as reconciliation checks it. + // ScriptAllows alone treats a Latin page as corroborating anything, so two + // pages tagged JA whose CJK glyphs failed to extract — leaving only Latin + // furniture — scored confidence 1.0 "corroborated by script" while + // Reconcile discarded the run outright. The stored row then asserted + // maximum confidence for a section that ended up unlabelled. + if ScriptCompatible(pages[i].Script, lang) { + agree++ + } + } + + run := Run{ + Source: SourcePageTag, Code: group.key, Lang: lang, + Start: group.start, End: group.end, Confidence: 0.9, + Note: "language printed on every page of the run", + } + switch { + case judged == 0: + // No script evidence either way; the tag stands on its own. + case float64(agree)/float64(judged) < tagScriptAgreementFraction: + run.Confidence = 0.2 + run.Note = fmt.Sprintf("printed tag %s disagrees with the page script on %d of %d pages", + group.key, judged-agree, judged) + case agree == judged: + // Script corroborates the tag: the strongest evidence available here. + run.Confidence = 1.0 + run.Note = "language printed on every page, corroborated by script" + } + runs = append(runs, run) + } + return runs +} + +// ScriptRuns builds runs from Unicode script analysis. +// +// A script names a language only when just one language uses it in practice — +// Greek, Hebrew, Thai, Japanese and so on. For Latin, and for Cyrillic with its +// several candidates, the run carries the script as its code and no language, +// because claiming one would invent information this signal does not have. +func ScriptRuns(pages []Page) []Run { + var runs []Run + for _, group := range groupBy(pages, func(p *Page) string { return p.Script }) { + if group.key == "" { + continue + } + run := Run{ + Source: SourceScript, Code: group.key, + Start: group.start, End: group.end, Confidence: 0.2, + Note: fmt.Sprintf("%s script", group.key), + } + if candidates := ScriptLanguages(group.key); len(candidates) == 1 { + if lang, ok := NormalizeCode(candidates[0]); ok { + run.Lang = lang + // Code carries the language, not the script name. A reconciled run + // that a script signal won must still be identified by the language + // it names, or a Hebrew section ends up labelled "Hebrew" and no + // longer matches the "HE" the document itself uses. + run.Code = strings.ToUpper(lang) + run.Confidence = 0.7 + run.Note = fmt.Sprintf("%s script is used by only one language here", group.key) + } + } + runs = append(runs, run) + } + return runs +} + +// minIndexEntriesPerPage is how many code/title/page triples a page needs before +// it is treated as a contents page. Three is enough to distinguish a real index +// from a page that merely happens to mention a language code. +const minIndexEntriesPerPage = 3 + +// indexLookaheadLines bounds how far after a code line the parser will look for +// that entry's page number. Titles run to one or two lines in practice. +const indexLookaheadLines = 4 + +// IndexRuns parses the manual's own printed contents table. +// +// The index is the only signal that supplies section *titles*, and the only one +// that can name a language no detector supports. What it cannot be trusted about +// is page numbers: on the measured fixture, 10 of 34 sections claim a printed page +// 1-2 away from the folio actually printed, because two sections run 17 pages +// rather than 16. +// +// So a claimed page is resolved through the folios actually printed on the pages, +// not through a global offset. That converts the claim into a PDF page faithfully, +// including its error — which is the point. The claim stays a hypothesis for +// reconciliation to test, rather than being silently corrected or silently +// trusted. +func IndexRuns(pages []Page) []Run { + type entry struct { + code, title string + printed int + } + var entries []entry + seen := make(map[string]bool, 32) + isContentsPage := make(map[int]bool, 4) + + for i := range pages { + p := &pages[i] + if !IsContentsPage(p) { + continue + } + isContentsPage[p.No] = true + for _, e := range parseIndexPage(p.Text) { + if seen[e.code] { + continue + } + seen[e.code] = true + entries = append(entries, entry{e.code, e.title, e.printed}) + } + } + if len(entries) == 0 { + return nil + } + + // A contents page's own trailing number is an index entry's page reference, + // not a folio. Including it maps a claimed page onto the contents page itself: + // on the measured fixture, page 2 ends with "194", so Arabic's claimed start + // of 194 resolved to page 2 and produced a one-page Arabic section at the front + // of the document. Contents pages therefore contribute no folios. + folioToPage := make(map[int]int, len(pages)) + for i := range pages { + p := &pages[i] + if p.Folio == nil || isContentsPage[p.No] { + continue + } + if _, dup := folioToPage[*p.Folio]; !dup { + folioToPage[*p.Folio] = p.No + } + } + + // Stable, so that two entries claiming the same printed page stay in the order + // the contents table printed them. An unstable sort ordered them arbitrarily, + // and reconciliation keeps whichever claim it sees last — which made the + // language of those pages depend on the sort's internals rather than on the + // document. + sort.SliceStable(entries, func(i, j int) bool { return entries[i].printed < entries[j].printed }) + + // Every claim is resolved before any boundary is derived from it. A rejected + // claim is not evidence of where the previous section ends: on an index listing + // AR at folio 5, CZ at 7 and EN at 9, where the CZ claim lands inside the Arabic + // section and is vetoed by script, ending Arabic at the page before a claim + // nobody believes cut the section in half. + runs := make([]Run, 0, len(entries)) + // placed indexes the runs that fixed a start, in the same order. + placed := make([]int, 0, len(entries)) + + for _, e := range entries { + printed := e.printed + run := Run{ + Source: SourceIndex, Code: e.code, Title: e.title, + PrintedPage: &printed, Confidence: 0.6, + Note: fmt.Sprintf("listed in the printed index at page %d", printed), + } + if lang, ok := NormalizeCode(e.code); ok { + run.Lang = lang + } + + start, resolved := folioToPage[e.printed] + if !resolved { + // The claimed page has no matching folio anywhere in the document, + // which is itself evidence the claim is wrong. Keep the entry for its + // label and title; it contributes no boundary. + run.Confidence = 0.3 + run.Note = fmt.Sprintf("printed index claims page %d, which no page in this document prints", printed) + runs = append(runs, run) + continue + } + + // Script vetoes a claim that lands on the wrong alphabet. The measured + // fixture's index lists Czech at printed page 207, which is inside the + // Arabic section — a typo the manual actually ships. Resolving it + // faithfully produces a Czech claim over Arabic pages, and without this + // check that claim fills gaps in stronger signals with a language that is + // nowhere near those pages. A cheap signal correcting a more informative + // one is the whole point of keeping several. + if run.Lang != "" && !ScriptAllows(scriptAt(pages, start), BaseLanguage(run.Lang)) { + run.Confidence = 0.1 + // Deliberately not called a typo. Both causes look identical here and + // the distinction matters to a reader: the claim may be wrong outright + // (a real manual lists Czech at a page deep inside the Arabic section) + // or merely off by a page, landing on the tail of the previous + // language. Either way it cannot be a boundary, and saying only what + // was observed avoids asserting which. + run.Note = fmt.Sprintf( + "printed index claims %s starts at page %d, but that page is %s script, so it fixes no boundary", + e.code, printed, scriptAt(pages, start)) + runs = append(runs, run) + continue + } + run.Start = start + runs = append(runs, run) + placed = append(placed, len(runs)-1) + } + + // A section ends where the next accepted claim begins, and runs to the end of + // the document when there is none. A later claim that resolves to an earlier + // page is the index contradicting itself rather than a boundary, so only a + // start beyond this one closes the run. + for i, idx := range placed { + runs[idx].End = lastPageNo(pages) + for _, next := range placed[i+1:] { + if runs[next].Start > runs[idx].Start { + runs[idx].End = runs[next].Start - 1 + break + } + } + } + return runs +} + +// parseIndexPage extracts code/title/page triples from a contents page. +// +// The shape a printed index takes is a language code, a title in that language, +// then the page it starts on. pdftotext emits them in that reading order. +func parseIndexPage(text string) []struct { + code, title string + printed int +} { + type triple struct { + code, title string + printed int + } + var out []triple + + // Formatting characters are stripped for the same reason as in pageTag: a + // contents table that lists right-to-left languages wraps their codes in bidi + // embedding marks. + lines := make([]string, 0, 64) + for line := range strings.Lines(text) { + if line = strings.TrimSpace(stripFormatting(line)); line != "" { + lines = append(lines, line) + } + } + + for i, line := range lines { + if len([]rune(line)) > maxRunesInCodeLine || !looksLikeLanguageCode(line) { + continue + } + // The token must name a language something recognises, not merely be shaped + // like a code. Without this the parser reads a page of service addresses as a + // contents table, and it is the only page of the measured column manual it + // reads at all: VIA from "Via Monte Rosa" claimed pages 28-45, FAX claimed + // 46-48, UA from a Ukrainian postal address claimed 49-68 with the title + // "Telefax", Z came from "Sp. z o.o." and NDE from "Neunkirchen" with a phone + // number for a page. Those reached the user as "68 pages in 2 languages, none + // of them yours. It has fax and Ukrainian." + // + // This is the same rule [KnownLanguage] states for regions, applied one layer + // earlier: an unrecognised code is still stored and still reportable when a + // document really prints one, but it may not drive a decision. Here the + // decision it was driving is whether the page is a contents table at all. + if normalised, ok := NormalizeCode(line); !ok || !KnownLanguage(normalised) { + continue + } + // Walk forward for this entry's page number, collecting the title on the + // way. Stop at the next code line: a missing page number means a + // malformed entry, not a licence to consume the following one. + var title []string + for j := i + 1; j < len(lines) && j <= i+indexLookaheadLines; j++ { + next := lines[j] + if looksLikeIndexLabel(next) { + break + } + if n, err := strconv.Atoi(next); err == nil && n > 0 { + out = append(out, triple{ + code: strings.ToUpper(line), + title: strings.Join(title, " "), + printed: n, + }) + break + } + title = append(title, next) + } + } + + result := make([]struct { + code, title string + printed int + }, len(out)) + for i, t := range out { + result[i] = struct { + code, title string + printed int + }{t.code, t.title, t.printed} + } + return result +} + +// maxRunesInIndexLabel bounds how long a line can be and still be the next +// contents entry's label. Four covers the three-letter codes manufacturers print +// and leaves ZH-HK to [looksLikeLanguageCode]. +const maxRunesInIndexLabel = 4 + +// looksLikeIndexLabel reports whether a contents-table line is the next entry's +// language label rather than part of this entry's title. +// +// [looksLikeLanguageCode] is too narrow for this: it matches only XX and XX-XX, +// while real manufacturers print POR, SPA, CHI and SRB. Such a line was taken for +// title text and the walk continued into the *following* entry's page number, so +// one entry claimed its neighbour's start page and carried its neighbour's title. +// +// Case is what keeps this from eating titles: a contents table prints its codes in +// capitals, and requiring all-uppercase ASCII leaves ordinary short title lines +// alone. +func looksLikeIndexLabel(s string) bool { + if looksLikeLanguageCode(s) { + return true + } + r := []rune(s) + if len(r) < 2 || len(r) > maxRunesInIndexLabel { + return false + } + for _, c := range r { + if c > unicode.MaxASCII || !unicode.IsUpper(c) { + return false + } + } + return true +} + +// group is a maximal span of consecutive pages sharing a key. +type group struct { + key string + start, end int + startIdx, endIdx int +} + +func (g group) pages() int { return g.end - g.start + 1 } + +// groupBy splits pages into maximal runs of consecutive page numbers sharing a +// key. Consecutiveness matters: a document whose page 10 and page 40 share a tag +// has two runs, not one 31-page run. +func groupBy(pages []Page, key func(*Page) string) []group { + var groups []group + for i := range pages { + p := &pages[i] + k := key(p) + if n := len(groups); n > 0 && groups[n-1].key == k && groups[n-1].end == p.No-1 { + groups[n-1].end = p.No + groups[n-1].endIdx = i + continue + } + groups = append(groups, group{key: k, start: p.No, end: p.No, startIdx: i, endIdx: i}) + } + return groups +} + +func lastPageNo(pages []Page) int { + if len(pages) == 0 { + return 0 + } + return pages[len(pages)-1].No +} + +// scriptAt returns the dominant script of a page by its 1-based number. +func scriptAt(pages []Page, no int) string { + for i := range pages { + if pages[i].No == no { + return pages[i].Script + } + } + return "" +} diff --git a/internal/extern/extern.go b/internal/extern/extern.go index c987073..5b8d3b9 100644 --- a/internal/extern/extern.go +++ b/internal/extern/extern.go @@ -88,6 +88,47 @@ var ( VersionArgs: []string{"-v"}, Install: popplerInstall, } + // PDFToHTML is listed separately from pdftotext because it answers a question + // pdftotext cannot: where on the page the text is. Only its XML output carries + // coordinates, and those are what a column is — on a manual whose languages run + // in parallel columns a language IS a column, and no per-page reading of such a + // document can be right. See docs/design/regions.md. + // + // Its output also carries font size, family and weight, which separate a heading + // from a paragraph and are what this tool was first added for. Measured on the + // fixture's English section: body text is 11pt regular at 58% of characters, + // while 17pt *regular* is safety body copy at another 15% — so a + // "larger than body means heading" rule promotes prose. Weight is the + // discriminator, and pdftotext does not report it. That use is still ahead. + // + // Optional, deliberately. A document with no positioned text still probes and + // still gets a per-page language map; what is lost is the column resolution, and + // the probe says so rather than failing. + PDFToHTML = Tool{ + Name: "pdftohtml", + Purpose: "read where text sits on the page, which is how parallel language columns are found", + VersionArgs: []string{"-v"}, + Install: popplerInstall, + } + // PDFToCairo is listed separately again, and for the same kind of reason + // PDFToHTML is: it reports something no other poppler tool reports at all. + // A manual's tables are found from the lines the page draws, and those lines + // are vector graphics rather than text. Measured on the parallel-columns + // fixture's page 57, which is visibly ruled: `pdftohtml -xml` emits exactly + // four kinds of element — pdf, page, fontspec and text — and not one path, + // so the rules are not merely hard to find there, they are absent. + // `pdftocairo -svg` reports them exactly, in the PDF's own points, which is + // the space pdftohtml uses divided by 1.5. See docs/design/conversion.md. + // + // Optional like the rest. A document whose tables cannot be read still + // converts to blocks; what is lost is the cell structure, and the caller says + // so rather than failing the document. + PDFToCairo = Tool{ + Name: "pdftocairo", + Purpose: "read the ruled lines a page draws, which is how tables are found", + VersionArgs: []string{"-v"}, + Install: popplerInstall, + } Tesseract = Tool{ Name: "tesseract", Purpose: "OCR scanned manuals and phone photos", @@ -108,7 +149,7 @@ var popplerInstall = map[string]string{ // All is every tool manualbox may use, in doctor-report order. func All() []Tool { - return []Tool{PDFToText, PDFToPPM, PDFImages, PDFInfo, Tesseract} + return []Tool{PDFToText, PDFToHTML, PDFToCairo, PDFToPPM, PDFImages, PDFInfo, Tesseract} } // ErrNotFound is returned by [Require] when a tool is not installed. diff --git a/internal/extern/extern_test.go b/internal/extern/extern_test.go index efb7850..e3f700c 100644 --- a/internal/extern/extern_test.go +++ b/internal/extern/extern_test.go @@ -141,3 +141,37 @@ func TestEveryKnownToolHasHelpfulMetadata(t *testing.T) { } } } + +func TestPDFToHTMLIsKnown(t *testing.T) { + // pdftohtml is listed separately from pdftotext because it answers a question + // pdftotext cannot: only its XML output carries font size and weight, and + // weight is what separates a heading from a paragraph. Measured on a real + // manual, 17pt *regular* text is safety body copy, so size alone misclassifies + // 15% of the characters on those pages as headings. + // + // It must be in All(), because that is what doctor and the instance endpoint + // enumerate — a tool the pipeline needs but never reports is one the user + // cannot be told to install. + var found bool + for _, tool := range All() { + if tool.Name == "pdftohtml" { + found = true + if tool.Purpose == "" { + t.Error("pdftohtml has no purpose, so doctor cannot explain why it matters") + } + if (Status{Tool: tool}).InstallHint() == "" { + t.Error("pdftohtml has no install hint for this platform") + } + } + } + if !found { + t.Fatal("pdftohtml is not in All(), so doctor will never mention it") + } + + // It ships in poppler-utils alongside pdftotext, so if one is present the + // other should be too. A split would mean the Docker image needs changing. + if Available(PDFToText) && !Available(PDFToHTML) { + t.Error("pdftotext is installed but pdftohtml is not — they are packaged together, " + + "so this means the deployment needs a separate package") + } +} diff --git a/internal/fixture/fixture.go b/internal/fixture/fixture.go index 47d594b..c7906b9 100644 --- a/internal/fixture/fixture.go +++ b/internal/fixture/fixture.go @@ -41,18 +41,127 @@ type Section struct { Note string `json:"note,omitempty"` } +// ColumnFact is one text column of a page. +type ColumnFact struct { + X0 int `json:"x0"` + X1 int `json:"x1"` + Runs int `json:"runs"` + // Lang is the column's language, or empty where the signal declined. An + // empty entry records that nothing was established, not that nothing is + // there. + Lang string `json:"lang"` + // Note explains an entry that needs it — in particular a column deliberately + // left unestablished because the signal got it wrong. + Note string `json:"note,omitempty"` +} + +// PageFact is what is known about a single page. +// +// Sections cannot describe every manual. A document whose languages sit in +// parallel columns has several on one page and no contiguous span for any of +// them, so the unit has to be the page and its columns. See +// docs/design/layouts.md. +type PageFact struct { + Page int `json:"page"` + Columns int `json:"columns"` + // Spanning counts runs crossing a gutter — headings and footers set across + // the full measure, which belong to no single column. + Spanning int `json:"spanning"` + Cols []ColumnFact `json:"cols,omitempty"` + + // Verified is how this entry came to be, and it is load-bearing rather than + // documentation. "image" means a human compared the page against its render; + // those entries are ground truth and may be asserted against. "detector" + // means the code under test produced it, so asserting against it would be + // circular — it records the current reading, not established truth. + Verified string `json:"verified"` +} + +// HumanVerified reports whether this page was checked against its render. +func (p PageFact) HumanVerified() bool { return p.Verified == "image" } + // Manifest describes a fixture document and what the pipeline should find in it. +// +// Sections and PageFacts are alternatives, not both: a sectioned manual records +// Sections, a column manual records PageFacts. Neither is required, because what +// a document can be held to depends on what has actually been measured about it. type Manifest struct { - Name string `json:"name"` - URL string `json:"url"` - SHA256 string `json:"sha256"` - Bytes int64 `json:"bytes"` - Pages int `json:"pages"` - HasTextLayer bool `json:"has_text_layer"` - MedianCharsPerPage int `json:"median_chars_per_page"` - ContentStartsOnPDFPage int `json:"content_starts_on_pdf_page"` - IndexPages []int `json:"index_pages"` - Sections []Section `json:"sections"` + Name string `json:"name"` + URL string `json:"url"` + SHA256 string `json:"sha256"` + Bytes int64 `json:"bytes"` + Pages int `json:"pages"` + HasTextLayer bool `json:"has_text_layer"` + MedianCharsPerPage int `json:"median_chars_per_page"` + ContentStartsOnPDFPage int `json:"content_starts_on_pdf_page"` + IndexPages []int `json:"index_pages"` + + // Layout names the arrangement, and LayoutNote records that it may vary + // within the one document — which it does in the measured column fixture. + Layout string `json:"layout,omitempty"` + LayoutNote string `json:"layout_note,omitempty"` + // Languages is every language present, however it is arranged. + Languages []string `json:"languages,omitempty"` + // KnownLimitations records what this fixture cannot settle, so a reader does + // not mistake its silence for coverage. + KnownLimitations []string `json:"known_limitations,omitempty"` + + Sections []Section `json:"sections,omitempty"` + PageFacts []PageFact `json:"page_facts,omitempty"` + + // PageBox is the page dimensions poppler's XML reports, and TextRuns how many + // positioned runs the whole document yields. Both are what a coordinate-space + // change would show up in first, and neither can be derived from the others. + // + // They are held to different standards on purpose. The box is exactly 1.5 + // times the PDF's own page size and so is a property of the output format — + // assert it exactly. The run count depends on how a version of poppler + // segments a line into runs, which may legitimately shift — assert it with a + // tolerance. + PageBox *PageBox `json:"page_box,omitempty"` + TextRuns int `json:"text_runs,omitempty"` +} + +// PageBox is a document's page dimensions in poppler's XML coordinate space. +type PageBox struct { + Width float64 `json:"width"` + Height float64 `json:"height"` +} + +// PageFact returns the ground truth for a page. +func (m *Manifest) PageFact(page int) (PageFact, bool) { + for i := range m.PageFacts { + if m.PageFacts[i].Page == page { + return m.PageFacts[i], true + } + } + return PageFact{}, false +} + +// VerifiedPages returns only the pages a human checked against the render. +// Those are the ones a detector may legitimately be held to. +func (m *Manifest) VerifiedPages() []PageFact { + var out []PageFact + for i := range m.PageFacts { + if m.PageFacts[i].HumanVerified() { + out = append(out, m.PageFacts[i]) + } + } + return out +} + +// EstablishedColumns counts the columns whose language is known, and the total. +// The gap between them is the honest limit of what this fixture can prove. +func (m *Manifest) EstablishedColumns() (known, total int) { + for i := range m.PageFacts { + for _, c := range m.PageFacts[i].Cols { + total++ + if c.Lang != "" { + known++ + } + } + } + return known, total } // Section returns the section for a language code. diff --git a/internal/fixture/fixture_test.go b/internal/fixture/fixture_test.go index dd6813d..1f4d3db 100644 --- a/internal/fixture/fixture_test.go +++ b/internal/fixture/fixture_test.go @@ -161,3 +161,94 @@ func parseIndexCodes(text string) map[string]bool { } return codes } + +// TestColumnFixtureLoads checks the manifest that describes a parallel-column +// manual. It is the counter-example to the sectioned fixture, and it exists +// because a pipeline built against that one alone found 1 of this document's 5 +// languages. +func TestColumnFixtureLoads(t *testing.T) { + m, err := Load(fixturesDir, "thomas-drybox-amfibia") + if err != nil { + t.Fatalf("load manifest: %v", err) + } + + if m.Layout != "parallel-columns" { + t.Errorf("layout = %q", m.Layout) + } + if len(m.Sections) != 0 { + t.Error("a column manual has no contiguous language sections; Sections should be empty") + } + if len(m.PageFacts) != m.Pages { + t.Errorf("%d page facts for a %d-page document", len(m.PageFacts), m.Pages) + } + if len(m.Languages) != 5 { + t.Errorf("languages = %v, want 5", m.Languages) + } + if len(m.KnownLimitations) == 0 { + t.Error("no known limitations recorded; silence would read as coverage") + } + + // Provenance is the point of this fixture. An entry produced by the detector + // cannot be used to judge the detector, so the two kinds must stay + // distinguishable and both must be present. + verified := m.VerifiedPages() + if len(verified) == 0 { + t.Fatal("no page was human-verified, so nothing here is ground truth") + } + if len(verified) == len(m.PageFacts) { + t.Error("every page claims human verification; only eight were checked against renders") + } + for _, p := range m.PageFacts { + if p.Verified != "image" && p.Verified != "detector" { + t.Errorf("page %d has provenance %q, want image or detector", p.Page, p.Verified) + } + } + + // The verified pages are the acceptance test, so their expected counts must + // be present and self-consistent. + for _, p := range verified { + if p.Columns != len(p.Cols) { + t.Errorf("page %d says %d columns but lists %d", p.Page, p.Columns, len(p.Cols)) + } + for _, c := range p.Cols { + if c.X1 <= c.X0 { + t.Errorf("page %d has an inverted column %d-%d", p.Page, c.X0, c.X1) + } + } + } + + known, total := m.EstablishedColumns() + if known == total { + t.Error("every column claims a language; the manifest records unknowns deliberately") + } + + // The properties that make this document the counter-example. + var multiLang, sameLangTwice int + for _, f := range m.PageFacts { + seen := map[string]int{} + for _, c := range f.Cols { + if c.Lang != "" { + seen[c.Lang]++ + } + } + if len(seen) > 1 { + multiLang++ + } + for _, n := range seen { + if n > 1 { + sameLangTwice++ + break + } + } + } + if multiLang == 0 { + t.Error("no page carries more than one language, so this is not a column fixture") + } + if sameLangTwice == 0 { + t.Error("no page carries one language in two columns — that case is why " + + "column count cannot be treated as language count") + } + t.Logf("%d pages human-verified, %d with several languages, %d with one language "+ + "in several columns, %d of %d columns named", + len(verified), multiLang, sameLangTwice, known, total) +} diff --git a/internal/ingest/convert.go b/internal/ingest/convert.go new file mode 100644 index 0000000..dab237a --- /dev/null +++ b/internal/ingest/convert.go @@ -0,0 +1,208 @@ +package ingest + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/jobs" + "github.com/gordon2/manualbox/internal/registry" +) + +// JobConvert is the job kind that turns the pages in scope into readable blocks. +// It is the first work in this pipeline the user has to authorise, which is what +// [Service.Approve] does; nothing enqueues it on its own. +const JobConvert = "doc.convert" + +// ConvertPayload is the job payload. +// +// It carries the document and nothing else — in particular not the languages. +// That is deliberate twice over. The household is read from configuration in the +// handler because the gate showed the user a specific scope and approving must +// mean that scope rather than one a caller chose; and because the dedupe key is +// the document, a payload that could vary would let two different scopes collapse +// into whichever job was queued first. +type ConvertPayload struct { + DocumentID string `json:"documentId"` +} + +// EnqueueConvert queues the conversion of a document. +// +// The dedupe key is the document, so approving twice — or a client retrying — +// does not convert twice. Priority is below the probe's: a probe is what a user is +// waiting on to make a decision, a conversion is work they have already decided to +// have done. +func (s *Service) EnqueueConvert(ctx context.Context, documentID string) (*jobs.Job, error) { + job, err := s.queue.Enqueue(ctx, JobConvert, ConvertPayload{DocumentID: documentID}, jobs.EnqueueOptions{ + DedupeKey: JobConvert + ":" + documentID, + Priority: 5, + }) + if err != nil && !errors.Is(err, jobs.ErrAlreadyQueued) { + return nil, fmt.Errorf("ingest: queue conversion: %w", err) + } + return job, nil +} + +// handleConvert converts the pages in scope and moves the document to ready. +// +// Idempotent, as every handler must be. [doc.Convert] is a pure function of the +// document's bytes and the household's languages, and [registry.Service.SaveConversion] +// replaces a document's blocks and figures wholesale inside one transaction, so a +// worker killed after doing the work leaves a reclaimed job that converges on the +// same rows rather than doubling them. +// +// # Why the probe's result is re-derived rather than read back +// +// [doc.Convert] needs a *doc.Result, and the probe stored its findings as rows +// rather than keeping the Result whole. This re-runs [doc.Analyze] instead of +// rebuilding one from doc_pages and doc_regions. Analyze is a pure function of the +// bytes — it is what makes the probe idempotent in the first place — whereas a +// reconstruction would be a second implementation of the same object, free to +// drift from the real one in ways no test compares. Measured on the two real +// manuals: about 3.6 s for the 560-page sequential one and about 8 s for the +// 68-page parallel-columns one, against conversions of 13 s and 26 s. It is a +// visible share of a job the user has authorised, and it buys the guarantee that +// what is converted is what the document says today. +func (s *Service) handleConvert(ctx context.Context, job *jobs.Job, report jobs.Reporter) error { + var payload ConvertPayload + if err := job.Unmarshal(&payload); err != nil { + return err + } + if payload.DocumentID == "" { + return errors.New("ingest: convert job has no document") + } + + document, err := s.registry.GetDocument(ctx, payload.DocumentID) + if err != nil { + return err + } + log := s.log.With("document", document.ID, "device", document.DeviceID) + + // Set on every attempt rather than only the first: a reclaimed job is a + // document that is being converted again, whatever the row said when it was + // picked up. + if err := s.registry.SetDocumentState(ctx, document.ID, registry.StateConverting, ""); err != nil { + return err + } + if err := report.Progress(ctx, 0.02, "preparing to convert"); err != nil { + return err + } + + path, err := s.store.Path(document.BlobSHA256) + if err != nil { + return s.jobFailed(ctx, job, document.ID, + fmt.Errorf("ingest: document %s content is missing: %w", document.ID, err)) + } + + if err := report.Progress(ctx, 0.05, "re-reading the document"); err != nil { + return err + } + started := time.Now() + result, err := doc.Analyze(ctx, path) + if err != nil { + return s.jobFailed(ctx, job, document.ID, + fmt.Errorf("ingest: re-read document %s: %w", document.ID, err)) + } + analyzed := time.Since(started) + + // The household from configuration, never from a caller. This is the promise + // the gate made concrete: it showed a specific scope, and approving means that + // scope. + household := s.cfg.Content.Languages + scope := result.ScopeFor(household) + + // And the extra scope from the document's own row, for the same reason and by the + // same rule: stored state, never the job payload. [Service.Approve] wrote it there + // when the user ticked the box, which is what keeps ConvertPayload document-only + // and its dedupe key sound — see migration 00007. + // + // Read from the document fetched above, so a re-run of this job converges on the + // scope that was approved instead of quietly reverting to the smaller one. + opts := doc.ConvertOptions{IncludeNeutralPages: document.IncludeNeutralPages} + + if err := report.Progress(ctx, 0.15, fmt.Sprintf( + "converting %d of %d pages", scope.Pages, result.Info.Pages)); err != nil { + return err + } + + // One call with no progress inside it, which is the honest shape: internal/doc + // takes no callback, and inventing per-page movement here would mean either + // changing that package or reporting a fraction nothing measured. The two + // stages either side of it are what a watching user sees move. + conv, err := doc.Convert(ctx, path, result, household, opts) + if err != nil { + return s.jobFailed(ctx, job, document.ID, + fmt.Errorf("ingest: convert document %s: %w", document.ID, err)) + } + converted := time.Since(started) - analyzed + + // The page furniture is dropped here and not stored, and this is the one line + // where "mark, do not delete" becomes a decision about a database. + // + // [doc.Conversion] flags a printed language tab, a folio and a running head + // rather than removing them, so that internal/verify can hold the rule against a + // second extraction of the page and refute it. doc_blocks is the other kind of + // consumer: everything reading it — the reader, the FTS index, an extraction + // citing a paragraph — wants what a person reads. Storing the furniture and + // filtering it in every one of those places is the same answer written four + // times, three of which would need a schema change to ask the question: kind has + // a CHECK, so a 'furniture' kind is a migration, and a boolean column is a + // migration too, plus a change to the FTS triggers 00006 built to keep the index + // correct without Go. + // + // What it costs is that a wrong rule cannot be undone with an UPDATE. That is a + // smaller cost here than it looks, because a block is wholly derived: the + // original is immutable in the blob store, Convert is a pure function of its + // bytes, and SaveConversion deletes and rewrites. Recovering from a bad rule is + // re-running this job, which is a button that already exists. + // + // Furniture sorts last within its region, so dropping it leaves each region's + // content on the contiguous 0..n-1 its natural key is meant to mean. + content := conv.ContentBlocks() + if err := report.Progress(ctx, 0.9, fmt.Sprintf( + "saving %d blocks and %d pictures", len(content), len(conv.Figures))); err != nil { + return err + } + + // The state is passed into SaveConversion rather than set after it, so that the + // claim and the content it rests on land in one transaction. A document cannot + // say "ready" with no blocks behind it. + if err := s.registry.SaveConversion(ctx, document.ID, content, + figuresOf(conv), s.store, registry.StateReady); err != nil { + return s.jobFailed(ctx, job, document.ID, + fmt.Errorf("ingest: save conversion of document %s: %w", document.ID, err)) + } + + log.Info("document converted", + "blocks", len(content), + "furniture", len(conv.Blocks)-len(content), + "figures", len(conv.Figures), + "pages", len(conv.Pages), + "neutral_pages", len(conv.NeutralPages), + "languages", len(conv.Scope.Languages), + "notes", len(conv.Notes), + "analyze_ms", analyzed.Milliseconds(), + "convert_ms", converted.Milliseconds()) + for _, note := range conv.Notes { + log.Warn("conversion note", "note", note) + } + + return report.Progress(ctx, 1, conv.Summary()) +} + +// figuresOf drops the language attribution a conversion worked out and hands the +// pictures on as plain figures. +// +// Nothing is lost by that: doc_figures deliberately has no language column, +// because a picture belonging to no language belongs to every language, and the +// attribution is re-derived at read time from the same stored regions +// [doc.Convert] used. See [registry.Service.FiguresByLang]. +func figuresOf(conv *doc.Conversion) []doc.Figure { + out := make([]doc.Figure, 0, len(conv.Figures)) + for i := range conv.Figures { + out = append(out, conv.Figures[i].Figure) + } + return out +} diff --git a/internal/ingest/convert_test.go b/internal/ingest/convert_test.go new file mode 100644 index 0000000..82f0173 --- /dev/null +++ b/internal/ingest/convert_test.go @@ -0,0 +1,262 @@ +package ingest_test + +import ( + "context" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/ingest" + "github.com/gordon2/manualbox/internal/jobs" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// approveAndConvert approves a document and runs the queued conversion job +// synchronously, the way the server's worker pool would. +func (h *harness) approveAndConvert(t *testing.T, documentID string) { + t.Helper() + ctx := context.Background() + + if _, err := h.ingest.Approve(ctx, documentID, ingest.ApproveScope{}); err != nil { + t.Fatalf("approve: %v", err) + } + + // Approving must move the document out of the gate immediately, or a user who + // has just approved is offered the same decision again while the queue picks + // the job up. + pending, err := h.registry.GetDocument(ctx, documentID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if pending.State != registry.StateConverting { + t.Fatalf("state after approving = %q, want %q", pending.State, registry.StateConverting) + } + + ran, err := h.pool.RunOnce(ctx) + if err != nil { + t.Fatalf("run convert job: %v", err) + } + if !ran { + t.Fatal("approving queued no conversion job") + } + + document, err := h.registry.GetDocument(ctx, documentID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if document.State == registry.StateFailed { + t.Fatalf("conversion failed: %s", document.LastError) + } +} + +func TestApprovingConvertsOnlyTheHouseholdsLanguages(t *testing.T) { + // The funnel's whole promise: a household that reads German gets the German + // section and nothing from the four beside it. This is the one failure a reader + // would notice immediately. + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE", "FR", "IT", "ES"}, 3, true)) + h.runProbe(t, document.ID) + h.approveAndConvert(t, document.ID) + + after, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if after.State != registry.StateReady { + t.Errorf("state = %q, want %q", after.State, registry.StateReady) + } + + blocks, err := h.registry.Blocks(ctx, document.ID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + if len(blocks) == 0 { + t.Fatal("the document is ready with no blocks at all") + } + + for i := range blocks { + b := &blocks[i] + if b.Lang != "de" { + t.Errorf("block %d on page %d is in %q, not German: %q", + b.Index, b.Page, b.Lang, b.Text) + } + // A section's own body text names its code, so a leaked section is visible + // in the text rather than only in the label. + for _, foreign := range []string{"Section EN", "Section FR", "Section IT", "Section ES"} { + if strings.Contains(b.Text, foreign) { + t.Errorf("a block carries text from a language nobody asked for: %q", b.Text) + } + } + } + + // And the German section did arrive, rather than the funnel having emptied + // everything: three pages of it, each naming itself. + pages := make(map[int]bool, 3) + for i := range blocks { + pages[blocks[i].Page] = true + } + if len(pages) != 3 { + t.Errorf("German converted to %d pages, want 3: %v", len(pages), pages) + } +} + +func TestConvertingTwiceConvergesRatherThanDuplicating(t *testing.T) { + // A worker can die after doing the work but before recording success, so the + // reclaimed job runs again. SaveConversion replaces wholesale; this checks the + // handler as a whole preserves that. + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"DE", "FR"}, 3, true)) + h.runProbe(t, document.ID) + + h.approveAndConvert(t, document.ID) + first, err := h.registry.Blocks(ctx, document.ID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + + h.approveAndConvert(t, document.ID) + second, err := h.registry.Blocks(ctx, document.ID) + if err != nil { + t.Fatalf("blocks after re-converting: %v", err) + } + + if len(first) != len(second) { + t.Errorf("re-converting produced %d blocks where the first run produced %d", + len(second), len(first)) + } + for i := range first { + if i >= len(second) { + break + } + if first[i].Text != second[i].Text || first[i].Index != second[i].Index { + t.Errorf("block %d changed on re-conversion: %q then %q", + i, first[i].Text, second[i].Text) + break + } + } +} + +func TestAReadyDocumentAlwaysHasItsContent(t *testing.T) { + // The state and the content land in one transaction, so there is no moment at + // which a document claims to be readable and is not. Checked by asserting the + // pairing rather than the ordering, which is what a caller can actually observe: + // ready implies blocks, and not-ready implies none. + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", testpdf.TaggedSections([]string{"DE"}, 2, false)) + h.runProbe(t, document.ID) + + atGate, err := h.registry.Blocks(ctx, document.ID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + if len(atGate) != 0 { + t.Errorf("a document at the gate already has %d blocks; nothing may be "+ + "converted before the user approves it", len(atGate)) + } + + h.approveAndConvert(t, document.ID) + + after, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + blocks, err := h.registry.Blocks(ctx, document.ID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + if after.State == registry.StateReady && len(blocks) == 0 { + t.Error("the document says ready and has no blocks, which reads as an empty manual") + } + if after.State != registry.StateReady { + t.Errorf("state = %q, want %q", after.State, registry.StateReady) + } +} + +func TestApprovingAScanIsRefusedRatherThanConvertedToNothing(t *testing.T) { + // A user who authorises spending on a scan must be told it needs OCR. Silently + // converting it to nothing and calling the document ready would show them an + // empty manual and no reason for it. + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + document := h.upload(t, "scan.pdf", testpdf.Blank(4)) + h.runProbe(t, document.ID) + + _, err := h.ingest.Approve(ctx, document.ID, ingest.ApproveScope{}) + if err == nil { + t.Fatal("approving a scan was accepted") + } + if !strings.Contains(err.Error(), "OCR") { + t.Errorf("the refusal does not say why: %v", err) + } + + after, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if after.State != registry.StateAwaitingScope { + t.Errorf("state = %q after a refused approval, want %q — a refusal must not "+ + "move the document", after.State, registry.StateAwaitingScope) + } +} + +func TestApprovingADocumentInNoneOfTheHouseholdsLanguagesIsRefused(t *testing.T) { + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", testpdf.TaggedSections([]string{"FR", "IT"}, 3, true)) + h.runProbe(t, document.ID) + + if _, err := h.ingest.Approve(ctx, document.ID, ingest.ApproveScope{}); err == nil { + t.Fatal("approving a document with nothing in scope was accepted") + } +} + +func TestAPermanentlyFailedConversionExplainsItselfOnTheDocument(t *testing.T) { + // The same requirement the probe has one stage earlier: a document whose job + // gave up must not sit in "converting" for ever while nothing is converting it. + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", testpdf.TaggedSections([]string{"DE"}, 2, false)) + h.runProbe(t, document.ID) + + // Remove the stored bytes so the conversion cannot succeed. Row and content + // have diverged, which no retry repairs. + if err := h.store.Delete(document.BlobSHA256); err != nil { + t.Fatalf("delete blob: %v", err) + } + // Queued directly with one attempt, so the first failure is the permanent one. + // Approve would use the default, and a job with retries left must not declare + // the document failed. + if _, err := h.queue.Enqueue(ctx, ingest.JobConvert, + ingest.ConvertPayload{DocumentID: document.ID}, + jobs.EnqueueOptions{MaxAttempts: 1}); err != nil { + t.Fatalf("enqueue: %v", err) + } + if _, err := h.pool.RunOnce(ctx); err != nil { + t.Fatalf("run convert job: %v", err) + } + + after, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if after.State == registry.StateConverting { + t.Error("the document is still converting with no job able to convert it") + } + if after.State != registry.StateFailed { + t.Errorf("state = %q, want %q", after.State, registry.StateFailed) + } + if after.LastError == "" { + t.Error("the document carries no explanation of why it failed") + } +} diff --git a/internal/ingest/gate.go b/internal/ingest/gate.go new file mode 100644 index 0000000..6e5f119 --- /dev/null +++ b/internal/ingest/gate.go @@ -0,0 +1,838 @@ +package ingest + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/jobs" + "github.com/gordon2/manualbox/internal/registry" +) + +// Gate is the pre-flight question: what manualbox is holding, what it would +// process, and what that would cost — asked before anything is spent. +// +// It is built entirely from stored probe results, so it survives a restart and +// costs nothing to render. Re-probing a document to answer "what is in this?" +// would defeat the purpose of having probed it. Every field below is therefore +// derived from doc_pages, doc_langs and doc_regions and from nothing else. +type Gate struct { + DocumentID string `json:"documentId"` + DeviceID string `json:"deviceId"` + Filename string `json:"filename,omitempty"` + Kind string `json:"kind"` + State string `json:"state"` + Probed bool `json:"probed"` + + Pages int `json:"pages"` + Encrypted bool `json:"encrypted"` + HasTextLayer bool `json:"hasTextLayer"` + MedianChars int `json:"medianChars"` + + // Chars is the document's named text: the characters of every language + // something could name, and the denominator [GateLanguage.Share] is taken + // against. Text nothing could name is excluded, because a share of it would + // silently shrink every language by however much the signals failed to read. + Chars int `json:"chars"` + + // Household is the configured reading languages, echoed back so the UI can + // explain why a section is in or out of scope. + Household []string `json:"household"` + + // InScope are the document's languages the household reads. + InScope []GateLanguage `json:"inScope"` + // Other are the languages present that the household does not read. They are + // listed, never discarded: the original is kept whole, so importing one later + // is a button rather than a re-upload. + Other []GateLanguage `json:"other"` + + // ScopePages counts the pages carrying an in-scope language, DISTINCT pages + // rather than a sum over languages. On a parallel-columns manual a page holds + // several, so summing per-language page counts reports 133 pages of a 68-page + // document. Where languages do not share pages the two agree, which is why the + // sequential manual's 16 is unaffected. + ScopePages int `json:"scopePages"` + ScopeFraction float64 `json:"scopeFraction"` + + // ScopeChars is the characters of the in-scope languages, and + // ScopeCharFraction its share of [Gate.Chars]. This is the honest measure of + // how much of a document a household actually reads: on the measured + // parallel-columns manual German is 26 of 68 pages, 38% by pages, and 20% by + // characters — because it occupies one column of each of those pages. + ScopeChars int `json:"scopeChars"` + ScopeCharFraction float64 `json:"scopeCharFraction"` + + // Conflicts is how many runs the signals disagreed about. Surfaced rather than + // resolved silently. + // + // Runs, deliberately, and not regions: the UI explains this number as the + // document's own contents table disagreeing with its pages, which is what a + // conflicting run means. A conflicting region is a different disagreement — a + // column's alphabet against the page's printed tab — and the sequential manual + // has 1 of the first and 32 of the second. Reporting 32 under the first + // sentence would be a lie. A region's dispute reaches the user through + // [GateLanguage.Conflict] on the languages regions named. + Conflicts int `json:"conflicts"` + // UnlabelledPages is how many content pages carry text that no signal could + // name. Front matter and a back cover are excluded: they carry text and belong + // to no section legitimately, so counting them would report a fault on every + // document. On the measured manuals it is 2 and 0. + UnlabelledPages int `json:"unlabelledPages"` + + // Neutral is the extra scope on offer: the pages that carry content and that no + // language owns. Nil when there are none, or when the question cannot be asked at + // this document's resolution — see [GateNeutral]. + Neutral *GateNeutral `json:"neutral,omitempty"` + + // RequiresApproval reports whether the document exceeds ingest.max_pages_auto + // and so may not be processed without the user saying yes. + RequiresApproval bool `json:"requiresApproval"` + MaxPagesAuto int `json:"maxPagesAuto"` + + // Cost is what processing the scope would cost. It is deliberately not a + // guess: see [CostEstimate]. + Cost CostEstimate `json:"cost"` + + // Summary is a one-line human description of the situation. + Summary string `json:"summary"` +} + +// GateLanguage is one of a document's languages as the gate reports it. +// +// It embeds the stored run so that every field the run carried stays present and +// keeps its meaning — title, printed page, span, confidence — and adds what only +// the region map can say. A language the per-page signals never named has no run +// at all, and then the embedded fields carry what the regions know: the printed +// code, the language, its page span, and no title or confidence, because regions +// store neither and inventing them would be an estimate. +type GateLanguage struct { + registry.LanguageRun + + // Chars is how much of this language the document holds, in runes. + // + // Characters lead and pages are context. A language occupying one of three + // columns on 26 of 68 pages is not 26 pages of reading, and the page count on + // its own says it is — see SharesPages. + Chars int `json:"chars"` + // Share is Chars as a fraction of [Gate.Chars], 0 to 1. + Share float64 `json:"share"` + // SharesPages reports that this language does not have its pages to itself: + // somewhere it occupies a box on a page another language also occupies. + // + // This is what stops the page count misleading, and it is worked out from a + // page carrying more than one region rather than from a region's x0 — a + // leftmost column legitimately begins at 0, so testing x0 would call every + // left-hand column whole-page. + SharesPages bool `json:"sharesPages"` +} + +// GateNeutral is the second scope the gate offers: the pages that carry content +// and that no language region claims. +// +// WHY IT IS OFFERED AT ALL. The funnel converts the pages a household's languages +// occupy, which is the whole point of it, and the pages outside every language are +// therefore never converted. Usually that is right — a cover, a colophon. On the +// measured sequential manual it is not: PDF page 5 is an exploded parts diagram +// whose four sub-drawings the figure pass finds, 31 places in the content pages say +// "see A-1", and no language section contains page 5, so no reader of any language +// has ever been served it. docs/design/conversion.md records the measurement and +// records the user's intended answer, which is this: offer the pages, do not guess. +// +// WHY IT REPORTS PICTURES AND NOT ONLY CHARACTERS. Measured on both fixtures, the +// character count alone points the wrong way — the sequential manual's 7 pages hold +// 1,656 characters and 61 pictures, the columns manual's 2 hold 11,256 characters +// and none. The set is worth taking on one document and is furniture on the other, +// and only the picture count says which. See doc.countNeutralInk. +// +// EVERY FIELD IS READ FROM STORED ROWS. Pages and Chars come from doc_pages and +// doc_regions, Figures from the census the probe stored on doc_pages. Nothing here +// is estimated, and Figures is absent rather than 0 when nothing counted. +type GateNeutral struct { + // Pages are the PDF pages on offer, ascending. Sent in full rather than as a + // count so the user can look them up in the original, which is what + // /documents/{id}/content is for and what the intended answer asks of them. + Pages []int `json:"pages"` + // Chars is how much text those pages hold, in runes. + Chars int `json:"chars"` + // Figures is how many pictures they hold, and NIL WHEN THEY WERE NOT COUNTED — + // never 0 for that, because 0 is the columns manual's real and useful answer and + // "not counted" is a different thing to tell a user. Absent when the set was too + // large to census; see doc.maxNeutralInkPages. + Figures *int `json:"figures,omitempty"` + // Note says why the pictures were not counted, when they were not. + Note string `json:"note,omitempty"` + // Included reports that this document has already been approved with these pages + // in scope. It is the stored decision echoed back, so a gate re-rendered after + // approval shows what was actually chosen rather than an unticked box. + Included bool `json:"included"` +} + +// CostEstimate is what the scope would cost to process. +// +// When no AI provider is configured there is no honest number to show. A token +// count depends on a specific model's tokeniser, and a currency figure depends on +// a billing mode — a metered key bills money, a subscription draws down a rolling +// window, a local model costs nothing at all. Inventing a figure from a character +// count would be the kind of estimate that turns out wrong by a factor of two, so +// Available stays false and Reason says why. See docs/design/providers.md. +type CostEstimate struct { + Available bool `json:"available"` + // Chars is measured, free, and always present: the extracted character count + // of the text in scope. It is a real quantity rather than a prediction, and it + // is the same number as [Gate.ScopeChars] — repeated here because this is the + // struct a caller asks about spending. + Chars int `json:"chars"` + Reason string `json:"reason,omitempty"` +} + +// Gate assembles the pre-flight answer for a document. +// +// The language map is read from the regions where there are regions, and from the +// per-page runs otherwise. That is not a preference between two equivalent +// sources. On a parallel-columns manual the per-page map names nothing — a page +// there holds three languages and no per-page answer about it can be right — so +// summarised from runs alone that document reported "68 pages, but no language +// could be identified" while its regions held five languages and 240,622 +// characters. Regions are the finer-grained record of the same reconciliation, so +// on a sequential manual the two agree and nothing it reports changes. +// +// An empty region set is not the claim that a manual has one language: a document +// probed on a host without pdftohtml has no regions and a complete per-page map, +// which is exactly what the fallback is for. +func (s *Service) Gate(ctx context.Context, documentID string) (*Gate, error) { + document, err := s.registry.GetDocument(ctx, documentID) + if err != nil { + return nil, err + } + + g := &Gate{ + DocumentID: document.ID, + DeviceID: document.DeviceID, + Filename: document.Filename, + Kind: document.Kind, + State: document.State, + Probed: document.Probed(), + Household: s.cfg.Content.Languages, + MaxPagesAuto: s.cfg.Ingest.MaxPagesAuto, + InScope: []GateLanguage{}, + Other: []GateLanguage{}, + } + + if document.PageCount != nil { + g.Pages = *document.PageCount + } + if document.Encrypted != nil { + g.Encrypted = *document.Encrypted + } + if document.HasTextLayer != nil { + g.HasTextLayer = *document.HasTextLayer + } + if document.MedianCharsPerPage != nil { + g.MedianChars = *document.MedianCharsPerPage + } + g.RequiresApproval = g.Pages > s.cfg.Ingest.MaxPagesAuto + + if !g.Probed { + g.Summary = "Not yet read." + g.Cost.Reason = "the document has not been read yet" + return g, nil + } + + runs, err := s.registry.LanguageRuns(ctx, document.ID, doc.SourceReconciled) + if err != nil { + return nil, err + } + regions, err := s.registry.Regions(ctx, document.ID) + if err != nil { + return nil, err + } + pages, err := s.registry.Pages(ctx, document.ID) + if err != nil { + return nil, err + } + + langs := collapseRuns(runs) + g.Conflicts = countConflicts(runs) + if len(regions) > 0 { + langs.addRegions(regions) + } else { + langs.sizeFromPages(pages) + } + langs.finish(g, s.cfg.Content.Languages) + + first, last := contentRange(document) + g.UnlabelledPages = unlabelledPages(regions, pages, first, last) + g.Neutral = neutralScope(regions, pages, document.IncludeNeutralPages) + + g.Cost = s.costEstimate() + g.Cost.Chars = g.ScopeChars + g.Summary = g.summarize() + return g, nil +} + +// languageMap accumulates one entry per language while both stored sources are +// read, keyed on base language so that a document printing ZH-HK and zh is one +// language rather than two. +type languageMap struct { + order []string + byLang map[string]*langEntry +} + +type langEntry struct { + lang GateLanguage + // pages is the distinct pages this language occupies according to the regions, + // which is how a page holding several languages is counted once. + pages map[int]bool + // runPages is the same count according to the runs, summed across them as it + // always was. Kept apart from the map because the two are different + // measurements and mixing them would double-count a page. + runPages int + // fromRun records that a stored run described this language. Where one did, + // the run's own span and page count stand, so a sequential manual reports + // exactly what it reported before regions were read here. + fromRun bool +} + +func newLanguageMap(size int) *languageMap { + return &languageMap{order: make([]string, 0, size), byLang: make(map[string]*langEntry, size)} +} + +// langKey is the language a label belongs to, falling back to the label itself so +// that a manual printing an unrecognised code still gets an entry. Storing the +// unrecognised is deliberate — see doc.KnownLanguage. +func langKey(lang, code string) string { + if k := doc.BaseLanguage(lang); k != "" { + return k + } + return code +} + +func (m *languageMap) at(k string) (*langEntry, bool) { + e, ok := m.byLang[k] + if ok { + return e, false + } + e = &langEntry{pages: make(map[int]bool, 16)} + m.byLang[k] = e + m.order = append(m.order, k) + return e, true +} + +// collapseRuns reduces the stored per-page runs to one entry per language, +// keeping the most specific label and the widest span. This is what the gate did +// before it read regions, unchanged, because a sequential manual must go on +// reporting exactly what it reported. +func collapseRuns(runs []registry.LanguageRun) *languageMap { + m := newLanguageMap(len(runs)) + for i := range runs { + r := &runs[i] + e, fresh := m.at(langKey(r.Lang, r.Code)) + e.fromRun = true + e.runPages += r.Pages + if fresh { + e.lang.LanguageRun = *r + continue + } + run := &e.lang.LanguageRun + if len(r.Lang) > len(run.Lang) { + *run = *r + } + if r.Start < run.Start { + run.Start = r.Start + } + if r.End > run.End { + run.End = r.End + } + } + for _, k := range m.order { + e := m.byLang[k] + e.lang.Pages = e.runPages + } + return m +} + +// countConflicts counts the runs the signals disagreed about. See +// [Gate.Conflicts] for why this is counted over runs and never over regions. +func countConflicts(runs []registry.LanguageRun) int { + n := 0 + for i := range runs { + if runs[i].Conflict { + n++ + } + } + return n +} + +// addRegions folds the region map in: characters and shared pages for every +// language, and a whole entry for a language only the regions named. +func (m *languageMap) addRegions(regions []registry.Region) { + perPage := make(map[int]int, len(regions)) + for i := range regions { + perPage[regions[i].Page]++ + } + + for i := range regions { + r := ®ions[i] + if r.Lang == "" && r.Code == "" { + // Nothing named it, so it belongs to no language's total. Its characters + // are still real, which is what UnlabelledPages reports. + continue + } + e, fresh := m.at(langKey(r.Lang, r.Code)) + e.lang.Chars += r.Chars + e.pages[r.Page] = true + if perPage[r.Page] > 1 { + e.lang.SharesPages = true + } + + if e.fromRun { + // A run already described this language and its record stands: the run + // carries a title, a confidence and a span the regions do not have. + continue + } + run := &e.lang.LanguageRun + if fresh { + run.Source, run.Code, run.Lang, run.Name = r.Source, r.Code, r.Lang, r.Name + run.Note, run.Conflict = r.Note, r.Conflict + run.Start, run.End = r.Page, r.Page + } + if len(r.Lang) > len(run.Lang) { + run.Code, run.Lang, run.Name = r.Code, r.Lang, r.Name + } + if r.Page < run.Start { + run.Start = r.Page + } + if r.Page > run.End { + run.End = r.Page + } + if r.Conflict { + run.Conflict, run.Note = true, r.Note + } + } + + for _, k := range m.order { + e := m.byLang[k] + if !e.fromRun { + e.lang.Pages = len(e.pages) + } + } +} + +// sizeFromPages measures each language from the per-page character counts, for a +// document that has no regions stored. +// +// The two measurements are not identical and the difference is known: whole-page +// counts come from pdftotext and a region's from positioned runs, and on the +// fixtures they disagree by 3.3% and 2.5% on a document's total. Regions are +// preferred where they exist for that reason; where they do not, a 3% difference +// beats reporting nothing. +func (m *languageMap) sizeFromPages(pages []registry.PageFact) { + chars := make(map[int]int, len(pages)) + for i := range pages { + chars[pages[i].Page] = pages[i].Chars + } + for _, k := range m.order { + e := m.byLang[k] + run := &e.lang.LanguageRun + // A run that named a language but could not place it starts at 0 and covers + // no pages, so it has no characters to find either. + if run.Start == 0 { + continue + } + for p := run.Start; p <= run.End; p++ { + e.lang.Chars += chars[p] + } + } +} + +// finish splits the languages into scope and the rest, and totals the document. +func (m *languageMap) finish(g *Gate, household []string) { + scopePages := make(map[int]bool, 64) + for _, k := range m.order { + e := m.byLang[k] + entry := e.lang + g.Chars += entry.Chars + + if _, reads := doc.MatchesAny(entry.Lang, household); reads { + g.InScope = append(g.InScope, entry) + g.ScopeChars += entry.Chars + if len(e.pages) == 0 { + // No regions named this language, so the runs are the only source and + // their page counts sum as they always did. + g.ScopePages += entry.Pages + } + for p := range e.pages { + scopePages[p] = true + } + } else { + g.Other = append(g.Other, entry) + } + } + g.ScopePages += len(scopePages) + + if g.Chars > 0 { + for i := range g.InScope { + g.InScope[i].Share = float64(g.InScope[i].Chars) / float64(g.Chars) + } + for i := range g.Other { + g.Other[i].Share = float64(g.Other[i].Chars) / float64(g.Chars) + } + g.ScopeCharFraction = float64(g.ScopeChars) / float64(g.Chars) + } + if g.Pages > 0 { + g.ScopeFraction = float64(g.ScopePages) / float64(g.Pages) + } +} + +// contentRange is the pages holding actual content, excluding front matter and +// back cover. A document probed before those were recorded has no range, and 0, 0 +// means every page counts. +func contentRange(document *registry.Document) (first, last int) { + if document.ContentStartPage != nil { + first = *document.ContentStartPage + } + if document.ContentEndPage != nil { + last = *document.ContentEndPage + } + return first, last +} + +// unlabelledPages counts the content pages carrying text that nothing could name. +// +// It is the honest measure of how much a statistical detector would add for this +// document, and it must be read from the regions where there are regions: the +// parallel-columns manual has no per-page language at all, so counted from +// doc_pages it reports all 68 of its pages as unnamed when 2 of them are — a +// number that would send the reader looking for a detector this document does not +// need. Counted from its regions it is 2, both of them the service-address pages +// at the back that genuinely name nothing. +func unlabelledPages(regions []registry.Region, pages []registry.PageFact, first, last int) int { + inRange := func(page int) bool { + return (first == 0 || page >= first) && (last == 0 || page <= last) + } + + if len(regions) > 0 { + named := make(map[int]bool, len(regions)) + chars := make(map[int]int, len(regions)) + for i := range regions { + r := ®ions[i] + chars[r.Page] += r.Chars + if r.Lang != "" || r.Code != "" { + named[r.Page] = true + } + } + n := 0 + for page, c := range chars { + if !named[page] && c >= doc.MinTextChars && inRange(page) { + n++ + } + } + return n + } + + n := 0 + for i := range pages { + p := &pages[i] + if p.Lang == "" && p.Chars >= doc.MinTextChars && inRange(p.Page) { + n++ + } + } + return n +} + +// neutralScope assembles the extra scope on offer, from stored rows only. +// +// IT IS NOT unlabelledPages WITH THE RANGE REMOVED, although the two look alike, +// and reusing either for the other would break it. That function answers "how much +// would a statistical detector add?" and so excludes front matter and the back +// cover on purpose — those carry text and belong to no section legitimately, and +// counting them would report a fault on every document. This one answers "what can +// a reader not reach?", and front matter is exactly where the unreachable content +// is: the diagram plate this exists for is PDF page 5. So no content range is +// applied here, and the two report different numbers for the same document — 4 and +// 7 on the sequential manual. +// +// Nil rather than an empty struct when there is nothing to offer, so that the UI's +// question is "is there a second scope?" and not "is its page list empty?". +// +// Nil also when the document has no regions at all. That is not the claim that +// every page is owned; it is that this question is asked of the region map and a +// document probed without pdftohtml has none. Answering it from the per-page runs +// instead would offer a different set under the same name. +func neutralScope(regions []registry.Region, pages []registry.PageFact, included bool) *GateNeutral { + if len(regions) == 0 { + return nil + } + + named := make(map[int]bool, len(regions)) + regionChars := make(map[int]int, len(regions)) + for i := range regions { + r := ®ions[i] + regionChars[r.Page] += r.Chars + if r.Lang != "" || r.Code != "" { + named[r.Page] = true + } + } + + out := &GateNeutral{Pages: []int{}, Included: included} + counted := 0 + for i := range pages { + p := &pages[i] + if named[p.Page] { + continue + } + // THE SAME FUNCTION the conversion applies, not the same rule written twice. + // This is what the gate offers and that is what the conversion takes, so a page + // offered here and skipped there would be a promise the funnel broke — and while + // these were two copies, a mutation to one of them was caught by only one of the + // two tests that should both have failed. + if !doc.CarriesContent(p.Chars, p.Figures) { + continue + } + out.Pages = append(out.Pages, p.Page) + // The region's own count where there is one, because that is the measurement + // every other character on this screen is taken with; the page's otherwise. + if c := regionChars[p.Page]; c > 0 { + out.Chars += c + } else { + out.Chars += p.Chars + } + if p.Figures != nil { + counted++ + figures := 0 + if out.Figures != nil { + figures = *out.Figures + } + figures += *p.Figures + out.Figures = &figures + } + } + + if len(out.Pages) == 0 { + return nil + } + // A partial census is not a total. If any offered page went uncounted the sum + // would understate the set, and an understated picture count is exactly the + // mistake that makes a user decline the pages worth taking — so it is withheld + // and said, rather than shown as a number that looks complete. + if counted < len(out.Pages) { + out.Figures = nil + out.Note = "the pictures on these pages were not counted" + } + return out +} + +// costEstimate reports what is known about cost, and admits what is not. +func (s *Service) costEstimate() CostEstimate { + est := CostEstimate{} + switch { + case !s.cfg.Providers.Translate.Enabled() && !s.cfg.Providers.Extract.Enabled(): + est.Reason = "no AI provider is configured, so nothing would be sent anywhere" + default: + // A provider exists but no adapter is implemented yet, so there is still no + // tokeniser to count with. Saying so beats printing a character-derived + // guess that a real count would contradict. + est.Reason = "a token estimate needs the configured provider's own tokeniser, which is not wired up yet" + } + return est +} + +// summarize renders the sentence the gate leads with. +// +// Characters lead and pages are context, which is the decision docs/design/ +// regions.md records: "48 of 560 pages" was always a proxy, and on a manual +// running its languages in parallel columns it is a wrong one, because a language +// filling one column of 26 pages is not 26 pages of reading. +func (g *Gate) summarize() string { + switch { + case g.Encrypted: + return "This document is password-protected, so it cannot be read. The original is stored unchanged." + case !g.HasTextLayer: + return fmt.Sprintf("%d pages with no text layer — this is a scan, and reading it needs OCR.", g.Pages) + case len(g.InScope) == 0 && len(g.Other) > 0: + return fmt.Sprintf("%d pages in %d languages, none of them yours. %s", + g.Pages, len(g.Other)+len(g.InScope), listLanguages(g.Other, 4)) + case len(g.InScope) == 0: + return fmt.Sprintf("%d pages, but no language could be identified.", g.Pages) + } + + total := len(g.InScope) + len(g.Other) + if total == 1 { + return fmt.Sprintf("%d pages in %s.", g.Pages, g.InScope[0].Name) + } + + yours := "Yours is 1 of them" + if len(g.InScope) > 1 { + yours = fmt.Sprintf("Yours are %d of them", len(g.InScope)) + } + return fmt.Sprintf("This manual contains %d languages across %d pages. %s — %s characters, %.0f%% of the text.", + total, g.Pages, yours, groupThousands(g.ScopeChars), 100*g.ScopeCharFraction) +} + +// groupThousands renders a count with thousands separators, because 47641 in a +// sentence a person reads is worse than 47,641. +func groupThousands(n int) string { + digits := strconv.Itoa(n) + sign := "" + if strings.HasPrefix(digits, "-") { + sign, digits = "-", digits[1:] + } + var b strings.Builder + for i, d := range digits { + if i > 0 && (len(digits)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteRune(d) + } + return sign + b.String() +} + +// listLanguages names up to limit languages for a human-readable sentence. +func listLanguages(langs []GateLanguage, limit int) string { + if len(langs) == 0 { + return "" + } + names := make([]string, 0, limit) + for i := range langs { + if i == limit { + return fmt.Sprintf("It has %s and %d more.", joinWords(names), len(langs)-limit) + } + names = append(names, langs[i].Name) + } + return fmt.Sprintf("It has %s.", joinWords(names)) +} + +func joinWords(words []string) string { + switch len(words) { + case 0: + return "" + case 1: + return words[0] + case 2: + return words[0] + " and " + words[1] + } + out := "" + for i, w := range words[:len(words)-1] { + if i > 0 { + out += ", " + } + out += w + } + return out + " and " + words[len(words)-1] +} + +// Decline records that the user does not want this document processed. The +// original is kept: declining is a decision about processing, never about storage. +func (s *Service) Decline(ctx context.Context, documentID string) error { + return s.registry.SetDocumentState(ctx, documentID, registry.StateDeclined, "") +} + +// ApproveScope is what the user chose at the gate, beyond the languages. +// +// One boolean, and the reason it is a boolean rather than a page list is the whole +// argument in [Service.Approve]: a caller may answer a question the gate asked, and +// may not compose a scope of its own. +type ApproveScope struct { + // IncludeNeutralPages asks for the pages no language owns, which the gate offered + // as [Gate.Neutral]. Ignored — with no error — when there was nothing to offer: + // the request is a yes to a question, and a yes to a question that was not asked + // converts nothing extra rather than failing. + IncludeNeutralPages bool +} + +// Approve is Decline's opposite: the user has seen what the gate reported and +// authorises the work. It moves the document to converting and queues the job. +// +// # What is approved is the scope the gate showed, not a scope the caller sends +// +// There is no language argument, and there must not be one. The gate rendered +// this household's languages out of configuration and told the user what +// converting them would involve; taking a different set from the request body +// would let the thing approved differ from the thing shown, which is the one +// promise the funnel makes. [Service.handleConvert] reads the same configuration +// again for the same reason. +// +// # Why one boolean may cross that line, and a page list may not +// +// [ApproveScope.IncludeNeutralPages] is a scope decision that arrives in the +// request, so it needs the promise restated rather than assumed. It keeps it, on +// three counts: +// +// 1. THE GATE SHOWED BOTH ANSWERS. It rendered the pages, their characters and +// their pictures as [Gate.Neutral] and asked the user to include them or not. +// Both outcomes are on the screen, so neither can differ from what was shown. +// The thing the original rule forbids is a scope the gate never displayed. +// 2. THE CALLER CANNOT NAME A PAGE. The set is [doc.Result.NeutralPages], +// recomputed from the stored region map when the conversion runs. A client +// holding a gate from before a re-probe can still only say yes or no; it cannot +// smuggle in a page, and it cannot enlarge the set. +// 3. IT COULD NOT HAVE BEEN CONFIGURATION. The household's languages belong in +// config because they are a property of the household. This is a property of the +// document — one manual's unowned pages are diagram plates, another's are a page +// of service addresses, and the fixtures are one of each — so a setting would be +// wrong for whichever document was uploaded second. +// +// What it must not become is a page list, or a language list, or anything else that +// lets a request describe work instead of accepting an offer. +// +// The decision is written to the document row, not carried in the job payload. See +// [registry.Service.ApproveScope] and migration 00007 for why: the payload's dedupe +// key is the document, so a scope in the payload would let an approval with the +// pages and one without collapse onto whichever was queued first. +// +// Refusals are up front rather than left to produce an empty conversion. A +// document that says "ready" with no blocks reads as an empty manual, and a user +// who authorised spending on a scan deserves to be told it needs OCR rather than +// shown nothing. +func (s *Service) Approve(ctx context.Context, documentID string, scope ApproveScope) (*jobs.Job, error) { + g, err := s.Gate(ctx, documentID) + if err != nil { + return nil, err + } + + switch { + case !g.Probed: + return nil, fmt.Errorf("%w: this document has not been read yet, so there is "+ + "nothing to approve", registry.ErrInvalid) + case g.Encrypted: + return nil, fmt.Errorf("%w: this document is password-protected, so its pages "+ + "cannot be read", registry.ErrInvalid) + case !g.HasTextLayer: + return nil, fmt.Errorf("%w: this document has no text layer — it is a scan, and "+ + "reading it needs OCR", registry.ErrInvalid) + case len(g.InScope) == 0: + return nil, fmt.Errorf("%w: none of this document's languages are ones this "+ + "household reads, so there is nothing in scope to convert", registry.ErrInvalid) + } + + // A yes to a question the gate did not ask converts nothing extra. Narrowing it + // here rather than trusting the flag means a client that always sends true cannot + // put a document into a scope its gate had nothing to offer for. + includeNeutral := scope.IncludeNeutralPages && g.Neutral != nil + + // The state moves before the job is queued, so a user who has just approved + // never sees the gate offer them the decision again while the queue picks the + // job up. The other order would race: a worker that started before this write + // would have its "ready" overwritten with "converting" and the document would + // sit converting for ever. + // + // The approved scope lands in the same statement as the state, so the row can + // never say "converting" under a scope nobody chose. + if err := s.registry.ApproveScope(ctx, documentID, registry.StateConverting, + includeNeutral); err != nil { + return nil, err + } + job, err := s.EnqueueConvert(ctx, documentID) + if err != nil { + // Put it back where it was, or the document is stuck in a state no job will + // ever leave. + if back := s.registry.SetDocumentState(ctx, documentID, g.State, ""); back != nil { + s.log.Error("restoring the document state after a failed enqueue failed", + "document", documentID, "state", g.State, "error", back) + } + return nil, err + } + return job, nil +} diff --git a/internal/ingest/gate_test.go b/internal/ingest/gate_test.go new file mode 100644 index 0000000..7829785 --- /dev/null +++ b/internal/ingest/gate_test.go @@ -0,0 +1,366 @@ +package ingest_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/ingest" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// The gate's language map comes from the stored regions where there are regions +// and from the per-page runs otherwise, so these tests drive both paths through +// the real pipeline. They generate their own PDFs: the shapes below are the two +// layouts docs/design/layouts.md describes, in miniature. + +// requirePDFToHTML skips a test that needs positioned text. Regions cannot be +// invented without coordinates, so without this tool the pipeline legitimately +// stores none and there is nothing to assert. +func requirePDFToHTML(t *testing.T) { + t.Helper() + if !extern.Available(extern.PDFToHTML) { + t.Skip("pdftohtml is not installed, so no regions are stored") + } +} + +// columnManual builds a manual that runs its languages in parallel columns: every +// page carries all of them side by side, each column headed by its own printed +// code. +// +// Two details are what make it the real shape rather than a convenient one. There +// is no contents table, because the measured manual's contents page is laid out in +// columns and does not parse — so no vocabulary of codes exists and the per-page +// tag reader cannot narrow anything. And each page opens with a heading set across +// the whole measure, so the first lines of the page are not a code either. The +// result is the case that matters: the per-page signals name nothing at all, and +// only the columns know what the document contains. +func columnManual(codes []string, pages int) testpdf.Doc { + return columnManualFrom(codes, pages, 40) +} + +// columnManualFrom is the same, with the left column's offset given, because +// whether a language shares its pages must not depend on where the leftmost +// column happens to start. +func columnManualFrom(codes []string, pages, leftEdge int) testpdf.Doc { + var d testpdf.Doc + for p := range pages { + page := testpdf.Page{Lines: []string{ + "Installation and maintenance of the appliance, page " + fmt.Sprint(p+1), + "Read the whole of this section before starting any work at all", + "Keep this booklet for later reference and for the next owner", + }} + for i, code := range codes { + lines := []string{code} + for range 10 { + lines = append(lines, "Maintenance information.") + } + page.Columns = append(page.Columns, testpdf.Column{ + X: leftEdge + i*190, + Lines: lines, + }) + } + d.Pages = append(d.Pages, page) + } + return d +} + +func TestGateReadsAColumnManualFromItsRegions(t *testing.T) { + // The bug this fixes, in miniature: a page holding three languages has no + // honest per-page answer, so the per-page map names nothing and the gate said + // "no language could be identified" about a document whose regions held every + // one of them. + requirePDFToHTML(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "columns.pdf", columnManual([]string{"DE", "PL", "NL"}, 4)) + h.runProbe(t, document.ID) + + // The premise, asserted rather than assumed: there is no per-page answer to + // read. If this ever stops being true the test below stops testing anything. + runs, err := h.registry.LanguageRuns(ctx, document.ID, doc.SourceReconciled) + if err != nil { + t.Fatalf("language runs: %v", err) + } + if len(runs) != 0 { + t.Fatalf("the per-page map named %d runs, so this document no longer exercises "+ + "the regions path: %+v", len(runs), runs) + } + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + + if len(gate.InScope) != 1 { + t.Fatalf("in scope = %d languages, want 1 (de): %+v", len(gate.InScope), gate.InScope) + } + if len(gate.Other) != 2 { + t.Errorf("other = %d languages, want 2 (pl, nl): %+v", len(gate.Other), gate.Other) + } + + german := gate.InScope[0] + if german.Lang != "de" { + t.Errorf("in-scope language = %q, want de", german.Lang) + } + if german.Chars == 0 { + t.Error("German reached the gate with no characters, which is the whole point of reading regions") + } + if german.Pages != 4 { + t.Errorf("German is on %d pages, want 4", german.Pages) + } + // The page count on its own would read as four pages of German reading. It is + // one column of each of four pages, and this is the field that says so. + if !german.SharesPages { + t.Error("German does not report sharing its pages, though every page holds three languages") + } + + // Characters lead: a third of the columns is roughly a third of the text, while + // the page count is all of them. + if gate.Chars <= german.Chars { + t.Errorf("document chars = %d, not more than German's %d", gate.Chars, german.Chars) + } + if german.Share <= 0 || german.Share >= 0.5 { + t.Errorf("German's share = %.3f, want a third-ish of a three-language document", german.Share) + } + if gate.ScopeChars != german.Chars { + t.Errorf("scope chars = %d, want German's %d", gate.ScopeChars, german.Chars) + } + if gate.Cost.Chars != gate.ScopeChars { + t.Errorf("cost.chars = %d, want the %d characters in scope — the field is documented "+ + "as measured and always present", gate.Cost.Chars, gate.ScopeChars) + } + + // Distinct pages, not a sum: three languages on four pages is four pages. + if gate.ScopePages != 4 { + t.Errorf("scope pages = %d, want 4", gate.ScopePages) + } + if !strings.Contains(gate.Summary, "3 languages") { + t.Errorf("summary does not name the languages found: %q", gate.Summary) + } + if strings.Contains(gate.Summary, "no language could be identified") { + t.Errorf("the gate still claims it read nothing: %q", gate.Summary) + } +} + +func TestGateSeesASharedPageWhoseLeftColumnBeginsAtTheEdge(t *testing.T) { + // Testing a region's x0 against zero looks like a way to tell a box from a whole + // page, and it is not: a leftmost column can legitimately begin at the page's + // left edge, and then the language filling it would report the page as its own. + // A page carrying more than one region is the robust test, and this document is + // where the two rules disagree. + requirePDFToHTML(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "flush.pdf", columnManualFrom([]string{"DE", "PL", "NL"}, 3, 0)) + h.runProbe(t, document.ID) + + regions, err := h.registry.Regions(ctx, document.ID) + if err != nil { + t.Fatalf("regions: %v", err) + } + // The premise: German's territory really does start at the page edge. + flush := false + for i := range regions { + if regions[i].Lang == "de" && regions[i].X0 == 0 { + flush = true + } + } + if !flush { + t.Skipf("no German region begins at x0 = 0, so this document does not "+ + "separate the two rules: %+v", regions) + } + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + if len(gate.InScope) != 1 { + t.Fatalf("in scope = %d languages, want 1: %+v", len(gate.InScope), gate.InScope) + } + if !gate.InScope[0].SharesPages { + t.Error("German claims its pages as its own, though it fills the left column of " + + "pages that hold three languages") + } +} + +func TestGateCountsPagesSharedByTwoInScopeLanguagesOnce(t *testing.T) { + // Summing per-language page counts is what makes a columns manual report more + // pages in scope than it has: on the measured 68-page manual, five languages of + // 26 to 27 pages each sum to 133. A household reading two of them is still + // reading the same pages. + requirePDFToHTML(t) + h := newHarness(t, []string{"de", "nl"}) + ctx := context.Background() + + document := h.upload(t, "columns.pdf", columnManual([]string{"DE", "PL", "NL"}, 4)) + h.runProbe(t, document.ID) + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + if len(gate.InScope) != 2 { + t.Fatalf("in scope = %d languages, want 2 (de, nl): %+v", len(gate.InScope), gate.InScope) + } + if gate.ScopePages != 4 { + t.Errorf("scope pages = %d, want 4 — two languages sharing the same four pages, "+ + "not 8", gate.ScopePages) + } + if gate.ScopeFraction > 1 { + t.Errorf("scope fraction = %.2f, more than the whole document", gate.ScopeFraction) + } + // Characters do add up, because two columns of a page are twice the reading. + if want := gate.InScope[0].Chars + gate.InScope[1].Chars; gate.ScopeChars != want { + t.Errorf("scope chars = %d, want %d — characters are the thing that sums", + gate.ScopeChars, want) + } +} + +func TestGateOnASequentialManualIsUnchangedByRegions(t *testing.T) { + // The other half of the acceptance in docs/design/regions.md: a change that + // improves the columns manual by altering the sequential one has broken + // something. Every field here is what the runs said before regions were read, + // and the new ones are additions to it. + requirePDFToHTML(t) + h := newHarness(t, []string{"en", "de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE", "FR", "IT", "ES"}, 3, true)) + h.runProbe(t, document.ID) + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + + if len(gate.InScope) != 2 || len(gate.Other) != 3 { + t.Fatalf("language map = %d in scope and %d other, want 2 and 3: %+v %+v", + len(gate.InScope), len(gate.Other), gate.InScope, gate.Other) + } + if gate.ScopePages != 6 { + t.Errorf("scope pages = %d, want 6 — two sections of three pages", gate.ScopePages) + } + + for i := range gate.InScope { + e := &gate.InScope[i] + // The run's own record survives: a language named per page keeps its + // section title, its source and its confidence, none of which a region + // stores. + if e.Source != string(doc.SourceReconciled) { + t.Errorf("%s reports source %q, want the reconciled run's own", e.Lang, e.Source) + } + if e.Title == "" { + t.Errorf("%s lost the section title the contents table printed", e.Lang) + } + if e.Pages != 3 { + t.Errorf("%s covers %d pages, want 3", e.Lang, e.Pages) + } + // This manual runs its languages in sequence, so no language shares a page + // with another. The field that stops a page count misleading must not fire + // where the page count is honest. + if e.SharesPages { + t.Errorf("%s reports sharing its pages, but this manual sets one language per page", e.Lang) + } + if e.Chars == 0 { + t.Errorf("%s reached the gate with no characters", e.Lang) + } + } + + if gate.ScopeChars == 0 || gate.Cost.Chars != gate.ScopeChars { + t.Errorf("scope chars = %d and cost.chars = %d, want both the same non-zero count", + gate.ScopeChars, gate.Cost.Chars) + } + if gate.ScopeCharFraction <= 0 || gate.ScopeCharFraction >= 1 { + t.Errorf("scope char fraction = %.3f for 2 of 5 languages", gate.ScopeCharFraction) + } +} + +func TestGateFallsBackToPerPageRunsWithNoRegions(t *testing.T) { + // An empty region set is not the claim that a manual has one language. A + // document probed on a host without pdftohtml has none stored and a complete + // per-page map, and it must go on reporting that map. + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE", "FR"}, 3, true)) + h.runProbe(t, document.ID) + + if _, err := h.db.Write().ExecContext(ctx, + "DELETE FROM doc_regions WHERE document_id = ?", document.ID); err != nil { + t.Fatalf("delete regions: %v", err) + } + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + + if len(gate.InScope) != 1 || len(gate.Other) != 2 { + t.Fatalf("with no regions the gate reports %d in scope and %d other, want 1 and 2", + len(gate.InScope), len(gate.Other)) + } + if got := gate.InScope[0].Lang; got != "de" { + t.Errorf("in-scope language = %q, want de", got) + } + if gate.ScopePages != 3 { + t.Errorf("scope pages = %d, want 3", gate.ScopePages) + } + // Characters are still measured, from the per-page counts rather than from + // boxes. The two differ by a few percent on a real document; reporting nothing + // would be worse. + if gate.InScope[0].Chars == 0 || gate.Cost.Chars != gate.InScope[0].Chars { + t.Errorf("chars = %d and cost.chars = %d with no regions, want both the German "+ + "section's per-page count", gate.InScope[0].Chars, gate.Cost.Chars) + } + if gate.InScope[0].SharesPages { + t.Error("a language shares pages according to a document with no regions stored") + } +} + +func TestGateCountsContentPagesNothingCouldName(t *testing.T) { + // UnlabelledPages is the honest measure of how much a statistical detector + // would add for this document, and it was declared and never assigned: always + // 0, on every document, however much of it went unread. + requirePDFToHTML(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + // Three pages of columns that name themselves, then a page of the same body + // text with no code anywhere on it. Nothing can name the last page: no printed + // tag, and its letters are the Latin the other pages use. + d := columnManual([]string{"DE", "PL", "NL"}, 3) + unnamed := testpdf.Page{Lines: []string{ + "Service addresses and contact details for every country listed", + }} + for range 12 { + unnamed.Lines = append(unnamed.Lines, + "Ordinary prose with no language code printed anywhere upon it.") + } + d.Pages = append(d.Pages, unnamed) + + document := h.upload(t, "columns.pdf", d) + h.runProbe(t, document.ID) + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + if gate.UnlabelledPages != 1 { + t.Errorf("unlabelled pages = %d, want 1 — the page carrying text that nothing named", + gate.UnlabelledPages) + } + // It is not counted as anybody's, either. + for _, e := range append(append([]ingest.GateLanguage{}, gate.InScope...), gate.Other...) { + if e.Pages > 3 { + t.Errorf("%s claims %d pages, more than the 3 that name it", e.Lang, e.Pages) + } + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go new file mode 100644 index 0000000..e43a371 --- /dev/null +++ b/internal/ingest/ingest.go @@ -0,0 +1,205 @@ +// Package ingest runs the document pipeline as background work and answers the +// question the pre-flight gate asks. +// +// The division of labour: internal/doc knows how to read a document and says +// nothing about databases; this package persists what it found, reports progress, +// and stops at the gate. Nothing here spends money, calls a model, or touches the +// network — the whole point of the funnel in docs/design/ingest.md is that the +// expensive step is the last one and the user authorises it first. +package ingest + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/gordon2/manualbox/internal/config" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/jobs" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// JobProbe is the job kind that runs stages 0 to 2 over an uploaded document. +const JobProbe = "doc.probe" + +// ProbePayload is the job payload. +type ProbePayload struct { + DocumentID string `json:"documentId"` +} + +// Service coordinates the pipeline. +type Service struct { + cfg config.Config + registry *registry.Service + store *store.Store + queue *jobs.Queue + log *slog.Logger +} + +// Deps are the collaborators the service needs. +type Deps struct { + Config config.Config + Registry *registry.Service + Store *store.Store + Jobs *jobs.Queue + Logger *slog.Logger +} + +// New returns an ingest service. +func New(d Deps) *Service { + if d.Logger == nil { + d.Logger = slog.New(slog.DiscardHandler) + } + return &Service{ + cfg: d.Config, registry: d.Registry, store: d.Store, + queue: d.Jobs, log: d.Logger, + } +} + +// Register wires the pipeline's handlers into a worker pool. +func (s *Service) Register(pool *jobs.Pool) { + pool.Register(JobProbe, s.handleProbe) + pool.Register(JobConvert, s.handleConvert) +} + +// EnqueueProbe queues the free stages for a document. +// +// The dedupe key is the document, so uploading the same file twice — or a client +// retrying — does not queue two probes. An identical pending job is a success for +// the caller, since the work will happen either way. +func (s *Service) EnqueueProbe(ctx context.Context, documentID string) (*jobs.Job, error) { + job, err := s.queue.Enqueue(ctx, JobProbe, ProbePayload{DocumentID: documentID}, jobs.EnqueueOptions{ + DedupeKey: JobProbe + ":" + documentID, + // Probing is cheap and everything the user sees waits on it, so it runs + // ahead of any bulk work queued behind it. + Priority: 10, + }) + if err != nil && !errors.Is(err, jobs.ErrAlreadyQueued) { + return nil, fmt.Errorf("ingest: queue probe: %w", err) + } + return job, nil +} + +// handleProbe runs stages 0 to 2 and stops at the gate. +// +// Idempotent, as every handler must be: a worker can be killed after doing the +// work but before recording success, and the reclaimed job runs again. Re-running +// re-derives the same facts from the same immutable bytes and upserts them, so a +// second run converges rather than duplicating. +func (s *Service) handleProbe(ctx context.Context, job *jobs.Job, report jobs.Reporter) error { + var payload ProbePayload + if err := job.Unmarshal(&payload); err != nil { + return err + } + if payload.DocumentID == "" { + return errors.New("ingest: probe job has no document") + } + + document, err := s.registry.GetDocument(ctx, payload.DocumentID) + if err != nil { + return err + } + + log := s.log.With("document", document.ID, "device", document.DeviceID) + + if err := report.Progress(ctx, 0.05, "reading the document"); err != nil { + return err + } + if err := s.registry.SetDocumentState(ctx, document.ID, registry.StateProbing, ""); err != nil { + return err + } + + path, err := s.store.Path(document.BlobSHA256) + if err != nil { + // The blob is gone: the row and the bytes have diverged, which no retry + // will repair. + return s.probeFailed(ctx, job, document.ID, + fmt.Errorf("ingest: document %s content is missing: %w", document.ID, err)) + } + + if err := report.Progress(ctx, 0.2, "looking for a text layer"); err != nil { + return err + } + + result, err := doc.Analyze(ctx, path) + if err != nil { + return s.probeFailed(ctx, job, document.ID, + fmt.Errorf("ingest: analyse document %s: %w", document.ID, err)) + } + + if err := report.Progress(ctx, 0.7, s.progressNote(result)); err != nil { + return err + } + + // Every probed document stops here in this milestone: conversion does not + // exist yet, so there is nothing to advance to even for a small document that + // ingest.max_pages_auto would permit. The gate's own answer about whether + // approval is needed is computed for the UI by [Service.Gate]. + if err := s.registry.SaveProbe(ctx, document.ID, result, registry.StateAwaitingScope); err != nil { + return s.probeFailed(ctx, job, document.ID, err) + } + + // regions_unnamed rather than result.Unlabelled, which counts pages carrying + // text that no PER-PAGE run named. On a parallel-columns manual the per-page map + // is legitimately empty — a page there holds three languages and no per-page + // answer about it can be right — so that number was 68 of 68 for a document the + // gate correctly reports 2 unnamed pages for. A log that disagrees with the + // screen by a factor of thirty is how a reader stops trusting both. + unnamed := 0 + for i := range result.Regions { + if result.Regions[i].Lang == "" { + unnamed++ + } + } + log.Info("document probed", + "pages", result.Info.Pages, + "text_layer", result.HasTextLayer, + "languages", len(result.Languages()), + "regions", len(result.Regions), + "regions_unnamed", unnamed) + + return report.Progress(ctx, 1, s.progressNote(result)) +} + +// progressNote describes the outcome in the terms the activity view shows. +func (s *Service) progressNote(res *doc.Result) string { + switch { + case res.Info.Encrypted: + return "the document is password-protected; stored without processing" + case !res.HasTextLayer: + return fmt.Sprintf("%d pages, no text layer — needs OCR", res.Info.Pages) + default: + scope := res.ScopeFor(s.cfg.Content.Languages) + return fmt.Sprintf("%d pages, %d languages; %d pages in yours", + res.Info.Pages, len(res.Languages()), scope.Pages) + } +} + +// probeFailed marks a document as failed once no further attempt will be made, +// and returns the error so the queue can retry until then. +func (s *Service) probeFailed(ctx context.Context, job *jobs.Job, documentID string, cause error) error { + return s.jobFailed(ctx, job, documentID, cause) +} + +// jobFailed records a permanent failure on the document itself, and returns the +// error so the queue can retry until the attempts are gone. +// +// Without this a document whose job exhausts its attempts stays in "probing" or +// "converting" forever, and the UI truthfully reports what the row says — +// "reading…" — while nothing is reading it. The failure has to be recorded where +// the user is looking, not only in the job row. +func (s *Service) jobFailed(ctx context.Context, job *jobs.Job, documentID string, cause error) error { + if job.Attempts >= job.MaxAttempts { + // Deliberately not ctx: on a cancelled context this write is the last + // chance to leave an explanation behind. + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if err := s.registry.SetDocumentState(writeCtx, documentID, registry.StateFailed, cause.Error()); err != nil { + s.log.Error("recording document failure failed", "document", documentID, "error", err) + } + } + return cause +} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go new file mode 100644 index 0000000..17ba0ba --- /dev/null +++ b/internal/ingest/ingest_test.go @@ -0,0 +1,405 @@ +package ingest_test + +import ( + "bytes" + "context" + "path/filepath" + "testing" + + "github.com/gordon2/manualbox/internal/config" + "github.com/gordon2/manualbox/internal/db" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/ingest" + "github.com/gordon2/manualbox/internal/jobs" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// These tests drive the whole pipeline the way the server does: store an upload, +// run the probe job, then read the gate. They need poppler, which CI installs, and +// they generate their own PDFs so nothing is downloaded and nothing is committed. + +type harness struct { + registry *registry.Service + ingest *ingest.Service + store *store.Store + queue *jobs.Queue + pool *jobs.Pool + // db is here for one purpose: deleting a document's regions, which is how a + // host without pdftohtml is simulated without uninstalling poppler. + db *db.DB +} + +func newHarness(t *testing.T, household []string) *harness { + t.Helper() + if !extern.Available(extern.PDFInfo) || !extern.Available(extern.PDFToText) { + t.Skip("poppler is not installed") + } + + ctx := context.Background() + database, err := db.Open(ctx, db.Options{Path: filepath.Join(t.TempDir(), "ingest.db")}) + if err != nil { + t.Fatalf("open database: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + + blobs, err := store.New(filepath.Join(t.TempDir(), "blobs")) + if err != nil { + t.Fatalf("open store: %v", err) + } + + cfg := config.Default() + cfg.Content.Languages = household + // Low enough that the multi-language fixtures in these tests exceed it, which + // is the case the gate exists for. + cfg.Ingest.MaxPagesAuto = 8 + + reg := registry.New(database, registry.Options{}) + queue := jobs.NewQueue(database, nil) + t.Cleanup(func() { queue.Broker().Close() }) + + svc := ingest.New(ingest.Deps{Config: cfg, Registry: reg, Store: blobs, Jobs: queue}) + pool := jobs.NewPool(queue, cfg.Jobs, nil) + svc.Register(pool) + + return &harness{registry: reg, ingest: svc, store: blobs, queue: queue, pool: pool, db: database} +} + +// upload stores a generated document against a new device and returns the +// document, exactly as the HTTP handler would. +func (h *harness) upload(t *testing.T, name string, d testpdf.Doc) *registry.Document { + t.Helper() + ctx := context.Background() + + device, err := h.registry.CreateDevice(ctx, registry.NewDevice{Name: "Test device"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + + ref, err := h.store.Put(ctx, bytes.NewReader(d.Build())) + if err != nil { + t.Fatalf("store upload: %v", err) + } + if err := h.registry.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + + document, _, err := h.registry.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, + Filename: name, MediaType: "application/pdf", Kind: registry.KindManual, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + return document +} + +// runProbe executes the queued probe job synchronously. +func (h *harness) runProbe(t *testing.T, documentID string) { + t.Helper() + ctx := context.Background() + + if _, err := h.ingest.EnqueueProbe(ctx, documentID); err != nil { + t.Fatalf("enqueue probe: %v", err) + } + ran, err := h.pool.RunOnce(ctx) + if err != nil { + t.Fatalf("run probe job: %v", err) + } + if !ran { + t.Fatal("no probe job was queued") + } + + // A handler that failed leaves the reason on the document, which is far more + // useful in a test failure than a bare assertion later on. + document, err := h.registry.GetDocument(ctx, documentID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if document.State == registry.StateFailed { + t.Fatalf("probe failed: %s", document.LastError) + } +} + +func TestProbeBuildsTheLanguageMapAndStopsAtTheGate(t *testing.T) { + h := newHarness(t, []string{"en", "de"}) + ctx := context.Background() + + // Five languages, three pages each, with a contents table: the shape of a real + // multi-language appliance manual in miniature. + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE", "FR", "IT", "ES"}, 3, true)) + + h.runProbe(t, document.ID) + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + + if gate.State != registry.StateAwaitingScope { + t.Errorf("state = %q, want %q: the pipeline must stop and ask", + gate.State, registry.StateAwaitingScope) + } + if !gate.Probed { + t.Error("gate reports the document as unprobed") + } + if gate.Pages != 16 { + t.Errorf("pages = %d, want 16 (1 contents + 5 sections x 3)", gate.Pages) + } + if !gate.HasTextLayer { + t.Error("has text layer = false, but the document is generated with text") + } + + if len(gate.InScope) != 2 { + t.Errorf("in scope = %d languages, want 2 (en, de): %+v", len(gate.InScope), gate.InScope) + } + if len(gate.Other) != 3 { + t.Errorf("other = %d languages, want 3 (fr, it, es): %+v", len(gate.Other), gate.Other) + } + if gate.ScopePages != 6 { + t.Errorf("scope pages = %d, want 6", gate.ScopePages) + } + + // A 16-page document against max_pages_auto of 8 must require approval. + if !gate.RequiresApproval { + t.Errorf("requires approval = false for a %d-page document with max_pages_auto=%d", + gate.Pages, gate.MaxPagesAuto) + } + + // No provider is configured, so there must be no cost figure — and a reason. + if gate.Cost.Available { + t.Error("a cost estimate was offered with no provider configured") + } + if gate.Cost.Reason == "" { + t.Error("cost is unavailable but no reason was given") + } +} + +func TestProbeIsIdempotent(t *testing.T) { + // A worker can be killed after doing its work but before recording success, so + // the reclaimed job runs again. Running twice must converge, not duplicate. + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE"}, 3, true)) + + h.runProbe(t, document.ID) + first, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + + h.runProbe(t, document.ID) + second, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate after re-probe: %v", err) + } + + if first.Pages != second.Pages || first.ScopePages != second.ScopePages { + t.Errorf("re-probing changed the result: %d/%d pages then %d/%d", + first.ScopePages, first.Pages, second.ScopePages, second.Pages) + } + if len(first.InScope) != len(second.InScope) || len(first.Other) != len(second.Other) { + t.Errorf("re-probing changed the language map: %d+%d then %d+%d", + len(first.InScope), len(first.Other), len(second.InScope), len(second.Other)) + } +} + +func TestUploadingTheSameBytesTwiceIsOneDocument(t *testing.T) { + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + d := testpdf.TaggedSections([]string{"EN"}, 2, false) + device, err := h.registry.CreateDevice(ctx, registry.NewDevice{Name: "Kettle"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + + var ids []string + for range 2 { + ref, err := h.store.Put(ctx, bytes.NewReader(d.Build())) + if err != nil { + t.Fatalf("store: %v", err) + } + if err := h.registry.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + document, created, err := h.registry.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, Filename: "m.pdf", Kind: registry.KindManual, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + ids = append(ids, document.ID) + if len(ids) == 2 && created { + t.Error("the second upload of identical bytes reported itself as new") + } + } + + if ids[0] != ids[1] { + t.Errorf("identical uploads produced two documents: %s and %s", ids[0], ids[1]) + } + documents, err := h.registry.ListDocumentsForDevice(ctx, device.ID) + if err != nil { + t.Fatalf("list documents: %v", err) + } + if len(documents) != 1 { + t.Errorf("device has %d documents, want 1", len(documents)) + } +} + +func TestScanWithNoTextLayerIsReportedNotFailed(t *testing.T) { + // A scan is a normal input, not an error. The probe must record that there is + // no text layer and say so, leaving OCR as a separate decision. + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + document := h.upload(t, "scan.pdf", testpdf.Blank(4)) + h.runProbe(t, document.ID) + + gate, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + if gate.State == registry.StateFailed { + t.Errorf("a scan was treated as a failure: %s", gate.State) + } + if gate.HasTextLayer { + t.Error("has text layer = true for a document with no text") + } + if gate.Pages != 4 { + t.Errorf("pages = %d, want 4", gate.Pages) + } + if len(gate.InScope) != 0 { + t.Errorf("languages were claimed for a document with no text: %+v", gate.InScope) + } + if gate.Summary == "" { + t.Error("no summary explaining what happened") + } +} + +func TestEverySignalsViewIsStored(t *testing.T) { + // "This manual also contains FR, IT, ES..." must be answerable without + // re-probing, and a disagreement must stay inspectable afterwards. That means + // each signal's own runs are persisted, not just the reconciled ones. + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE", "FR"}, 3, true)) + h.runProbe(t, document.ID) + + for _, source := range []struct { + name string + want int + }{ + {"page-tag", 3}, + {"index", 3}, + {"reconciled", 3}, + } { + runs, err := h.registry.LanguageRuns(ctx, document.ID, doc.Source(source.name)) + if err != nil { + t.Fatalf("language runs for %s: %v", source.name, err) + } + if len(runs) != source.want { + t.Errorf("%s produced %d runs, want %d: %+v", source.name, len(runs), source.want, runs) + } + } +} + +func TestAPermanentlyFailedProbeExplainsItselfOnTheDocument(t *testing.T) { + // A job that exhausts its attempts must leave the reason where the user is + // looking. Without this the document stays in "probing" for ever and the UI + // truthfully reports "reading…" while nothing is reading it — which is worse + // than an error, because it never resolves. + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", testpdf.TaggedSections([]string{"EN"}, 2, false)) + + // Remove the stored bytes so the probe cannot succeed. Row and content have + // diverged, which no retry repairs. + if err := h.store.Delete(document.BlobSHA256); err != nil { + t.Fatalf("delete blob: %v", err) + } + + // One attempt, so the first failure is the permanent one. + if _, err := h.queue.Enqueue(ctx, ingest.JobProbe, + ingest.ProbePayload{DocumentID: document.ID}, + jobs.EnqueueOptions{MaxAttempts: 1}); err != nil { + t.Fatalf("enqueue: %v", err) + } + if _, err := h.pool.RunOnce(ctx); err != nil { + t.Fatalf("run job: %v", err) + } + + after, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if after.State != registry.StateFailed { + t.Errorf("state = %q, want %q — a document whose probe gave up must say so", + after.State, registry.StateFailed) + } + if after.LastError == "" { + t.Error("the document carries no explanation of why it failed") + } +} + +func TestAProbeThatWillRetryDoesNotMarkTheDocumentFailed(t *testing.T) { + // The converse: while attempts remain, the document must not be declared + // failed. Doing so would flash an error at the user for a job that is about to + // succeed on its next attempt. + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", testpdf.TaggedSections([]string{"EN"}, 2, false)) + if err := h.store.Delete(document.BlobSHA256); err != nil { + t.Fatalf("delete blob: %v", err) + } + + if _, err := h.queue.Enqueue(ctx, ingest.JobProbe, + ingest.ProbePayload{DocumentID: document.ID}, + jobs.EnqueueOptions{MaxAttempts: 3}); err != nil { + t.Fatalf("enqueue: %v", err) + } + if _, err := h.pool.RunOnce(ctx); err != nil { + t.Fatalf("run job: %v", err) + } + + after, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if after.State == registry.StateFailed { + t.Errorf("document was declared failed on attempt 1 of 3") + } +} + +func TestDecliningKeepsTheDocument(t *testing.T) { + // Declining is a decision about processing, never about storage. + h := newHarness(t, []string{"en"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", testpdf.TaggedSections([]string{"FR"}, 3, false)) + h.runProbe(t, document.ID) + + if err := h.ingest.Decline(ctx, document.ID); err != nil { + t.Fatalf("decline: %v", err) + } + + after, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if after.State != registry.StateDeclined { + t.Errorf("state = %q, want %q", after.State, registry.StateDeclined) + } + if !h.store.Exists(after.BlobSHA256) { + t.Error("declining deleted the original; the upload must be kept regardless") + } +} diff --git a/internal/ingest/neutral_internal_test.go b/internal/ingest/neutral_internal_test.go new file mode 100644 index 0000000..abd2441 --- /dev/null +++ b/internal/ingest/neutral_internal_test.go @@ -0,0 +1,138 @@ +package ingest + +import ( + "testing" + + "github.com/gordon2/manualbox/internal/registry" +) + +// neutralScope is reached directly here because the states worth pinning are ones a +// generated document cannot produce on demand: a picture census that covered only +// some of the offered pages needs pdftocairo to fail on exactly one page. + +func figs(n int) *int { return &n } + +func region(page int, lang, code string, chars int) registry.Region { + return registry.Region{Page: page, Lang: lang, Code: code, Chars: chars} +} + +// TestAPartialCensusIsWithheldRatherThanSummed is the rule that keeps the offer from +// understating itself. +// +// An understated picture count is not a small error here: it is the one that makes a +// user decline the pages worth taking, which is the entire failure this feature +// exists to prevent. So a sum over a census that missed a page is not shown at all, +// and the panel says the pictures were not counted instead. +func TestAPartialCensusIsWithheldRatherThanSummed(t *testing.T) { + regions := []registry.Region{ + region(1, "", "", 10), + region(2, "", "", 10), + region(3, "de", "DE", 900), + } + pages := []registry.PageFact{ + {Page: 1, Chars: 10, Figures: figs(31)}, + // The second offered page went uncounted: pdftocairo failed on it alone. + {Page: 2, Chars: 900, Figures: nil}, + {Page: 3, Chars: 900}, + } + + got := neutralScope(regions, pages, false) + if got == nil { + t.Fatal("nothing was offered; pages 1 and 2 belong to no language") + } + if len(got.Pages) != 2 { + t.Fatalf("offered %v, want pages 1 and 2", got.Pages) + } + if got.Figures != nil { + t.Errorf("figures = %d over a census that covered 1 of 2 pages; a partial "+ + "total understates the set and must be withheld", *got.Figures) + } + if got.Note == "" { + t.Error("the pictures were withheld and nothing says why") + } +} + +// TestACompleteCensusOfZeroIsReportedAsZero is the other side, and the reason the +// field is a pointer. Zero is the columns manual's real, useful answer — its two +// unowned pages are a print code and a page of service addresses — and a user is +// entitled to see it rather than be told nothing is known. +func TestACompleteCensusOfZeroIsReportedAsZero(t *testing.T) { + regions := []registry.Region{region(67, "", "", 67), region(68, "", "", 10715)} + pages := []registry.PageFact{ + {Page: 67, Chars: 67, Figures: figs(0)}, + {Page: 68, Chars: 11189, Figures: figs(0)}, + } + + got := neutralScope(regions, pages, false) + if got == nil { + t.Fatal("nothing was offered") + } + if got.Figures == nil { + t.Fatal("a complete census of zero came back as 'not counted'") + } + if *got.Figures != 0 { + t.Errorf("figures = %d, want 0", *got.Figures) + } + if got.Note != "" { + t.Errorf("note = %q; the census was complete", got.Note) + } + // Measured on the real columns manual: the gate takes the region's own character + // count where there is one, because that is the measurement the rest of the screen + // uses. 67 + 10,715, not 67 + 11,189. + if got.Chars != 10782 { + t.Errorf("chars = %d, want 10,782 from the regions", got.Chars) + } +} + +// TestNoContentRangeIsAppliedToTheOffer is the difference from UnlabelledPages that a +// reader of this file is most likely to "fix" by accident. +// +// That count excludes front matter deliberately, because it answers "how much would a +// detector add?" and front matter legitimately belongs to no section. This answers +// "what can a reader not reach?", and front matter is exactly where the unreachable +// diagram is. Applying a content range here removes the whole feature. +func TestNoContentRangeIsAppliedToTheOffer(t *testing.T) { + regions := []registry.Region{ + region(5, "", "", 189), // the plate, in front matter + region(7, "de", "DE", 900), + region(560, "", "", 445), // the colophon, past the end + } + pages := []registry.PageFact{ + {Page: 5, Chars: 348, Figures: figs(31)}, + {Page: 7, Chars: 900}, + {Page: 560, Chars: 450, Figures: figs(2)}, + } + + got := neutralScope(regions, pages, false) + if got == nil { + t.Fatal("nothing was offered") + } + if len(got.Pages) != 2 || got.Pages[0] != 5 || got.Pages[1] != 560 { + t.Fatalf("offered %v, want the front-matter plate and the colophon", got.Pages) + } +} + +// TestTheStoredDecisionIsEchoedBack keeps a reloaded gate honest about what was +// actually approved. +func TestTheStoredDecisionIsEchoedBack(t *testing.T) { + regions := []registry.Region{region(1, "", "", 10), region(2, "de", "DE", 900)} + pages := []registry.PageFact{{Page: 1, Chars: 10, Figures: figs(4)}, {Page: 2, Chars: 900}} + + if got := neutralScope(regions, pages, true); got == nil || !got.Included { + t.Error("a document approved with these pages does not report them as included") + } + if got := neutralScope(regions, pages, false); got == nil || got.Included { + t.Error("a document that was never approved reports them as included") + } +} + +// TestNoRegionsMeansNoOffer pins that the question is asked of the region map. A +// document probed without pdftohtml has a complete per-page language map and no +// coordinates, and offering a set built from the other source under the same name +// would be a different answer to a different question. +func TestNoRegionsMeansNoOffer(t *testing.T) { + pages := []registry.PageFact{{Page: 1, Chars: 900}, {Page: 2, Chars: 900}} + if got := neutralScope(nil, pages, false); got != nil { + t.Errorf("offered %v with no regions stored", got.Pages) + } +} diff --git a/internal/ingest/neutral_test.go b/internal/ingest/neutral_test.go new file mode 100644 index 0000000..a40593d --- /dev/null +++ b/internal/ingest/neutral_test.go @@ -0,0 +1,297 @@ +package ingest_test + +import ( + "context" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/ingest" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// plateManual is the shape of the defect, generated rather than fetched: two pages +// of front matter that no language owns, then the language sections. +// +// THE TWO PAGES ARE DELIBERATELY DIFFERENT, because they cover the two arms of +// doc.CarriesContent and one of them cannot cover the other: +// +// - PAGE 1 is the plate: four drawings and seven characters of part numbers. It +// is far under MinTextChars, so only the picture arm can offer it — which is the +// real case, since a diagram plate's labels are inside its drawings. +// - PAGE 2 is prose with no language code and no drawings. Only the text arm can +// offer it, AND it is the page that makes "no language is invented for them" +// testable: it holds enough text to become blocks, so a change that let an +// unowned region into a household's scope would show up as blocks on page 2. +// +// A single page cannot do both jobs. The first version of this used only the plate, +// and a mutation that gave unowned regions the household's language survived the +// whole suite because seven characters produce no blocks either way. +func plateManual(codes []string) testpdf.Doc { + // 40 strokes each, which TestAGeneratedDrawingComesBackAsAFigure records as well + // over minFigureInk — a generated drawing is clean, so the shape count is the only + // thing standing between it and the ink guard. + plate := testpdf.Page{ + Lines: []string{"A-1", "C-5"}, + Drawings: []testpdf.Drawing{ + {X: 70, Y: 480, W: 190, H: 170, Strokes: 40}, + {X: 340, Y: 480, W: 190, H: 170, Strokes: 40}, + {X: 70, Y: 180, W: 190, H: 170, Strokes: 40}, + {X: 340, Y: 180, W: 190, H: 170, Strokes: 40}, + }, + } + + // Twelve lines, because a page holding a single text run gets no region at all and + // so would never be a candidate. See unownedPage in internal/doc. + lines := make([]string, 0, 12) + for i := range 12 { + lines = append(lines, + strings.Repeat("Maintenance information for this appliance. ", 2)+string(rune('a'+i))) + } + prose := testpdf.Page{Lines: lines} + + d := testpdf.TaggedSections(codes, 3, false) + d.Pages = append([]testpdf.Page{plate, prose}, d.Pages...) + return d +} + +func requireCairo(t *testing.T) { + t.Helper() + for _, tool := range []extern.Tool{extern.PDFToHTML, extern.PDFToCairo} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } +} + +// TestTheGateOffersThePagesNoLanguageOwns is the gate half: the extra scope is +// reported, it is reported from stored rows, and it says what is on the pages. +func TestTheGateOffersThePagesNoLanguageOwns(t *testing.T) { + requireCairo(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", plateManual([]string{"EN", "DE", "FR"})) + h.runProbe(t, document.ID) + + g, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + if g.Neutral == nil { + t.Fatal("the gate offered no extra scope; page 1 is a plate of drawings that " + + "no language section contains") + } + if len(g.Neutral.Pages) != 2 || g.Neutral.Pages[0] != 1 || g.Neutral.Pages[1] != 2 { + t.Errorf("offered pages = %v, want the plate on 1 and the prose on 2", g.Neutral.Pages) + } + // The picture count is the field that makes the offer worth reading — see + // GateNeutral — so its absence is a failure and not a detail. + if g.Neutral.Figures == nil { + t.Fatalf("the pictures were not counted: %q", g.Neutral.Note) + } + if *g.Neutral.Figures != 4 { + t.Errorf("figures on the offered pages = %d, want the plate's 4", *g.Neutral.Figures) + } + if g.Neutral.Included { + t.Error("Included is true before anything was approved") + } + + // Read from stored rows and not from a fresh read of the document, which is the + // gate's contract. Asserted by asking the same question of the registry: the + // figure census has to be on doc_pages, or the gate could not have answered. + pages, err := h.registry.Pages(ctx, document.ID) + if err != nil { + t.Fatalf("pages: %v", err) + } + stored, counted := 0, 0 + for i := range pages { + if pages[i].Figures != nil { + counted++ + stored += *pages[i].Figures + } + } + if counted != 2 || stored != 4 { + t.Errorf("doc_pages holds a census of %d page(s) totalling %d figures; the gate's "+ + "answer must come from there, over both offered pages", counted, stored) + } +} + +// TestTheGateDoesNotOfferAPageALanguageOwns is the negative, and it is the one that +// keeps the offer honest: a document whose every page is claimed has no second +// scope, so the UI must have nothing to draw rather than an empty box. +func TestTheGateDoesNotOfferAPageALanguageOwns(t *testing.T) { + requireCairo(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE", "FR"}, 3, false)) + h.runProbe(t, document.ID) + + g, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + if g.Neutral != nil { + t.Errorf("the gate offered %v; every page of this document carries its own "+ + "language tag", g.Neutral.Pages) + } +} + +// TestApprovingWithTheExtraPagesConvertsThem is the whole path through the seam the +// brief was most careful about: the choice arrives as one boolean, is stored on the +// document, is read back by the job from stored state, and the pictures land. +func TestApprovingWithTheExtraPagesConvertsThem(t *testing.T) { + requireCairo(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", plateManual([]string{"EN", "DE", "FR"})) + h.runProbe(t, document.ID) + + if _, err := h.ingest.Approve(ctx, document.ID, + ingest.ApproveScope{IncludeNeutralPages: true}); err != nil { + t.Fatalf("approve: %v", err) + } + + // The decision is on the row before the job runs. If it were in the payload + // instead, the dedupe key being the document would let an approval with the pages + // and one without collapse onto whichever was queued first. + approved, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if !approved.IncludeNeutralPages { + t.Fatal("the approved scope was not recorded on the document") + } + if approved.State != registry.StateConverting { + t.Fatalf("state = %q, want %q", approved.State, registry.StateConverting) + } + + if ran, err := h.pool.RunOnce(ctx); err != nil || !ran { + t.Fatalf("run convert job: ran=%t err=%v", ran, err) + } + final, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if final.State == registry.StateFailed { + t.Fatalf("conversion failed: %s", final.LastError) + } + + // The pictures are served to German, which is the funnel's promise for a page no + // language owns: it belongs to every language in scope. Asked through + // FiguresByLang rather than Figures, because that is the call the reader makes and + // the one that has to attribute a neutral figure without a language column. + german, err := h.registry.FiguresByLang(ctx, document.ID, "de") + if err != nil { + t.Fatalf("figures for de: %v", err) + } + onPlate := 0 + for i := range german { + if german[i].Page == 1 { + onPlate++ + } + } + if onPlate == 0 { + t.Fatalf("a German reader was served %d figures and none of them is on the "+ + "plate; the diagram is still unreachable", len(german)) + } + + // AND NO LANGUAGE WAS INVENTED FOR THOSE PAGES. Their regions are unnamed, so they + // contribute pictures and no text — page 2 holds over a thousand characters of + // prose and must produce no blocks at all. A change that let an unowned region into + // the household's scope would put German blocks on a page nothing established as + // German, which is the one thing the funnel may not do. + blocks, err := h.registry.Blocks(ctx, document.ID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + for i := range blocks { + if blocks[i].Page <= 2 { + t.Errorf("page %d produced a %s block in %q: %q — a page no language owns "+ + "must contribute no text", blocks[i].Page, blocks[i].Kind, blocks[i].Lang, + blocks[i].Text) + } + } + + // And the gate re-rendered after approval shows what was actually chosen, so a + // user reloading the screen does not see an unticked box over converted pages. + g, err := h.ingest.Gate(ctx, document.ID) + if err != nil { + t.Fatalf("gate: %v", err) + } + if g.Neutral == nil || !g.Neutral.Included { + t.Error("the gate does not report that the extra pages were included") + } +} + +// TestApprovingWithoutTheExtraPagesLeavesThemOut is the constraint stated as a +// test: nothing changes for a household that does not opt in. +func TestApprovingWithoutTheExtraPagesLeavesThemOut(t *testing.T) { + requireCairo(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + document := h.upload(t, "manual.pdf", plateManual([]string{"EN", "DE", "FR"})) + h.runProbe(t, document.ID) + h.approveAndConvert(t, document.ID) + + stored, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if stored.IncludeNeutralPages { + t.Error("the flag was set by an approval that did not ask for it") + } + + figures, err := h.registry.Figures(ctx, document.ID) + if err != nil { + t.Fatalf("figures: %v", err) + } + for i := range figures { + if figures[i].Page <= 2 { + t.Fatalf("the front matter on page %d was converted without being asked for", + figures[i].Page) + } + } + + blocks, err := h.registry.Blocks(ctx, document.ID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + for i := range blocks { + if blocks[i].Page <= 2 { + t.Errorf("page %d produced a block %q without being asked for", + blocks[i].Page, blocks[i].Text) + } + } +} + +// TestAYesToAnOfferThatWasNotMadeConvertsNothingExtra pins the narrowing in +// Approve. A client that always sends the flag must not be able to put a document +// into a scope its own gate had nothing to offer for. +func TestAYesToAnOfferThatWasNotMadeConvertsNothingExtra(t *testing.T) { + requireCairo(t) + h := newHarness(t, []string{"de"}) + ctx := context.Background() + + // Every page tagged, so the gate offers nothing. + document := h.upload(t, "manual.pdf", + testpdf.TaggedSections([]string{"EN", "DE", "FR"}, 3, false)) + h.runProbe(t, document.ID) + + if _, err := h.ingest.Approve(ctx, document.ID, + ingest.ApproveScope{IncludeNeutralPages: true}); err != nil { + t.Fatalf("approve: %v", err) + } + stored, err := h.registry.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if stored.IncludeNeutralPages { + t.Error("a yes was recorded against a document whose gate offered nothing") + } +} diff --git a/internal/jobs/worker.go b/internal/jobs/worker.go index 2bbb969..3feefe5 100644 --- a/internal/jobs/worker.go +++ b/internal/jobs/worker.go @@ -129,6 +129,25 @@ func (p *Pool) Run(ctx context.Context) error { return nil } +// RunOnce claims and runs a single job, reporting whether there was one to run. +// +// It exists so a caller can drive the queue deterministically instead of starting +// workers and waiting for them, which is what makes pipeline tests assert on a +// finished job rather than on a timeout. It goes through the same claim, lease and +// completion path as a real worker, so what it exercises is the real thing — +// including that a handler is idempotent when the job is run again. +func (p *Pool) RunOnce(ctx context.Context) (bool, error) { + job, err := p.claim(ctx, "run-once") + if err != nil { + return false, err + } + if job == nil { + return false, nil + } + p.execute(ctx, p.log, job) + return true, nil +} + // worker claims and runs jobs until ctx is cancelled. func (p *Pool) worker(ctx context.Context, name string) { log := p.log.With("worker", name) diff --git a/internal/registry/conversion.go b/internal/registry/conversion.go new file mode 100644 index 0000000..c9ae55b --- /dev/null +++ b/internal/registry/conversion.go @@ -0,0 +1,583 @@ +package registry + +import ( + "bytes" + "context" + "database/sql" + "fmt" + + "github.com/gordon2/manualbox/internal/db" + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/store" +) + +// figureMediaType is what a rendered figure is. doc.renderFigure calls pdftoppm +// with -png and verifies the signature and IHDR before returning, so this is +// checked rather than assumed. +const figureMediaType = "image/png" + +// SaveConversion records everything a conversion produced, in one transaction. +// +// All of it or none of it, for [Service.SaveProbe]'s reason one stage further on: +// a document whose row says 'ready' but whose blocks are missing looks converted +// and reads as an empty manual. The state moves in the same transaction as the +// content that justifies it. +// +// The write is idempotent, because a worker can die after doing the work and have +// the job run again. Blocks and figures are keyed naturally and upserted, and both +// are replaced wholesale first -- see [saveBlocks] for why the delete is required +// rather than tidy. +// +// # Why the blob store is a parameter +// +// A figure's bytes are content like any other, so they go to the content-addressed +// store on disk and the row holds the digest, exactly as an uploaded document +// does. [Service] does not hold a store because nothing else in the registry needs +// one; internal/ingest, which is the caller, already has it. +// +// # Why the bytes are written before the transaction opens +// +// A store.Put is a filesystem write and cannot be rolled back with the rows. Doing +// it first means a rolled-back conversion can leave PNG bytes on disk with no row +// pointing at them, and that is the harmless direction: the store is content +// addressed, so the retry writes the same digest and reuses them rather than +// duplicating. The other order -- rows first, bytes after -- would leave a row +// pointing at a picture that does not exist, which is a broken reader. +// +// It also has to be this way round. blobs(sha256) is a foreign key and SQLite +// checks it immediately, so the blobs row must exist inside the transaction before +// the doc_figures row that references it. +func (s *Service) SaveConversion( + ctx context.Context, + documentID string, + blocks []doc.Block, + figures []doc.Figure, + blobs *store.Store, + state string, +) error { + if documentID == "" { + return fmt.Errorf("%w: a conversion needs a document", ErrInvalid) + } + if len(figures) > 0 && blobs == nil { + return fmt.Errorf("%w: %d figures to store and no blob store to put them in", + ErrInvalid, len(figures)) + } + + // Outside the transaction, for the reason above. The refs come back in a + // parallel slice rather than being written into the caller's figures: filling + // in a field of someone else's struct as a side effect of saving it is the kind + // of thing a caller discovers by having it happen. + refs, err := putFigures(ctx, blobs, figures) + if err != nil { + return err + } + + now := db.Millis(s.now()) + return s.db.Tx(ctx, func(tx *sql.Tx) error { + q := gen.New(tx) + + if err := saveBlocks(ctx, q, documentID, blocks, now); err != nil { + return err + } + if err := saveFigures(ctx, q, documentID, figures, refs, now); err != nil { + return err + } + + // Last, so that a document only claims to be converted once its content is + // in the same transaction as the claim. + if err := q.SetDocumentState(ctx, gen.SetDocumentStateParams{ + State: state, + LastError: "", + UpdatedAt: now, + ID: documentID, + }); err != nil { + return fmt.Errorf("set state %s: %w", state, err) + } + return nil + }) +} + +// putFigures writes each figure's PNG to the blob store and returns the refs in +// the same order, checking as it goes that the digest internal/doc computed is the +// name the store gave the bytes. +// +// The digest is verified rather than trusted, and it is a string comparison +// against an expensive mistake: doc.Figure.Digest is the SHA-256 of exactly what +// pdftoppm wrote, and if it ever disagreed with the store's own then the row would +// describe one picture and point at another. store.Put returns the digest it +// actually used, so the check is free. +// +// A figure with no bytes is rejected rather than stored with an empty digest. That +// is what a caller gets from doc.FindFigures, which is pure geometry and renders +// nothing, where it meant doc.PageFigures -- an easy substitution to make and an +// impossible one to notice afterwards, because the row would be complete in every +// respect except the picture. +func putFigures(ctx context.Context, blobs *store.Store, figures []doc.Figure) ([]store.Ref, error) { + refs := make([]store.Ref, len(figures)) + for i := range figures { + fig := &figures[i] + if len(fig.PNG) == 0 { + return nil, fmt.Errorf("%w: the figure at index %d on page %d has no bytes; it was "+ + "found but never rendered", ErrInvalid, fig.Index, fig.Page) + } + ref, err := blobs.Put(ctx, bytes.NewReader(fig.PNG)) + if err != nil { + return nil, fmt.Errorf("registry: store figure %d on page %d: %w", + fig.Index, fig.Page, err) + } + if fig.Digest != "" && ref.SHA256 != fig.Digest { + return nil, fmt.Errorf("registry: figure %d on page %d digests as %s but was "+ + "recorded as %s", fig.Index, fig.Page, ref.SHA256, fig.Digest) + } + refs[i] = ref + } + return refs, nil +} + +// saveBlocks replaces a document's blocks inside SaveConversion's transaction. +// +// REPLACE, NOT MERGE, and the delete is load-bearing: a re-conversion can produce +// FEWER blocks than the one before it. Indices run consecutively from 0 within a +// region, so a region that converted to 12 blocks and now converts to 9 leaves +// rows at idx 9, 10 and 11 which a reader renders as three paragraphs of the +// previous run's text, in order, indistinguishable from content. Every threshold +// in internal/doc's block builder is one measurement away from moving and most of +// them merge. See the foot of 00005_doc_blocks.sql. +// +// Unlike saveRegions there is no "the tool was missing, leave what is there" +// case. A probe can run on a host without pdftohtml and legitimately have no +// opinion about regions; a conversion that could not read the document fails and +// never reaches here, because there is no partial conversion worth storing. +func saveBlocks(ctx context.Context, q *gen.Queries, documentID string, blocks []doc.Block, now int64) error { + if err := q.DeleteDocBlocks(ctx, documentID); err != nil { + return fmt.Errorf("clear blocks: %w", err) + } + for i := range blocks { + b := &blocks[i] + if err := q.UpsertDocBlock(ctx, gen.UpsertDocBlockParams{ + DocumentID: documentID, + Page: int64(b.Page), + // The same rounding doc_regions.x0 got, from the same function, so a + // block joins to the region it was read from rather than landing one + // rounding away from the only row it can join to. + RegionX0: roundCoord(b.RegionX0), + Idx: int64(b.Index), + Kind: string(b.Kind), + Level: int64(b.Level), + Text: b.Text, + Lang: b.Lang, + X0: b.X0, + X1: b.X1, + Y0: b.Y0, + Y1: b.Y1, + Lines: int64(b.Lines), + Chars: int64(b.Chars), + Note: b.Note, + CreatedAt: now, + }); err != nil { + return fmt.Errorf("save block %d of the region at x %.0f on page %d: %w", + b.Index, b.RegionX0, b.Page, err) + } + } + return nil +} + +// saveFigures replaces a document's figures inside SaveConversion's transaction, +// recording each one's bytes as a blob first. +// +// The blobs row is written here rather than through [Service.RecordBlob], and that +// is not a style choice. RecordBlob uses s.db.Write(), whose pool is capped at one +// connection, and this transaction is holding it -- so calling RecordBlob from +// inside the transaction waits for a connection that only the transaction can +// release. Measured: with a 3 s context it returns "context deadline exceeded" +// after 3.00 s, and a background job passing a context with no deadline would hang +// for good. The statement is the same one; only the handle differs. +// +// Deleting a document's figure ROWS never deletes the blobs they point at. Two +// documents can legitimately render the same picture -- the same diagram in five +// languages' sections is one set of bytes -- so a blob is collected by counting +// references, which is what documents.blob_sha256 already does. +func saveFigures( + ctx context.Context, + q *gen.Queries, + documentID string, + figures []doc.Figure, + refs []store.Ref, + now int64, +) error { + if err := q.DeleteDocFigures(ctx, documentID); err != nil { + return fmt.Errorf("clear figures: %w", err) + } + // The labels too, explicitly. Deleting the figures cascades to them, and relying on + // that alone would leave the labels of any figure whose row the delete did not + // remove -- which is every figure, once the upserts below have run. Clearing them + // here means the loop can insert without reasoning about what survived. + if err := q.DeleteDocFigureLabels(ctx, documentID); err != nil { + return fmt.Errorf("clear figure labels: %w", err) + } + for i := range figures { + f := &figures[i] + if err := q.UpsertBlob(ctx, gen.UpsertBlobParams{ + Sha256: refs[i].SHA256, + SizeBytes: refs[i].Size, + MediaType: figureMediaType, + CreatedAt: now, + }); err != nil { + return fmt.Errorf("record figure %d on page %d as a blob: %w", f.Index, f.Page, err) + } + if err := q.UpsertDocFigure(ctx, gen.UpsertDocFigureParams{ + DocumentID: documentID, + Page: int64(f.Page), + Idx: int64(f.Index), + X0: f.Rect.X0, + Y0: f.Rect.Y0, + X1: f.Rect.X1, + Y1: f.Rect.Y1, + Ink: int64(f.Ink), + TextFraction: f.TextFraction, + Dpi: int64(f.DPI), + PixelWidth: int64(f.PixelWidth), + PixelHeight: int64(f.PixelHeight), + BlobSha256: refs[i].SHA256, + CreatedAt: now, + }); err != nil { + return fmt.Errorf("save figure %d on page %d: %w", f.Index, f.Page, err) + } + // After the figure, never before: the labels carry a composite foreign key onto + // its row, so inserting one first fails on a key that does not exist yet. + // The slice index IS the stored idx, so the order doc.figureLabels sorted into + // -- down the page, then across it -- is the order a reader reads them back in. + for j, text := range f.Labels { + if err := q.UpsertDocFigureLabel(ctx, gen.UpsertDocFigureLabelParams{ + DocumentID: documentID, + Page: int64(f.Page), + FigureIdx: int64(f.Index), + Idx: int64(j), + Text: text, + CreatedAt: now, + }); err != nil { + return fmt.Errorf("save label %d of figure %d on page %d: %w", + j, f.Index, f.Page, err) + } + } + } + return nil +} + +// Block is one stored piece of readable content. +// +// RegionX0 is an integer here because that is what is stored and what doc_regions +// stores, so it is the value a caller joins on. The block's own box is float64, +// unrounded, because nothing keys on it: a caller drawing the block on a +// pdftoppm -r 108 render wants what internal/doc measured. +type Block struct { + Page int `json:"page"` + RegionX0 int `json:"regionX0"` + Index int `json:"index"` + Kind string `json:"kind"` + // Level is the heading level, 1 for the most prominent, and 0 for anything + // that is not a heading. + Level int `json:"level,omitempty"` + Text string `json:"text"` + // Lang is the region's language, empty where none was established, and Name is + // that for a person to read: the UI shows "Ukrainian", not "uk". + Lang string `json:"lang,omitempty"` + Name string `json:"name,omitempty"` + + X0 float64 `json:"x0"` + X1 float64 `json:"x1"` + Y0 float64 `json:"y0"` + Y1 float64 `json:"y1"` + + Lines int `json:"lines"` + Chars int `json:"chars"` + Note string `json:"note,omitempty"` +} + +// Blocks returns a document's readable content in reading order: down the pages, +// then left to right across each, then in order within a region. +// +// Empty is not the same claim as absent, for [Service.Regions]' reason: a document +// that has not been converted has no blocks, and that is not the claim that it has +// no content. [Document.State] is what distinguishes them. +func (s *Service) Blocks(ctx context.Context, documentID string) ([]Block, error) { + rows, err := gen.New(s.db.Read()).ListDocBlocks(ctx, documentID) + if err != nil { + return nil, fmt.Errorf("registry: list blocks: %w", err) + } + return blocksFrom(rows), nil +} + +// BlocksByLang returns one language's content, which is the funnel's own query: a +// household that reads German gets the German column of each page rather than the +// page, measured in conversion.md as a fifth of the work. +// +// Passing "" asks for the blocks whose language was never established. Those are +// returned by no other language's call, so this is how they stay reachable rather +// than becoming invisible. +func (s *Service) BlocksByLang(ctx context.Context, documentID, lang string) ([]Block, error) { + rows, err := gen.New(s.db.Read()).ListDocBlocksByLang(ctx, gen.ListDocBlocksByLangParams{ + DocumentID: documentID, + Lang: lang, + }) + if err != nil { + return nil, fmt.Errorf("registry: list blocks for %q: %w", lang, err) + } + return blocksFrom(rows), nil +} + +// BlocksForPage returns one page's blocks, in reading order across its regions. +func (s *Service) BlocksForPage(ctx context.Context, documentID string, page int) ([]Block, error) { + rows, err := gen.New(s.db.Read()).ListDocBlocksForPage(ctx, gen.ListDocBlocksForPageParams{ + DocumentID: documentID, + Page: int64(page), + }) + if err != nil { + return nil, fmt.Errorf("registry: list blocks on page %d: %w", page, err) + } + return blocksFrom(rows), nil +} + +func blocksFrom(rows []gen.DocBlock) []Block { + out := make([]Block, 0, len(rows)) + for i := range rows { + r := &rows[i] + out = append(out, Block{ + Page: int(r.Page), + RegionX0: int(r.RegionX0), + Index: int(r.Idx), + Kind: r.Kind, + Level: int(r.Level), + Text: r.Text, + Lang: r.Lang, + Name: doc.DisplayName(r.Lang), + X0: r.X0, X1: r.X1, Y0: r.Y0, Y1: r.Y1, + Lines: int(r.Lines), + Chars: int(r.Chars), + Note: r.Note, + }) + } + return out +} + +// Figure is one stored illustration. +// +// There is no language field, and that is the contract rather than an omission: a +// picture that belongs to no language belongs to every language, so a reader +// scoped to German selects the figures of the pages German occupies instead of +// selecting figures by language. +type Figure struct { + Page int `json:"page"` + Index int `json:"index"` + + X0 float64 `json:"x0"` + Y0 float64 `json:"y0"` + X1 float64 `json:"x1"` + Y1 float64 `json:"y1"` + + // Ink is how many drawn shapes the figure holds and TextFraction how much of + // its area is covered by text: the shape guard's evidence and the text guard's, + // kept rather than reduced to the verdict. + Ink int `json:"ink"` + TextFraction float64 `json:"textFraction"` + + DPI int `json:"dpi"` + PixelWidth int `json:"pixelWidth"` + PixelHeight int `json:"pixelHeight"` + + // SHA256 is the blob store's name for the PNG, which is also the PNG's digest. + SHA256 string `json:"sha256"` + + // Labels are the figure's callout labels as printed, in the figure's own reading + // order — down the page, then across it. Empty for a picture nothing points at, + // which is most of them. + // + // These are TEXT THE CROP ALREADY CONTAINS, and that is the point. The crop is a + // band: the drawing together with every run the claim rule reaches for it, so the + // PNG prints its labels exactly where the paper does and nothing re-lays them out. + // What the strings are for is the picture's accessible description — a client that + // draws only the PNG is showing the labels, but a reader who cannot see it is not. + // That is why there is no position here any more; 00009 records the change and what + // it cost. + // + // ONE STRING IS ONE PRINTED LINE, NOT ONE LABEL, and a wrapped label therefore + // arrives as several: page 521's `Вентиляционное отверстие системы автоопорожнения` + // is three entries. That is the claim rule's own unit -- a continuation line is its + // own claim, see internal/doc's continuesLabel -- and it was invisible while the + // reader placed each line against the drawing, because the paper's own arrangement + // put them back together. It is visible now: an alt text reads them as three labels + // separated by semicolons. Joining a chain back up is a real improvement and is not + // done here, because which lines continue which is internal/doc's answer to give. + Labels []string `json:"labels,omitempty"` +} + +// Figures returns a document's illustrations in page order, each with its callout +// labels. +// +// Two queries rather than a join, and deliberately. A join returns one row per label +// and repeats every figure's fourteen columns across them, so the pixel size and the +// digest of a 34-label diagram arrive 34 times and the scan has to fold them back into +// one figure. Read separately, each row is what it says it is. The cost is bounded: the +// sequential manual's largest served language is 126 figures and 199 labels. +func (s *Service) Figures(ctx context.Context, documentID string) ([]Figure, error) { + q := gen.New(s.db.Read()) + rows, err := q.ListDocFigures(ctx, documentID) + if err != nil { + return nil, fmt.Errorf("registry: list figures: %w", err) + } + labels, err := q.ListDocFigureLabels(ctx, documentID) + if err != nil { + return nil, fmt.Errorf("registry: list figure labels: %w", err) + } + return withLabels(figuresFrom(rows), labels), nil +} + +// withLabels attaches each stored label to the figure it belongs to. +// +// Keyed on (page, figure index), which is doc_figures' natural key less the document +// every row here already shares. A label whose figure is absent is DROPPED rather than +// collected under a zero key: the foreign key makes that unreachable through this +// package, and silently inventing a figure to hang it off would turn a schema +// violation into a picture with someone else's labels on it. +// +// The order comes from the query and nowhere else. Both callers order by idx within a +// figure, so appending in row order reproduces the sequence doc.figureLabels sorted -- +// which is now the order a screen reader reads the description in, so it is load-bearing +// rather than incidental. Nothing here groups or re-sorts, and nothing here ever did: +// when a label also carried a side, the side was carried through as a field and the +// sequence was still the stored idx. +func withLabels(figures []Figure, rows []gen.DocFigureLabel) []Figure { + if len(rows) == 0 { + return figures + } + type key struct{ page, idx int } + at := make(map[key]int, len(figures)) + for i := range figures { + at[key{figures[i].Page, figures[i].Index}] = i + } + for i := range rows { + r := &rows[i] + j, ok := at[key{int(r.Page), int(r.FigureIdx)}] + if !ok { + continue + } + figures[j].Labels = append(figures[j].Labels, r.Text) + } + return figures +} + +// figureRegionSlack is how far a figure may reach past a region's edge and still +// count as inside it. The same one unit doc.Convert allows, and it has to be the +// same: this is the read-side half of the attribution that file made at write +// time, so a different tolerance would give a reader a different set of pictures +// than the conversion decided on. +const figureRegionSlack = 1.0 + +// FiguresByLang returns the illustrations belonging to one language: every figure +// inside one of that language's own regions, plus every figure inside no named +// region of its page. +// +// The second half is the rule docs/design/conversion.md settles — a picture that +// belongs to no language belongs to every language — and it is why doc_figures has +// no language column to select on. The attribution is therefore worked out here +// from the stored regions, which is the same input and the same test doc.Convert's +// `attribute` applied when it decided which figures to keep at all. +// +// Deriving it rather than storing it is what makes a mixed household correct. A +// household reading German and Ukrainian stores the union of both languages' +// figures, so asking for German by page would hand a reader the pictures out of +// the Ukrainian column of every page the two share. Asking geometrically gives +// German the 40 figures the conversion attributed to German, which is the number +// conversion.md records. +// +// Passing "" asks for the figures belonging to no named language at all, which is +// the only way those stay reachable on a document whose regions were never named. +func (s *Service) FiguresByLang(ctx context.Context, documentID, lang string) ([]Figure, error) { + figures, err := s.Figures(ctx, documentID) + if err != nil { + return nil, err + } + regions, err := s.Regions(ctx, documentID) + if err != nil { + return nil, err + } + + byPage := make(map[int][]int, len(regions)) + for i := range regions { + byPage[regions[i].Page] = append(byPage[regions[i].Page], i) + } + + want := doc.BaseLanguage(lang) + out := make([]Figure, 0, len(figures)) + for i := range figures { + f := &figures[i] + // The neutral case is the second arm, and it is the whole rule: a reader of + // German gets the 2 figures inside a German column AND the 38 belonging to no + // column, which is the 40 conversion.md records. Matching only the first arm + // would leave a reader of a parallel-columns manual with almost no pictures. + if got := figureLang(f, regions, byPage[f.Page]); got == want || got == "" { + out = append(out, *f) + } + } + return out, nil +} + +// figureLang names the language a figure sits inside, or "" for one sitting inside +// no named region of its page — which includes a figure straddling two of them and +// a figure on a page whose regions nothing could name. +func figureLang(f *Figure, regions []Region, onPage []int) string { + for _, i := range onPage { + r := ®ions[i] + if f.X0 < float64(r.X0)-figureRegionSlack || f.X1 > float64(r.X1)+figureRegionSlack { + continue + } + // An unnamed region is not a language, so a figure inside one has none + // either and falls through to the neutral rule. That is the stance + // everywhere: an unnamed region is a reportable state, never a language. + if base := doc.BaseLanguage(r.Lang); base != "" { + return base + } + } + return "" +} + +// FiguresForPage returns one page's illustrations in reading order, each with its +// callout labels. +func (s *Service) FiguresForPage(ctx context.Context, documentID string, page int) ([]Figure, error) { + q := gen.New(s.db.Read()) + rows, err := q.ListDocFiguresForPage(ctx, gen.ListDocFiguresForPageParams{ + DocumentID: documentID, + Page: int64(page), + }) + if err != nil { + return nil, fmt.Errorf("registry: list figures on page %d: %w", page, err) + } + labels, err := q.ListDocFigureLabelsForPage(ctx, gen.ListDocFigureLabelsForPageParams{ + DocumentID: documentID, + Page: int64(page), + }) + if err != nil { + return nil, fmt.Errorf("registry: list figure labels on page %d: %w", page, err) + } + return withLabels(figuresFrom(rows), labels), nil +} + +func figuresFrom(rows []gen.DocFigure) []Figure { + out := make([]Figure, 0, len(rows)) + for i := range rows { + r := &rows[i] + out = append(out, Figure{ + Page: int(r.Page), + Index: int(r.Idx), + X0: r.X0, Y0: r.Y0, X1: r.X1, Y1: r.Y1, + Ink: int(r.Ink), + TextFraction: r.TextFraction, + DPI: int(r.Dpi), + PixelWidth: int(r.PixelWidth), + PixelHeight: int(r.PixelHeight), + SHA256: r.BlobSha256, + }) + } + return out +} diff --git a/internal/registry/conversion_test.go b/internal/registry/conversion_test.go new file mode 100644 index 0000000..8558918 --- /dev/null +++ b/internal/registry/conversion_test.go @@ -0,0 +1,663 @@ +package registry_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/color" + "image/png" + "path/filepath" + "reflect" + "testing" + + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// newBlobStore is a blob store in a temp directory. Figures are the first thing in +// the pipeline that writes derived bytes there, so every test here needs one. +func newBlobStore(t *testing.T) *store.Store { + t.Helper() + blobs, err := store.New(filepath.Join(t.TempDir(), "blobs")) + if err != nil { + t.Fatalf("open blob store: %v", err) + } + return blobs +} + +// pngBytes makes a real PNG in memory, because no image may be committed to this +// repository. A distinct size gives distinct bytes and therefore a distinct +// digest, which is what lets a test tell two figures apart by their content. +func pngBytes(t *testing.T, w, h int) []byte { + t.Helper() + img := image.NewGray(image.Rect(0, 0, w, h)) + img.Set(0, 0, color.Gray{Y: 255}) + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatalf("encode png: %v", err) + } + return buf.Bytes() +} + +// figure builds a rendered figure the way doc.PageFigures returns one: bytes, +// pixel size and the digest of exactly those bytes. +func figure(t *testing.T, page, index, w, h int) doc.Figure { + t.Helper() + raw := pngBytes(t, w, h) + sum := sha256.Sum256(raw) + return doc.Figure{ + Page: page, Index: index, + Rect: doc.CellRect{X0: 43.5, Y0: 200.25, X1: 300.75, Y1: 460.5}, + Ink: 42, + TextFraction: 0.0234, + DPI: 216, + PixelWidth: w, PixelHeight: h, + Digest: hex.EncodeToString(sum[:]), + PNG: raw, + } +} + +func TestConversionRoundTrip(t *testing.T) { + // Every field, because a block that survives with the wrong box or the wrong + // character count is worse than one that fails to save: it reads as + // authoritative. The fractional coordinates are the point of half of this -- + // region_x0 is rounded on the way in because it is in the primary key, and the + // block's own box is not rounded at all because nothing keys on it. + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "a1") + + blocks := []doc.Block{ + { + Page: 62, RegionX0: 42.5, Index: 0, + Kind: doc.BlockHeading, Level: 1, + Text: "Hinweis zur Entsorgung", Lang: "de", + X0: 43.2, X1: 300.8, Y0: 100.5, Y1: 118.5, + Lines: 1, Chars: 22, + Note: "18pt bold at 22 characters, 0.29 of the measure", + }, + { + Page: 62, RegionX0: 42.5, Index: 1, + Kind: doc.BlockParagraph, + Text: "Die Verpackung schuetzt das Geraet vor Transportschaeden.", Lang: "de", + X0: 43.2, X1: 304.9, Y0: 122.0, Y1: 170.75, + Lines: 3, Chars: 57, + }, + { + Page: 62, RegionX0: 42.5, Index: 2, + Kind: doc.BlockListItem, + Text: "Typenbezeichnung: 788/M", Lang: "de", + X0: 43.2, X1: 250.1, Y0: 490.0, Y1: 506.0, + Lines: 1, Chars: 23, + Note: "opens with a marker", + }, + { + Page: 57, RegionX0: 0, Index: 0, + Kind: doc.BlockTable, + Text: "Spannungsversorgung", Lang: "de", + X0: 29.7, X1: 173.3, Y0: 210.0, Y1: 226.0, + Lines: 1, Chars: 19, + Note: "row 1, column 1 of 2", + }, + } + figures := []doc.Figure{figure(t, 57, 0, 12, 9)} + + if err := s.SaveConversion(ctx, docID, blocks, figures, blobs, registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + + got, err := s.Blocks(ctx, docID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + want := []registry.Block{ + // Page 57 sorts before page 62; within page 62 the region at x0 43 holds + // three blocks in index order. + {Page: 57, RegionX0: 0, Index: 0, Kind: "table", Text: "Spannungsversorgung", + Lang: "de", Name: "German", X0: 29.7, X1: 173.3, Y0: 210.0, Y1: 226.0, + Lines: 1, Chars: 19, Note: "row 1, column 1 of 2"}, + // 42.5 rounds to 43 rather than truncating to 42, which is what makes this + // value the same one doc_regions stores for the same column. + {Page: 62, RegionX0: 43, Index: 0, Kind: "heading", Level: 1, + Text: "Hinweis zur Entsorgung", Lang: "de", Name: "German", + X0: 43.2, X1: 300.8, Y0: 100.5, Y1: 118.5, Lines: 1, Chars: 22, + Note: "18pt bold at 22 characters, 0.29 of the measure"}, + {Page: 62, RegionX0: 43, Index: 1, Kind: "paragraph", + Text: "Die Verpackung schuetzt das Geraet vor Transportschaeden.", + Lang: "de", Name: "German", X0: 43.2, X1: 304.9, Y0: 122.0, Y1: 170.75, + Lines: 3, Chars: 57}, + {Page: 62, RegionX0: 43, Index: 2, Kind: "list-item", + Text: "Typenbezeichnung: 788/M", Lang: "de", Name: "German", + X0: 43.2, X1: 250.1, Y0: 490.0, Y1: 506.0, Lines: 1, Chars: 23, + Note: "opens with a marker"}, + } + if len(got) != len(want) { + t.Fatalf("got %d blocks, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("block %d:\n got %+v\nwant %+v", i, got[i], want[i]) + } + } + + gotFigs, err := s.Figures(ctx, docID) + if err != nil { + t.Fatalf("figures: %v", err) + } + wantFigs := []registry.Figure{{ + Page: 57, Index: 0, + X0: 43.5, Y0: 200.25, X1: 300.75, Y1: 460.5, + Ink: 42, TextFraction: 0.0234, + DPI: 216, PixelWidth: 12, PixelHeight: 9, + SHA256: figures[0].Digest, + }} + if len(gotFigs) != 1 { + t.Fatalf("got %d figures, want 1: %+v", len(gotFigs), gotFigs) + } + // reflect.DeepEqual rather than ==, because a Figure now carries its callout + // labels and a slice is not comparable. This figure has none, which is the normal + // case: nothing points at most pictures. + if !reflect.DeepEqual(gotFigs[0], wantFigs[0]) { + t.Errorf("figure:\n got %+v\nwant %+v", gotFigs[0], wantFigs[0]) + } + + // The state moved in the same transaction as the content that justifies it. + document, err := s.GetDocument(ctx, docID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if document.State != registry.StateReady { + t.Errorf("state = %q, want %q", document.State, registry.StateReady) + } +} + +func TestFigureBytesLandInTheBlobStore(t *testing.T) { + // A figure's row is a digest and nothing else, so the bytes have to be + // somewhere. If they are not in the store, every reader shows a broken picture + // and no test that only reads rows back would notice. + s, database := newServiceWithDB(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "b2") + + // Two different pictures, so a store holding one set of bytes twice cannot pass. + first, second := figure(t, 11, 0, 12, 9), figure(t, 11, 1, 20, 30) + if first.Digest == second.Digest { + t.Fatal("the two figures have the same bytes; this test cannot tell them apart") + } + + if err := s.SaveConversion(ctx, docID, nil, []doc.Figure{first, second}, blobs, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + + for _, want := range []doc.Figure{first, second} { + if !blobs.Exists(want.Digest) { + t.Errorf("the blob store has no %s; the figure's bytes were not written", want.Digest[:8]) + continue + } + // Read them back and digest them again: the row must point at the bytes it + // describes, not merely at a file of the right name. + raw, err := blobs.ReadAll(want.Digest) + if err != nil { + t.Errorf("read %s: %v", want.Digest[:8], err) + continue + } + if sum := sha256.Sum256(raw); hex.EncodeToString(sum[:]) != want.Digest { + t.Errorf("the bytes stored as %s digest as something else", want.Digest[:8]) + } + if !bytes.Equal(raw, want.PNG) { + t.Errorf("the bytes stored as %s are not the figure's PNG", want.Digest[:8]) + } + } + + // And the blobs table knows about them, which is the foreign key doc_figures + // holds. A figure blob is also indexed with its real media type rather than + // inheriting the document's. + for _, want := range []doc.Figure{first, second} { + row, err := gen.New(database.Read()).GetBlob(ctx, want.Digest) + if err != nil { + t.Errorf("blobs row for %s: %v", want.Digest[:8], err) + continue + } + if row.MediaType != "image/png" { + t.Errorf("blob %s media type = %q, want image/png", want.Digest[:8], row.MediaType) + } + if row.SizeBytes != int64(len(want.PNG)) { + t.Errorf("blob %s size = %d, want %d", want.Digest[:8], row.SizeBytes, len(want.PNG)) + } + } + + // The rows reference the digests, in page reading order. + got, err := s.Figures(ctx, docID) + if err != nil { + t.Fatalf("figures: %v", err) + } + if len(got) != 2 || got[0].SHA256 != first.Digest || got[1].SHA256 != second.Digest { + t.Errorf("figure rows reference %+v; want %s and %s", + got, first.Digest[:8], second.Digest[:8]) + } +} + +func TestAFigureThatWasNeverRenderedIsRejected(t *testing.T) { + // doc.FindFigures is pure geometry and renders nothing; doc.PageFigures renders. + // Substituting one for the other is easy and the result would be undetectable + // afterwards: a complete row pointing at a blob that does not exist. + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "c3") + + found := doc.Figure{Page: 11, Index: 0, Ink: 42, + Rect: doc.CellRect{X0: 43, Y0: 200, X1: 300, Y1: 460}} + err := s.SaveConversion(ctx, docID, nil, []doc.Figure{found}, blobs, registry.StateReady) + if !errors.Is(err, registry.ErrInvalid) { + t.Fatalf("saving an unrendered figure returned %v, want ErrInvalid", err) + } + + // And nothing was written: the rejection happens before the transaction opens. + if got, err := s.Figures(ctx, docID); err != nil || len(got) != 0 { + t.Errorf("figures = %+v, %v; want none", got, err) + } +} + +func TestSavingTheSameConversionTwiceLeavesTheSameRows(t *testing.T) { + // A conversion job can run twice: a worker may die after doing the work but + // before recording success, and the reclaimed job runs again. The second run + // must converge on the same rows rather than duplicating them. Values, not + // counts -- a converging count with a corrupted row would pass a count check. + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "d4") + + blocks := []doc.Block{ + {Page: 62, RegionX0: 42.5, Index: 0, Kind: doc.BlockHeading, Level: 1, + Text: "Garantie", Lang: "de", X0: 43, X1: 200, Y0: 100, Y1: 118, + Lines: 1, Chars: 8}, + {Page: 62, RegionX0: 42.5, Index: 1, Kind: doc.BlockParagraph, + Text: "Gemaess nachstehenden Bedingungen.", Lang: "de", + X0: 43, X1: 305, Y0: 122, Y1: 170, Lines: 2, Chars: 34}, + {Page: 62, RegionX0: 463, Index: 0, Kind: doc.BlockParagraph, + Text: "Kundendienst.", Lang: "de", X0: 463, X1: 720, Y0: 100, Y1: 118, + Lines: 1, Chars: 13}, + } + figures := []doc.Figure{figure(t, 62, 0, 12, 9), figure(t, 62, 1, 20, 30)} + + if err := s.SaveConversion(ctx, docID, blocks, figures, blobs, registry.StateReady); err != nil { + t.Fatalf("save first conversion: %v", err) + } + firstBlocks, err := s.Blocks(ctx, docID) + if err != nil { + t.Fatalf("blocks after first conversion: %v", err) + } + firstFigures, err := s.Figures(ctx, docID) + if err != nil { + t.Fatalf("figures after first conversion: %v", err) + } + + if err := s.SaveConversion(ctx, docID, blocks, figures, blobs, registry.StateReady); err != nil { + t.Fatalf("save second conversion: %v", err) + } + secondBlocks, err := s.Blocks(ctx, docID) + if err != nil { + t.Fatalf("blocks after second conversion: %v", err) + } + secondFigures, err := s.Figures(ctx, docID) + if err != nil { + t.Fatalf("figures after second conversion: %v", err) + } + + if len(firstBlocks) != 3 { + t.Fatalf("first conversion stored %d blocks, want 3", len(firstBlocks)) + } + if len(secondBlocks) != len(firstBlocks) { + t.Fatalf("re-converting changed the block count from %d to %d", + len(firstBlocks), len(secondBlocks)) + } + for i := range firstBlocks { + if firstBlocks[i] != secondBlocks[i] { + t.Errorf("block %d changed on re-convert:\nfirst %+v\nsecond %+v", + i, firstBlocks[i], secondBlocks[i]) + } + } + + if len(firstFigures) != 2 || len(secondFigures) != len(firstFigures) { + t.Fatalf("re-converting changed the figure count from %d to %d", + len(firstFigures), len(secondFigures)) + } + for i := range firstFigures { + // DeepEqual reaches the callout labels too, which is the half of this that a + // second run could plausibly break: the labels are deleted and rewritten, and + // their index is their position in a sorted order rather than anything the + // page prints. + if !reflect.DeepEqual(firstFigures[i], secondFigures[i]) { + t.Errorf("figure %d changed on re-convert:\nfirst %+v\nsecond %+v", + i, firstFigures[i], secondFigures[i]) + } + } +} + +func TestAReconversionThatProducesFewerBlocksLeavesNoStaleRow(t *testing.T) { + // THIS IS THE CASE THE WHOLESALE REPLACE EXISTS FOR, and the reason + // SaveConversion deletes before inserting. + // + // Indices run consecutively from 0 within a region, so a region that converted + // to 4 blocks and now converts to 2 leaves rows at idx 2 and 3. An upsert alone + // cannot remove them, because an upsert only ever touches the keys it is given. + // What a reader then shows is two paragraphs of the PREVIOUS run's text, in + // order, at the end of the region, indistinguishable from content -- which is + // worse than a crash, because nothing reports it. + // + // It is not a hypothetical. Every threshold in internal/doc's block builder is + // one measurement away from moving and most of them merge: the paragraph gap + // factor folds two paragraphs into one, and the heading share cut turns two + // headings into one. + // + // If this test still passes with the delete removed from saveBlocks, it is not + // testing what it claims. + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "e5") + + four := []doc.Block{ + {Page: 62, RegionX0: 43, Index: 0, Kind: doc.BlockParagraph, Text: "Erstens.", + Lang: "de", X0: 43, X1: 305, Y0: 100, Y1: 116, Lines: 1, Chars: 8}, + {Page: 62, RegionX0: 43, Index: 1, Kind: doc.BlockParagraph, Text: "Zweitens.", + Lang: "de", X0: 43, X1: 305, Y0: 120, Y1: 136, Lines: 1, Chars: 9}, + {Page: 62, RegionX0: 43, Index: 2, Kind: doc.BlockParagraph, Text: "Drittens.", + Lang: "de", X0: 43, X1: 305, Y0: 140, Y1: 156, Lines: 1, Chars: 9}, + {Page: 62, RegionX0: 43, Index: 3, Kind: doc.BlockParagraph, Text: "Viertens.", + Lang: "de", X0: 43, X1: 305, Y0: 160, Y1: 176, Lines: 1, Chars: 9}, + } + if err := s.SaveConversion(ctx, docID, four, nil, blobs, registry.StateReady); err != nil { + t.Fatalf("save first conversion: %v", err) + } + + // The gap factor moved and the four paragraphs folded into two. + two := []doc.Block{ + {Page: 62, RegionX0: 43, Index: 0, Kind: doc.BlockParagraph, + Text: "Erstens. Zweitens.", Lang: "de", + X0: 43, X1: 305, Y0: 100, Y1: 136, Lines: 2, Chars: 18}, + {Page: 62, RegionX0: 43, Index: 1, Kind: doc.BlockParagraph, + Text: "Drittens. Viertens.", Lang: "de", + X0: 43, X1: 305, Y0: 140, Y1: 176, Lines: 2, Chars: 19}, + } + if err := s.SaveConversion(ctx, docID, two, nil, blobs, registry.StateReady); err != nil { + t.Fatalf("save re-conversion: %v", err) + } + + got, err := s.Blocks(ctx, docID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d blocks after a re-conversion that produced 2, want 2 -- the tail of "+ + "the previous run lingered and reads as content: %+v", len(got), got) + } + for i := range got { + if got[i].Text != two[i].Text { + t.Errorf("block %d text = %q, want %q", i, got[i].Text, two[i].Text) + } + } + + // The same, one table down: a figure the trim rule now rejects must disappear. + if err := s.SaveConversion(ctx, docID, two, + []doc.Figure{figure(t, 62, 0, 12, 9), figure(t, 62, 1, 20, 30)}, + blobs, registry.StateReady); err != nil { + t.Fatalf("save conversion with two figures: %v", err) + } + if err := s.SaveConversion(ctx, docID, two, []doc.Figure{figure(t, 62, 0, 12, 9)}, + blobs, registry.StateReady); err != nil { + t.Fatalf("save conversion with one figure: %v", err) + } + gotFigs, err := s.Figures(ctx, docID) + if err != nil { + t.Fatalf("figures: %v", err) + } + if len(gotFigs) != 1 { + t.Errorf("got %d figures after a re-conversion that produced 1, want 1 -- a rejected "+ + "figure outlived the run that rejected it: %+v", len(gotFigs), gotFigs) + } +} + +func TestTwoLanguagesOnOnePageAreKeptApartByTheirRegion(t *testing.T) { + // THIS IS THE doc_regions COLLISION, ONE LEVEL DOWN. A parallel-columns page + // sets several languages side by side, and their blocks share a page and an + // index: block 0 of the German column and block 0 of the Polish column are both + // "page 62, index 0". Only region_x0 tells them apart, which is exactly why it + // is in the key -- without it one column silently overwrites the other and the + // funnel returns half a page. + // + // The coordinates are the columns manual's real page 2 edges. + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "f6") + + blocks := []doc.Block{ + {Page: 2, RegionX0: 42.5, Index: 0, Kind: doc.BlockHeading, Level: 1, + Text: "Sicherheitshinweise", Lang: "de", + X0: 43, X1: 305, Y0: 100, Y1: 118, Lines: 1, Chars: 19}, + {Page: 2, RegionX0: 42.5, Index: 1, Kind: doc.BlockParagraph, + Text: "Lesen Sie diese Anleitung.", Lang: "de", + X0: 43, X1: 305, Y0: 122, Y1: 170, Lines: 3, Chars: 26}, + {Page: 2, RegionX0: 322.6, Index: 0, Kind: doc.BlockHeading, Level: 1, + Text: "Wskazowki bezpieczenstwa", Lang: "pl", + X0: 323, X1: 585, Y0: 100, Y1: 118, Lines: 1, Chars: 24}, + {Page: 2, RegionX0: 322.6, Index: 1, Kind: doc.BlockParagraph, + Text: "Przeczytaj te instrukcje.", Lang: "pl", + X0: 323, X1: 585, Y0: 122, Y1: 170, Lines: 3, Chars: 25}, + } + if err := s.SaveConversion(ctx, docID, blocks, nil, blobs, registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + + got, err := s.Blocks(ctx, docID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + if len(got) != 4 { + t.Fatalf("got %d blocks on page 2, want 4 -- a column was lost to an index "+ + "collision: %+v", len(got), got) + } + // Left column first, then right, and index order within each. + if got[0].RegionX0 != 43 || got[1].RegionX0 != 43 || + got[2].RegionX0 != 323 || got[3].RegionX0 != 323 { + t.Errorf("the columns are not ordered left to right: %+v", got) + } + if got[0].Index != 0 || got[1].Index != 1 || got[2].Index != 0 || got[3].Index != 1 { + t.Errorf("the indices are not in order within each column: %+v", got) + } + if got[0].Text != "Sicherheitshinweise" || got[2].Text != "Wskazowki bezpieczenstwa" { + t.Errorf("index 0 of each column is not distinct: %q and %q", got[0].Text, got[2].Text) + } + + // And the funnel: asking for German returns the German column and no Polish at + // all, which conversion.md calls the one failure a reader would notice + // immediately. + german, err := s.BlocksByLang(ctx, docID, "de") + if err != nil { + t.Fatalf("blocks by lang: %v", err) + } + if len(german) != 2 { + t.Fatalf("German has %d blocks, want 2: %+v", len(german), german) + } + for i := range german { + if german[i].Lang != "de" { + t.Errorf("a %s block came back in the German conversion: %+v", + german[i].Lang, german[i]) + } + } + if german[0].Text != "Sicherheitshinweise" || german[1].Text != "Lesen Sie diese Anleitung." { + t.Errorf("the German column is not in reading order: %+v", german) + } + + // One page's blocks, both columns, which is what a page view asks for. + onPage, err := s.BlocksForPage(ctx, docID, 2) + if err != nil { + t.Fatalf("blocks for page: %v", err) + } + if len(onPage) != 4 { + t.Errorf("page 2 has %d blocks, want 4", len(onPage)) + } +} + +func TestDeletingADocumentRemovesItsBlocksAndFigures(t *testing.T) { + // ON DELETE CASCADE, on both tables. Without it they accumulate rows pointing at + // documents that no longer exist, and the FK clause is easy to copy without the + // cascade with nothing else noticing. + s, database := newServiceWithDB(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "07") + + blocks := []doc.Block{{ + Page: 62, RegionX0: 43, Index: 0, Kind: doc.BlockParagraph, + Text: "Garantie.", Lang: "de", X0: 43, X1: 305, Y0: 100, Y1: 116, + Lines: 1, Chars: 9, + }} + fig := figure(t, 62, 0, 12, 9) + if err := s.SaveConversion(ctx, docID, blocks, []doc.Figure{fig}, blobs, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + if got, err := s.Blocks(ctx, docID); err != nil || len(got) != 1 { + t.Fatalf("blocks before delete = %+v, %v; want 1 row", got, err) + } + if got, err := s.Figures(ctx, docID); err != nil || len(got) != 1 { + t.Fatalf("figures before delete = %+v, %v; want 1 row", got, err) + } + + if err := gen.New(database.Write()).DeleteDocument(ctx, docID); err != nil { + t.Fatalf("delete document: %v", err) + } + if _, err := s.GetDocument(ctx, docID); !errors.Is(err, registry.ErrNotFound) { + t.Fatalf("document survived deletion: %v", err) + } + + if got, err := s.Blocks(ctx, docID); err != nil || len(got) != 0 { + t.Errorf("%d blocks outlived their document: %+v (%v)", len(got), got, err) + } + if got, err := s.Figures(ctx, docID); err != nil || len(got) != 0 { + t.Errorf("%d figures outlived their document: %+v (%v)", len(got), got, err) + } + + // The BYTES are deliberately still there. A blob outlives the rows pointing at + // it and is collected by counting references, because two documents can + // legitimately render the same picture -- the same diagram in five languages' + // sections is one set of bytes. + if !blobs.Exists(fig.Digest) { + t.Error("deleting a document deleted a figure's bytes from the content-addressed store") + } +} + +func TestAConversionThatFailsToSaveLeavesTheStateAlone(t *testing.T) { + // The state moves in the same transaction as the content it rests on, so a save + // that fails part-way through leaves a document that still says what it said. + // Setting the state on its own handle -- or before the rows -- would leave this + // document claiming to be readable with no blocks behind it, which is exactly + // what a reader sees as an empty manual. + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "b7") + + // Page 0 violates doc_blocks' CHECK (page >= 1), so this fails inside the + // transaction and after the point a wrongly-ordered state write would already + // have committed. + blocks := []doc.Block{ + {Page: 12, RegionX0: 0, Index: 0, Kind: doc.BlockParagraph, Text: "gut", Lang: "de"}, + {Page: 0, RegionX0: 0, Index: 0, Kind: doc.BlockParagraph, Text: "impossible", Lang: "de"}, + } + if err := s.SaveConversion(ctx, docID, blocks, nil, blobs, registry.StateReady); err == nil { + t.Fatal("a block on page 0 was accepted") + } + + document, err := s.GetDocument(ctx, docID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if document.State == registry.StateReady { + t.Errorf("the document says %q after a failed save; the state must not outlive "+ + "the content it claims", document.State) + } + + got, err := s.Blocks(ctx, docID) + if err != nil { + t.Fatalf("blocks: %v", err) + } + if len(got) != 0 { + t.Errorf("a rolled-back conversion left %d blocks behind", len(got)) + } +} + +func TestOneLanguagesFiguresAreItsOwnPlusTheNeutralOnes(t *testing.T) { + // The read side of the rule conversion.md settles: a picture belonging to no + // language belongs to every language. A household reading two languages stores + // the union of both, so selecting a language's pictures by page would hand a + // German reader everything out of the Ukrainian column of every page they share. + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "c3") + + // One page, two columns, a picture in each and one spanning both. + if err := s.SaveProbe(ctx, docID, resultWith( + doc.Region{Page: 1, X0: 40, X1: 440, Lang: "de", Source: doc.SourceRepertoire, Chars: 900, Runs: 30}, + doc.Region{Page: 1, X0: 460, X1: 860, Lang: "uk", Source: doc.SourceRepertoire, Chars: 900, Runs: 30}, + ), registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + german := figure(t, 1, 0, 12, 9) + german.Rect = doc.CellRect{X0: 60, Y0: 100, X1: 400, Y1: 300} + ukrainian := figure(t, 1, 1, 13, 9) + ukrainian.Rect = doc.CellRect{X0: 480, Y0: 100, X1: 840, Y1: 300} + neutral := figure(t, 1, 2, 14, 9) + neutral.Rect = doc.CellRect{X0: 40, Y0: 400, X1: 860, Y1: 600} + + if err := s.SaveConversion(ctx, docID, nil, + []doc.Figure{german, ukrainian, neutral}, blobs, registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + + for _, tc := range []struct { + lang string + want []string + }{ + {"de", []string{german.Digest, neutral.Digest}}, + {"uk", []string{ukrainian.Digest, neutral.Digest}}, + // The unnamed question: only the picture belonging to no column. Asked by no + // other language's call, which is how it stays reachable. + {"", []string{neutral.Digest}}, + // A language this document does not print still gets the neutral picture. + // Nothing here filters by scope, because the conversion already did. + {"fr", []string{neutral.Digest}}, + } { + got, err := s.FiguresByLang(ctx, docID, tc.lang) + if err != nil { + t.Fatalf("figures for %q: %v", tc.lang, err) + } + digests := make([]string, 0, len(got)) + for i := range got { + digests = append(digests, got[i].SHA256) + } + if len(digests) != len(tc.want) { + t.Errorf("%q got %d figures, want %d: %v", tc.lang, len(digests), len(tc.want), digests) + continue + } + for i := range tc.want { + if digests[i] != tc.want[i] { + t.Errorf("%q figure %d is %s, want %s", tc.lang, i, digests[i], tc.want[i]) + } + } + } +} diff --git a/internal/registry/documents.go b/internal/registry/documents.go new file mode 100644 index 0000000..7a6ab99 --- /dev/null +++ b/internal/registry/documents.go @@ -0,0 +1,636 @@ +package registry + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "time" + + "github.com/gordon2/manualbox/internal/db" + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/id" + "github.com/gordon2/manualbox/internal/store" +) + +// Document states. A document moves through these as the pipeline works on it. +const ( + // StateUploaded means the bytes are stored and a probe is queued. + StateUploaded = "uploaded" + // StateProbing means a worker is reading it. + StateProbing = "probing" + // StateAwaitingScope means the probe finished and the user must decide what to + // process. This is the gate: nothing is spent before it. + StateAwaitingScope = "awaiting_scope" + // StateDeclined means the user chose not to process it. The original is kept. + StateDeclined = "declined" + // StateConverting means a worker is turning the pages in scope into readable + // blocks. It is the first state past the gate, so reaching it means the user + // authorised the spend. + // + // The value has been in 00002's CHECK since the schema was written, with + // nothing setting it. [Service.SaveConversion] is what ends it; what begins it + // is the conversion job, which is not wired yet. + StateConverting = "converting" + // StateReady means nothing further will happen automatically. + StateReady = "ready" + // StateFailed means probing failed permanently. + StateFailed = "failed" +) + +// Document kinds. The kind is not cosmetic: receipts and warranties are never +// sent to a cloud provider, so the class must be known before one is called. +const ( + KindManual = "manual" + KindReceipt = "receipt" + KindWarranty = "warranty" + KindPhoto = "photo" + KindOther = "other" +) + +// Document is an uploaded file belonging to a device. +type Document struct { + ID string `json:"id"` + DeviceID string `json:"deviceId"` + BlobSHA256 string `json:"blobSha256"` + Filename string `json:"filename,omitempty"` + MediaType string `json:"mediaType,omitempty"` + Kind string `json:"kind"` + State string `json:"state"` + LastError string `json:"lastError,omitempty"` + + // Probe results, nil until the document has been probed. + PageCount *int `json:"pageCount,omitempty"` + Encrypted *bool `json:"encrypted,omitempty"` + Tagged *bool `json:"tagged,omitempty"` + HasTextLayer *bool `json:"hasTextLayer,omitempty"` + MedianCharsPerPage *int `json:"medianCharsPerPage,omitempty"` + ContentStartPage *int `json:"contentStartPage,omitempty"` + ContentEndPage *int `json:"contentEndPage,omitempty"` + + // IncludeNeutralPages is the extra scope the user approved at the gate: convert + // the pages no language owns as well. + // + // Stored on the document rather than sent to the conversion, so that the handler + // takes its whole scope from stored state exactly as it takes the household from + // configuration — see 00007's header for why a request field or a job payload + // field would each break something real. False for a document that has not been + // approved, and for every document approved before this existed. + IncludeNeutralPages bool `json:"includeNeutralPages"` + + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + ProbedAt *time.Time `json:"probedAt,omitempty"` +} + +// Probed reports whether the free stages have run. +func (d *Document) Probed() bool { return d.ProbedAt != nil } + +// NewDocument is the input to [Service.CreateDocument]. +type NewDocument struct { + DeviceID string + BlobSHA256 string + Filename string + MediaType string + Kind string +} + +// CreateDocument records an uploaded file against a device. +// +// It is idempotent by content: the same bytes uploaded twice against the same +// device return the existing document rather than creating a second one. That is +// enforced by a unique index, so two concurrent uploads cannot both win. +func (s *Service) CreateDocument(ctx context.Context, in NewDocument) (*Document, bool, error) { + switch { + case in.DeviceID == "": + return nil, false, fmt.Errorf("%w: a document needs a device", ErrInvalid) + case in.BlobSHA256 == "": + return nil, false, fmt.Errorf("%w: a document needs stored content", ErrInvalid) + } + if in.Kind == "" { + in.Kind = KindManual + } + + now := db.Millis(s.now()) + q := gen.New(s.db.Write()) + inserted, err := q.CreateDocument(ctx, gen.CreateDocumentParams{ + ID: id.New(id.Document), + DeviceID: in.DeviceID, + BlobSha256: in.BlobSHA256, + Filename: in.Filename, + MediaType: in.MediaType, + Kind: in.Kind, + State: StateUploaded, + CreatedAt: now, + UpdatedAt: now, + }) + if err != nil { + return nil, false, fmt.Errorf("registry: create document: %w", err) + } + + row, err := q.GetDocumentByDeviceAndBlob(ctx, gen.GetDocumentByDeviceAndBlobParams{ + DeviceID: in.DeviceID, + BlobSha256: in.BlobSHA256, + }) + if err != nil { + return nil, false, fmt.Errorf("registry: read back document: %w", err) + } + return documentFrom(row), inserted == 1, nil +} + +// RecordBlob indexes stored bytes so a document can reference them. +// +// The blob table is the metadata for the content-addressed store on disk, and the +// insert is a no-op when the digest is already present: identical bytes are one +// blob however many documents point at it. +func (s *Service) RecordBlob(ctx context.Context, ref store.Ref, mediaType string) error { + err := gen.New(s.db.Write()).UpsertBlob(ctx, gen.UpsertBlobParams{ + Sha256: ref.SHA256, + SizeBytes: ref.Size, + MediaType: mediaType, + CreatedAt: db.Millis(s.now()), + }) + if err != nil { + return fmt.Errorf("registry: record blob: %w", err) + } + return nil +} + +// GetDocument returns one document. +func (s *Service) GetDocument(ctx context.Context, documentID string) (*Document, error) { + row, err := gen.New(s.db.Read()).GetDocument(ctx, documentID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: document %s", ErrNotFound, documentID) + } + return nil, fmt.Errorf("registry: get document: %w", err) + } + return documentFrom(row), nil +} + +// ListDocumentsForDevice returns a device's documents, newest first. +func (s *Service) ListDocumentsForDevice(ctx context.Context, deviceID string) ([]Document, error) { + rows, err := gen.New(s.db.Read()).ListDocumentsForDevice(ctx, deviceID) + if err != nil { + return nil, fmt.Errorf("registry: list documents: %w", err) + } + out := make([]Document, 0, len(rows)) + for i := range rows { + out = append(out, *documentFrom(rows[i])) + } + return out, nil +} + +// SetDocumentState moves a document to a new state. +func (s *Service) SetDocumentState(ctx context.Context, documentID, state, lastError string) error { + err := gen.New(s.db.Write()).SetDocumentState(ctx, gen.SetDocumentStateParams{ + State: state, + LastError: lastError, + UpdatedAt: db.Millis(s.now()), + ID: documentID, + }) + if err != nil { + return fmt.Errorf("registry: set document state: %w", err) + } + return nil +} + +// ApproveScope records the scope the user approved together with the state that +// says the work is authorised. +// +// One statement, so a crash cannot leave a document converting under a scope +// nobody chose. It is the write half of the promise [ingest.Service.Approve] +// makes: the flag lands in the row before the job is queued, and the handler reads +// it back from the row rather than from anything a caller sent. +// +// includeNeutralPages is a flag and never a list of pages. The set those pages +// make up is recomputed from the stored region map when the conversion runs, so a +// client holding a stale gate cannot cause a page it was never offered to be +// converted. +func (s *Service) ApproveScope(ctx context.Context, documentID, state string, + includeNeutralPages bool) error { + flag := int64(0) + if includeNeutralPages { + flag = 1 + } + err := gen.New(s.db.Write()).ApproveDocumentScope(ctx, gen.ApproveDocumentScopeParams{ + IncludeNeutralPages: flag, + State: state, + UpdatedAt: db.Millis(s.now()), + ID: documentID, + }) + if err != nil { + return fmt.Errorf("registry: approve document scope: %w", err) + } + return nil +} + +// SaveProbe records everything the free stages discovered, in one transaction. +// +// All of it or none of it: a document whose row claims it was probed but whose +// pages are missing would look complete and behave as though the manual had no +// languages. The write is also idempotent — page rows, language runs and regions +// are keyed naturally and upserted, and runs and regions are replaced wholesale — +// because a worker can die after doing the work and have the job run again. +func (s *Service) SaveProbe(ctx context.Context, documentID string, res *doc.Result, state string) error { + now := db.Millis(s.now()) + + return s.db.Tx(ctx, func(tx *sql.Tx) error { + q := gen.New(tx) + + if err := q.RecordDocumentProbe(ctx, gen.RecordDocumentProbeParams{ + PageCount: intPtr(res.Info.Pages), + Encrypted: boolToInt(res.Info.Encrypted), + Tagged: boolToInt(res.Info.Tagged), + HasTextLayer: boolToInt(res.HasTextLayer), + MedianCharsPerPage: intPtr(res.MedianChars), + ContentStartPage: intPtrOrNil(res.ContentStart), + ContentEndPage: intPtrOrNil(res.ContentEnd), + State: state, + ProbedAt: &now, + UpdatedAt: now, + ID: documentID, + }); err != nil { + return fmt.Errorf("record probe: %w", err) + } + + for i := range res.Pages { + p := &res.Pages[i] + lang, source := res.PageLang(p.No) + if err := q.UpsertDocPage(ctx, gen.UpsertDocPageParams{ + DocumentID: documentID, + PageNo: int64(p.No), + Chars: int64(p.Chars), + Script: p.Script, + PageTag: p.Tag, + PrintedFolio: intPtrFrom(p.Folio), + Lang: lang, + LangSource: string(source), + // Nil stays nil rather than becoming 0. The probe counted the pictures on + // the pages no named region claims and nowhere else, and a row that says 0 + // where nobody looked would make the gate offer a diagram plate as empty. + Figures: intPtrFrom(p.Figures), + }); err != nil { + return fmt.Errorf("save page %d: %w", p.No, err) + } + } + + // Every signal's view is stored, not just the reconciled one, so that + // "this manual also contains FR, IT, ES..." is answerable without + // re-probing and a conflict stays inspectable afterwards. + all := make(map[doc.Source][]doc.Run, len(res.BySource)+1) + for source, runs := range res.BySource { + all[source] = runs + } + all[doc.SourceReconciled] = res.Runs + + for source, runs := range all { + // Replace rather than merge: a run the latest probe no longer believes + // in must disappear, not linger from the previous attempt. + if err := q.DeleteDocLangsBySource(ctx, gen.DeleteDocLangsBySourceParams{ + DocumentID: documentID, + Source: string(source), + }); err != nil { + return fmt.Errorf("clear %s runs: %w", source, err) + } + for _, r := range runs { + if err := q.UpsertDocLang(ctx, gen.UpsertDocLangParams{ + DocumentID: documentID, + Source: string(source), + PdfStart: int64(r.Start), + PdfEnd: int64(max(r.End, r.Start)), + Code: r.Code, + Lang: r.Lang, + Title: r.Title, + PrintedPage: intPtrFrom(r.PrintedPage), + Confidence: r.Confidence, + Conflict: boolInt(r.Conflict), + Note: r.Note, + CreatedAt: now, + }); err != nil { + return fmt.Errorf("save %s run %s: %w", source, r.Code, err) + } + } + } + + if err := saveRegions(ctx, q, documentID, res, now); err != nil { + return err + } + return nil + }) +} + +// saveRegions stores the language territories the probe read, inside SaveProbe's +// transaction. +// +// WHETHER TO WRITE AT ALL IS THE DECISION HERE, and it turns on RegionNote rather +// than on len(Regions). +// +// A non-empty RegionNote means positioned text could not be read at all: pdftohtml +// is absent or failed, so the probe has no opinion about regions rather than the +// opinion that there are none. Existing rows are then left exactly as they are. +// Deleting a good region map because an optional tool went missing from the host +// would be destructive, and it is the likely case — poppler is optional at runtime +// here, so the same document can be probed with regions available and then without. +// +// An empty RegionNote means the probe did read the document, so its answer replaces +// what was there even when that answer is no regions at all. An encrypted document +// and one with no text layer both land here legitimately: doc.Analyze returns early +// for them with no regions and no note, and "this document has none" is a real +// result that must overwrite a stale map rather than hide behind it. +// +// Replace, not merge: source is part of the primary key and a region's attribution +// can change between probes, so an upsert alone would leave the superseded row +// behind at the same x0 and the page would report itself twice. The delete is +// load-bearing, not tidying. See the note at the foot of 00004_doc_regions.sql. +func saveRegions(ctx context.Context, q *gen.Queries, documentID string, res *doc.Result, now int64) error { + if res.RegionNote != "" { + return nil + } + + if err := q.DeleteDocRegions(ctx, documentID); err != nil { + return fmt.Errorf("clear regions: %w", err) + } + for i := range res.Regions { + r := &res.Regions[i] + if err := q.UpsertDocRegion(ctx, gen.UpsertDocRegionParams{ + DocumentID: documentID, + Source: string(r.Source), + Page: int64(r.Page), + X0: roundCoord(r.X0), + X1: roundCoord(r.X1), + Code: r.Code, + Lang: r.Lang, + Chars: int64(r.Chars), + Runs: int64(r.Runs), + Conflict: boolInt(r.Conflict), + Note: r.Note, + CreatedAt: now, + }); err != nil { + return fmt.Errorf("save region on page %d at x %.0f: %w", r.Page, r.X0, err) + } + } + return nil +} + +// roundCoord narrows a region's float coordinate to the integer the schema stores. +// +// Rounded, not truncated. A float in a primary key would need two probes to produce +// bit-identical floats before the upsert converged, and one unit here is one pixel +// of a pdftoppm -r 108 raster, so sub-unit precision describes nothing about a +// column boundary. Truncation would instead bias every edge left by up to a unit. +// Negative coordinates are clamped: the schema requires x0 >= 0, and a run parked +// off the left edge of the page is furniture, not a column that starts at -3. +func roundCoord(v float64) int64 { + if v <= 0 { + return 0 + } + return int64(math.Round(v)) +} + +// LanguageRun is one stored language run. +type LanguageRun struct { + Source string `json:"source"` + Code string `json:"code"` + Lang string `json:"lang"` + Name string `json:"name"` + Title string `json:"title,omitempty"` + Start int `json:"start"` + End int `json:"end"` + Pages int `json:"pages"` + PrintedPage *int `json:"printedPage,omitempty"` + Confidence float64 `json:"confidence"` + Conflict bool `json:"conflict"` + Note string `json:"note,omitempty"` +} + +// LanguageRuns returns a document's runs for one signal. Passing +// doc.SourceReconciled gives the map manualbox believes. +func (s *Service) LanguageRuns(ctx context.Context, documentID string, source doc.Source) ([]LanguageRun, error) { + rows, err := gen.New(s.db.Read()).ListDocLangsBySource(ctx, gen.ListDocLangsBySourceParams{ + DocumentID: documentID, + Source: string(source), + }) + if err != nil { + return nil, fmt.Errorf("registry: list language runs: %w", err) + } + out := make([]LanguageRun, 0, len(rows)) + for i := range rows { + r := &rows[i] + out = append(out, LanguageRun{ + Source: r.Source, Code: r.Code, Lang: r.Lang, + Name: doc.DisplayName(r.Lang), + Title: r.Title, + Start: int(r.PdfStart), End: int(r.PdfEnd), + Pages: pageSpan(r.PdfStart, r.PdfEnd), + PrintedPage: intFromPtr(r.PrintedPage), + Confidence: r.Confidence, + Conflict: r.Conflict == 1, + Note: r.Note, + }) + } + return out, nil +} + +// PageFact is what the probe stored about one page: the facts that genuinely are +// per page, which is why they did not move to doc_regions. Language is here too, +// and it is the one field that can be absent where a region names something: a +// page holding three languages has no honest per-page answer, so a +// parallel-columns manual stores ” on every page and its languages live only in +// its regions. +type PageFact struct { + Page int `json:"page"` + Chars int `json:"chars"` + // Script is the dominant Unicode script, and Tag the language code printed on + // the page, both empty when nothing was read. + Script string `json:"script,omitempty"` + Tag string `json:"tag,omitempty"` + // PrintedFolio is the page number the page prints, which is usually offset from + // the PDF's own. + PrintedFolio *int `json:"printedFolio,omitempty"` + // Lang is the reconciled per-page language, empty when the per-page signals + // named nothing, and LangSource says which signal named it. + Lang string `json:"lang,omitempty"` + LangSource string `json:"langSource,omitempty"` + // Figures is how many pictures the page holds, nil when nobody counted it. + // + // Nil on almost every page of almost every document, by design: the probe counts + // only the pages no named region claims, because counting a page's drawings is a + // pdftocairo spawn. Nil is "not counted" and 0 is "counted, none found" — see + // doc.Page.Figures and 00007's header for why conflating them would tell the gate + // that a diagram plate is empty. + Figures *int `json:"figures,omitempty"` +} + +// Pages returns the stored per-page facts in page order. +// +// The gate reads these to answer two questions that need a page's size rather than +// its language: how many characters a language covers when no regions were stored, +// and how many content pages carry text that nothing could name. +func (s *Service) Pages(ctx context.Context, documentID string) ([]PageFact, error) { + rows, err := gen.New(s.db.Read()).ListDocPages(ctx, documentID) + if err != nil { + return nil, fmt.Errorf("registry: list pages: %w", err) + } + out := make([]PageFact, 0, len(rows)) + for i := range rows { + r := &rows[i] + out = append(out, PageFact{ + Page: int(r.PageNo), + Chars: int(r.Chars), + Script: r.Script, + Tag: r.PageTag, + PrintedFolio: intFromPtr(r.PrintedFolio), + Lang: r.Lang, + LangSource: r.LangSource, + Figures: intFromPtr(r.Figures), + }) + } + return out, nil +} + +// Region is one stored language territory on a page. +// +// X0 and X1 are integers here because that is what is stored, and the coordinate +// space is poppler's: one unit is one pixel of a pdftoppm -r 108 raster, 1.5 times +// the PDF's own points. A whole-page region runs from 0 to the page width, which is +// why no field says whether a region is boxed — a caller clipping to the box gets +// the whole page and needs no special case. +type Region struct { + Page int `json:"page"` + X0 int `json:"x0"` + X1 int `json:"x1"` + Source string `json:"source"` + Code string `json:"code"` + Lang string `json:"lang"` + Name string `json:"name"` + Chars int `json:"chars"` + Runs int `json:"runs"` + Conflict bool `json:"conflict"` + Note string `json:"note,omitempty"` +} + +// Regions returns a document's language territories in reading order: down the +// page, then left to right across it. +// +// Empty is not the same claim as absent. A document probed without pdftohtml +// available has no regions stored and its per-page language map is still complete, +// so a caller must not read an empty result as "this manual has one language". +func (s *Service) Regions(ctx context.Context, documentID string) ([]Region, error) { + rows, err := gen.New(s.db.Read()).ListDocRegions(ctx, documentID) + if err != nil { + return nil, fmt.Errorf("registry: list regions: %w", err) + } + out := make([]Region, 0, len(rows)) + for i := range rows { + r := &rows[i] + out = append(out, Region{ + Page: int(r.Page), + X0: int(r.X0), X1: int(r.X1), + Source: r.Source, + Code: r.Code, + Lang: r.Lang, + // The UI shows "Ukrainian", not "uk", and the manual's own label may be + // neither: it prints UA. + Name: doc.DisplayName(r.Lang), + Chars: int(r.Chars), + Runs: int(r.Runs), + Conflict: r.Conflict == 1, + Note: r.Note, + }) + } + return out, nil +} + +func documentFrom(r gen.Document) *Document { + return &Document{ + ID: r.ID, + DeviceID: r.DeviceID, + BlobSHA256: r.BlobSha256, + Filename: r.Filename, + MediaType: r.MediaType, + Kind: r.Kind, + State: r.State, + LastError: r.LastError, + PageCount: intFromPtr(r.PageCount), + Encrypted: boolFromPtr(r.Encrypted), + Tagged: boolFromPtr(r.Tagged), + HasTextLayer: boolFromPtr(r.HasTextLayer), + MedianCharsPerPage: intFromPtr(r.MedianCharsPerPage), + ContentStartPage: intFromPtr(r.ContentStartPage), + ContentEndPage: intFromPtr(r.ContentEndPage), + + IncludeNeutralPages: r.IncludeNeutralPages == 1, + + CreatedAt: db.Time(r.CreatedAt), + UpdatedAt: db.Time(r.UpdatedAt), + ProbedAt: db.TimePtr(r.ProbedAt), + } +} + +// pageSpan is how many pages a stored run covers. +// +// A start of 0 means the signal named a language but could not place it, which the +// schema documents and the API must not turn into a section: the arithmetic span +// reported the fixture's unplaceable HE, AR and CZ index entries as one-page +// Arabic, Hebrew and Czech sections spanning 0-0. +func pageSpan(start, end int64) int { + if start == 0 { + return 0 + } + return int(end - start + 1) +} + +func intPtr(n int) *int64 { + v := int64(n) + return &v +} + +// intPtrOrNil keeps 0 out of the database as a claim. A content range of 0 means +// "not established", which NULL says and 0 does not. +func intPtrOrNil(n int) *int64 { + if n == 0 { + return nil + } + return intPtr(n) +} + +func intPtrFrom(n *int) *int64 { + if n == nil { + return nil + } + return intPtr(*n) +} + +func intFromPtr(n *int64) *int { + if n == nil { + return nil + } + v := int(*n) + return &v +} + +func boolToInt(b bool) *int64 { + var v int64 + if b { + v = 1 + } + return &v +} + +func boolInt(b bool) int64 { + if b { + return 1 + } + return 0 +} + +func boolFromPtr(n *int64) *bool { + if n == nil { + return nil + } + v := *n == 1 + return &v +} diff --git a/internal/registry/figurelabels_test.go b/internal/registry/figurelabels_test.go new file mode 100644 index 0000000..709cb58 --- /dev/null +++ b/internal/registry/figurelabels_test.go @@ -0,0 +1,184 @@ +package registry_test + +import ( + "context" + "reflect" + "testing" + + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" +) + +// TestFigureLabelsRoundTrip covers the storage half of carrying a callout label as +// text, and every part of it that could plausibly be lost on the way through. +// +// THE ORDER is why this is exhaustive rather than a smoke test, now that the geometry +// is gone (00009). A label's idx is its place in one sorted sequence — down the page, +// then across it — and that sequence is the whole meaning of the stored set: the +// strings are a picture's accessible description, read in order, so a scan that +// returned them shuffled would leave every count and every string correct and still +// describe the diagram wrongly. The four below are deliberately NOT in alphabetical +// order, in no order the map iteration or a re-sort could reproduce by luck, and one of +// them repeats an earlier string so that a set comparison cannot stand in for a +// sequence one. The text is Cyrillic because the document this was built for is. +func TestFigureLabelsRoundTrip(t *testing.T) { + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "a1") + + fig := figure(t, 521, 0, 12, 9) + fig.Labels = []string{ + // Page 521's real labels, in the order figureLabels sorts them into. A wrapped + // label arrives as one string with its later lines joined on, which is what + // TestNoLabelIsCarriedWithoutItsLaterLines pins on the other side of the seam. + "Основная щетка", + "Датчики перепада высоты", + // A bare number, and then the SAME STRING AGAIN at a later index. Two labels of + // one figure may read alike — a plate numbers its parts and the paper reuses a + // digit — so idx is what tells them apart, and a read path that de-duplicated or + // keyed on the text would lose one here and nowhere else. + "12", + "A-1", + "12", + } + // A SECOND LABELLED FIGURE ON THE SAME PAGE, at a non-zero index. This is what + // makes the (page, index) key testable: with labels only ever on figure 0, keying + // on the page alone gives the same answer and a mutation that drops the index + // survives every assertion below. + second := figure(t, 521, 1, 10, 10) + second.Labels = []string{"Кнопка сброса"} + // A third with none at all, which is the normal case — most pictures have nothing + // pointing at them — and which must stay empty rather than collecting either set. + bare := figure(t, 521, 2, 8, 8) + + if err := s.SaveConversion(ctx, docID, nil, []doc.Figure{fig, second, bare}, blobs, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + + got, err := s.Figures(ctx, docID) + if err != nil { + t.Fatalf("figures: %v", err) + } + if len(got) != 3 { + t.Fatalf("got %d figures, want 3", len(got)) + } + // Compared as a SLICE, so this asserts the sequence and not merely the membership. + want := []string{ + "Основная щетка", + "Датчики перепада высоты", + "12", + "A-1", + "12", + } + if !reflect.DeepEqual(got[0].Labels, want) { + t.Errorf("figure 0's labels:\n got %+v\nwant %+v", got[0].Labels, want) + } + // EACH FIGURE GETS ITS OWN, which is the assertion that the key is (page, index) + // and not merely the page. All three of these are on page 521. + wantSecond := []string{"Кнопка сброса"} + if !reflect.DeepEqual(got[1].Labels, wantSecond) { + t.Errorf("figure 1's labels:\n got %+v\nwant %+v", got[1].Labels, wantSecond) + } + if len(got[2].Labels) != 0 { + t.Errorf("the unlabelled figure on the same page collected %d labels: %+v", + len(got[2].Labels), got[2].Labels) + } + + // FiguresForPage reads through a different pair of queries and must agree. + onPage, err := s.FiguresForPage(ctx, docID, 521) + if err != nil { + t.Fatalf("figures for page: %v", err) + } + if len(onPage) != 3 || !reflect.DeepEqual(onPage[0].Labels, want) || + !reflect.DeepEqual(onPage[1].Labels, wantSecond) { + t.Errorf("FiguresForPage disagrees with Figures: %+v", onPage) + } + if len(onPage[2].Labels) != 0 { + t.Errorf("FiguresForPage gave the unlabelled figure %d labels", len(onPage[2].Labels)) + } + + // Re-converting converges rather than appending a second copy. The labels are + // deleted and rewritten every time, and their index is a position in a sorted order + // rather than anything the page prints — see doc.figureLabels on why it sorts. + if err := s.SaveConversion(ctx, docID, nil, []doc.Figure{fig, second, bare}, blobs, + registry.StateReady); err != nil { + t.Fatalf("re-save conversion: %v", err) + } + again, err := s.Figures(ctx, docID) + if err != nil { + t.Fatalf("figures after re-save: %v", err) + } + if !reflect.DeepEqual(again, got) { + t.Errorf("re-converting changed the figures:\n got %+v\nwant %+v", again, got) + } +} + +// TestAFigureLabelIsCascadedWithItsFigure covers the composite foreign key. A +// re-conversion that no longer finds a picture must not leave its labels behind: figure +// indices run consecutively from 0 within a page, so an orphaned row would be served +// against whatever figure later took that index. It is the same hazard 00005's header +// records for blocks, one level down. +func TestAFigureLabelIsCascadedWithItsFigure(t *testing.T) { + s, database := newServiceWithDB(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "a1") + + fig := figure(t, 521, 0, 12, 9) + fig.Labels = []string{"12"} + if err := s.SaveConversion(ctx, docID, nil, []doc.Figure{fig}, blobs, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + rows, err := gen.New(database.Read()).ListDocFigureLabels(ctx, docID) + if err != nil { + t.Fatalf("list labels: %v", err) + } + if len(rows) != 1 { + t.Fatalf("stored %d label rows, want 1", len(rows)) + } + + // The trim rule now rejects that picture, so the second conversion finds none. + if err := s.SaveConversion(ctx, docID, nil, nil, blobs, registry.StateReady); err != nil { + t.Fatalf("re-save with no figures: %v", err) + } + // Asked of the table directly, because that is the point: reading through Figures + // would report success either way, since it only attaches a label to a figure it + // can find. An orphan is invisible there and still in the database. + rows, err = gen.New(database.Read()).ListDocFigureLabels(ctx, docID) + if err != nil { + t.Fatalf("list labels after re-save: %v", err) + } + if len(rows) != 0 { + t.Errorf("%d label row(s) outlived the figure they belong to", len(rows)) + } +} + +// TestAnEmptyFigureLabelIsRejected pins the CHECK on text. An empty label is a claim +// that produced no text, which is a defect in the claim rule rather than something a +// reader can be shown — and it would render as an empty box floating beside a picture. +func TestAnEmptyFigureLabelIsRejected(t *testing.T) { + s := newService(t) + blobs := newBlobStore(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "a1") + + fig := figure(t, 521, 0, 12, 9) + fig.Labels = []string{""} + err := s.SaveConversion(ctx, docID, nil, []doc.Figure{fig}, blobs, registry.StateReady) + if err == nil { + t.Fatal("saved a label with no text; the schema must refuse it") + } + // And the whole conversion is rolled back, which is what makes the CHECK safe to + // rely on rather than a source of half-written state. + got, figErr := s.Figures(ctx, docID) + if figErr != nil { + t.Fatalf("figures: %v", figErr) + } + if len(got) != 0 { + t.Errorf("the failed conversion left %d figures behind: %+v", len(got), got) + } +} diff --git a/internal/registry/folio.go b/internal/registry/folio.go new file mode 100644 index 0000000..349f485 --- /dev/null +++ b/internal/registry/folio.go @@ -0,0 +1,137 @@ +package registry + +import ( + "context" + "fmt" + + "github.com/gordon2/manualbox/internal/db/gen" +) + +// The rule for turning a document's folio histogram into one offset. +// +// A contents entry names a page printed on the paper, and the reader has to open a +// page of the PDF. The two differ by the front matter the printer bound in front of +// page 1, so `pdf = printed + offset` -- and the whole question is whether one +// offset is true for the whole document. +// +// Measured against both real manuals' stored doc_pages, which is the pipeline's own +// answer rather than a re-reading of the PDFs: +// +// sequential (560pp): 558 pages print a folio, 552 of them at offset 6 +// columns ( 68pp): 67 pages print a folio, 65 of them at offset 0 +// +// The runner-up covers exactly one page in each, so the margin is 552-to-1 and +// 65-to-1. Every deviation is a misread of a short line that is not a folio: the +// sequential manual's contents pages read their own body numbers (194, 403, 533), +// its diagram plates read a callout number, and the columns manual's back cover +// reads 2735. +const ( + // minFolioSupport is the share of the folio-bearing pages the modal offset must + // hold before it is offered at all. + // + // The mode, not the mean -- for the same reason internal/doc's columnPitch takes + // the mode of its line gaps rather than their median, recorded there: a handful + // of readings that are not measurements of the thing at all drag an average off + // the value every real member sits exactly on. Here it is worse than a drag. One + // page misread as folio 2735 puts the mean 40 pages out, and 6 of the sequential + // manual's outliers are negative offsets in the hundreds. + // + // 0.6 is chosen from the two measurements above and from what the failure looks + // like. A document whose folios genuinely restart per section has no majority: + // if the sequential manual's 34 sections each began again at 1, its biggest + // section would hold 22 of 553 pages and the best offset would have 4.0% + // support. The case that could still fool a bare plurality is a document bound + // in two halves, where the larger half is near 50%. So the floor has to be + // above a half, and being above a half buys a second property for nothing: at + // most one offset can hold more than half the pages, so the mode is unique by + // construction and no tie-break policy is needed. + // + // What it costs, said plainly: a document that really does have one offset, but + // whose folios are read so badly that fewer than three pages in five agree, is + // refused and its contents entries stay plain text. That is the right side to + // fail on -- a link to the wrong page is worse than no link -- and it is a long + // way from anything observed: the two real manuals disagree on 6 of 558 and 2 of + // 67. Refusing to answer is also the correct outcome for a genuinely restarting + // document, which is the case this floor exists to catch. + minFolioSupport = 0.6 + + // minFolioPages is how many pages must print a folio before their agreement + // means anything. + // + // Below four, minFolioSupport is satisfied by 1 of 1, 2 of 2 or 2 of 3, none of + // which is evidence of a constant that holds across a document. 3 of 4 is the + // smallest reading in which an outlier has actually been outvoted. + minFolioPages = 4 +) + +// FolioOffset is how far a document's PDF pages run ahead of its printed folios: +// the PDF page for a printed page number is printed + Offset. +// +// Pages and Support are the evidence, carried so a caller can say why rather than +// only what. Offset is very often 0 -- the columns manual's really is -- so the +// answer is a pointer at every layer above this one, and "no confident answer" must +// never be flattened into "offset zero". +type FolioOffset struct { + Offset int + // Pages is how many pages agree on Offset, of the FolioPages that print one. + Pages int + FolioPages int + // Support is Pages over FolioPages, between 0 and 1. + Support float64 +} + +// FolioOffset reports the one offset that maps this document's printed page numbers +// onto its PDF pages, or nil where the stored folios do not agree on one. +// +// Derived from doc_pages on every call rather than stored: see the query's own +// header for why, and note that it is asked once per conversion response, not per +// entry. +func (s *Service) FolioOffset(ctx context.Context, documentID string) (*FolioOffset, error) { + rows, err := gen.New(s.db.Read()).DocPageFolioOffsets(ctx, documentID) + if err != nil { + return nil, fmt.Errorf("registry: folio offsets: %w", err) + } + counts := make([]FolioOffsetCount, 0, len(rows)) + for i := range rows { + counts = append(counts, FolioOffsetCount{ + Offset: int(rows[i].FolioOffset), + Pages: int(rows[i].Pages), + }) + } + return modalFolioOffset(counts), nil +} + +// FolioOffsetCount is one bar of the histogram: an offset and how many of the +// document's pages read that way. +type FolioOffsetCount struct { + Offset int + Pages int +} + +// modalFolioOffset applies the rule above to a histogram. +// +// Separated from the query so the rule can be exercised on the real outliers +// without a database. The input need not be sorted. +func modalFolioOffset(counts []FolioOffsetCount) *FolioOffset { + total := 0 + best := -1 + for i := range counts { + total += counts[i].Pages + if best < 0 || counts[i].Pages > counts[best].Pages { + best = i + } + } + if best < 0 || total < minFolioPages { + return nil + } + support := float64(counts[best].Pages) / float64(total) + if support < minFolioSupport { + return nil + } + return &FolioOffset{ + Offset: counts[best].Offset, + Pages: counts[best].Pages, + FolioPages: total, + Support: support, + } +} diff --git a/internal/registry/folio_fixture_test.go b/internal/registry/folio_fixture_test.go new file mode 100644 index 0000000..95196eb --- /dev/null +++ b/internal/registry/folio_fixture_test.go @@ -0,0 +1,121 @@ +package registry_test + +import ( + "context" + "os" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" + "github.com/gordon2/manualbox/internal/registry" +) + +// TestFolioOffsetOnRealManuals pins the offset both real manuals produce, and the +// margin it wins by. +// +// These four numbers are the ones a change to internal/doc's pageFolio would move, +// which is exactly why they are here rather than only in a design doc. The offset +// itself is the user-visible one -- it is what a contents entry jumps by -- and the +// support is what says the offset was believed for a reason. A change that leaves +// the offset alone but halves its support has broken the folio reader without +// breaking any link yet, and that is worth being told about before it gets worse. +// +// Both documents are asserted in one test because their two answers are the +// contrast the whole feature rests on: the columns manual's real offset is zero, so +// "no mapping" and "offset zero" are two different answers that must not converge. +func TestFolioOffsetOnRealManuals(t *testing.T) { + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixtures and run the real-document tests", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFInfo, extern.PDFToText, extern.PDFToHTML} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + + tests := []struct { + name string + fixture string + digest string + shape string + offset int + pages int + folioPgs int + }{ + { + // 558 of its 560 pages print a folio. The six that disagree are all + // misreads of a short line that is not a folio: pages 2-4 are contents + // pages whose own body numbers were read (194, 403, 533), pages 5-6 are + // diagram plates where a callout number was, and page 509 reads 2. + name: "the sequential manual, offset 6", fixture: "dreame-l40-ultra", + digest: "a1", shape: "560 pages of sections one after another", + offset: 6, pages: 552, folioPgs: 558, + }, + { + // 67 of its 68 pages print a folio, and the offset really is zero: this + // manual's page 1 is its cover. Page 12 reads 10 and the back cover reads + // 2735. + name: "the columns manual, offset 0", fixture: "thomas-drybox-amfibia", + digest: "b2", shape: "68 pages of five languages in parallel columns", + offset: 0, pages: 65, folioPgs: 67, + }, + } + + ctx := context.Background() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manifest, err := fixture.Load(fixturesDir, tt.fixture) + if err != nil { + t.Fatalf("load manifest: %v", err) + } + path, err := manifest.Fetch(ctx) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + res, err := doc.Analyze(ctx, path) + if err != nil { + t.Fatalf("analyze: %v", err) + } + + s := newService(t) + docID := newProbedDocument(t, s, tt.digest) + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + got, err := s.FolioOffset(ctx, docID) + if err != nil { + t.Fatalf("folio offset: %v", err) + } + if got == nil { + t.Fatalf("no offset was offered for %s.\n"+ + "This document's folios used to agree on %d for %d of the %d pages that "+ + "print one. Losing the answer entirely means pageFolio now reads folios "+ + "that disagree, and every contents entry in this manual has stopped being "+ + "a link.", tt.shape, tt.offset, tt.pages, tt.folioPgs) + } + if got.Offset != tt.offset { + t.Errorf("offset %d, want %d.\n"+ + "This is what a contents entry jumps by, so every link in %s now lands "+ + "%d pages from where it should. Either pageFolio reads a different line "+ + "than it did, or this manual's front matter changed.", + got.Offset, tt.offset, tt.shape, got.Offset-tt.offset) + } + if got.FolioPages != tt.folioPgs { + t.Errorf("%d pages print a folio, want %d.\n"+ + "pageFolio is finding folios on a different set of pages than it did; "+ + "the offset above may still be right by luck.", + got.FolioPages, tt.folioPgs) + } + if got.Pages != tt.pages { + t.Errorf("%d of %d pages agree on offset %d, want %d.\n"+ + "The answer has not changed but the evidence for it has. Fewer agreeing "+ + "pages means pageFolio is misreading more lines as folios; more means it "+ + "has started reading folios it used to miss. Both are real changes and "+ + "neither is visible in the offset alone.", + got.Pages, got.FolioPages, got.Offset, tt.pages) + } + }) + } +} diff --git a/internal/registry/folio_internal_test.go b/internal/registry/folio_internal_test.go new file mode 100644 index 0000000..ee32363 --- /dev/null +++ b/internal/registry/folio_internal_test.go @@ -0,0 +1,137 @@ +package registry + +import "testing" + +// The rule, exercised on the histograms both real manuals actually produce plus the +// cases the floor exists to refuse. Hermetic: modalFolioOffset takes the histogram, +// so none of this needs a database or a PDF. +func TestModalFolioOffset(t *testing.T) { + t.Parallel() + + // The sequential manual's real histogram, read from the stored doc_pages of both + // converted manuals. 558 pages print a folio; six of them are misreads of a + // short line that is not a folio -- two contents pages, two diagram plates whose + // callout number was read, and page 509 reading 2. + sequential := []FolioOffsetCount{ + {Offset: 6, Pages: 552}, + {Offset: 507, Pages: 1}, + {Offset: 3, Pages: 1}, + {Offset: 2, Pages: 1}, + {Offset: -192, Pages: 1}, + {Offset: -400, Pages: 1}, + {Offset: -529, Pages: 1}, + } + // The columns manual's real histogram. Its true offset is zero, which is exactly + // the value that must not be confusable with "no answer". Page 12 reads 10 and + // the back cover reads 2735. + columns := []FolioOffsetCount{ + {Offset: 0, Pages: 65}, + {Offset: 2, Pages: 1}, + {Offset: -2667, Pages: 1}, + } + + tests := []struct { + name string + counts []FolioOffsetCount + // want is nil where the document must get no answer. + want *FolioOffset + }{ + { + name: "the sequential manual, six misreads and all", + counts: sequential, + want: &FolioOffset{Offset: 6, Pages: 552, FolioPages: 558}, + }, + { + name: "the columns manual, whose real offset is zero", + counts: columns, + want: &FolioOffset{Offset: 0, Pages: 65, FolioPages: 67}, + }, + { + name: "one misread does not move the mode", + // The mean of these is 336, which is not a page of anything. + counts: []FolioOffsetCount{{Offset: 4, Pages: 20}, {Offset: 2735, Pages: 1}}, + want: &FolioOffset{Offset: 4, Pages: 20, FolioPages: 21}, + }, + { + name: "folios restarting in each of 34 sections have no majority", + // What the sequential manual's histogram would be if every section began + // again at 1: the biggest section holds 22 of 553 pages, so the best + // offset has 4.0% support and there is no document-wide answer to give. + counts: restarting(34, 553), + want: nil, + }, + { + name: "a document bound in two halves is refused, near-majority and all", + // The case a bare plurality would get wrong: 55% is the largest share a + // two-part restart can hand its bigger part while still being a document + // with two offsets rather than one. + counts: []FolioOffsetCount{{Offset: 0, Pages: 55}, {Offset: 30, Pages: 45}}, + want: nil, + }, + { + name: "no page prints a folio at all", + counts: nil, + want: nil, + }, + { + name: "too few folios for their agreement to mean anything", + // 2 of 2 is 100% support and no evidence whatever. + counts: []FolioOffsetCount{{Offset: 6, Pages: 2}}, + want: nil, + }, + { + name: "four folios, one of them outvoted, is the smallest real reading", + counts: []FolioOffsetCount{{Offset: 6, Pages: 3}, {Offset: 100, Pages: 1}}, + want: &FolioOffset{Offset: 6, Pages: 3, FolioPages: 4}, + }, + { + name: "the mode is read from the counts, not from the order", + // The histogram arrives sorted by count, so a rule that took the first row + // would pass every case above. This one is deliberately out of order. + counts: []FolioOffsetCount{{Offset: 99, Pages: 2}, {Offset: 6, Pages: 20}}, + want: &FolioOffset{Offset: 6, Pages: 20, FolioPages: 22}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := modalFolioOffset(tt.counts) + if tt.want == nil { + if got != nil { + t.Fatalf("offset %+v was offered; these folios agree on nothing and the "+ + "document must get no mapping rather than a plausible-looking one", *got) + } + return + } + if got == nil { + t.Fatalf("no offset was offered, want %+v", *tt.want) + } + if got.Offset != tt.want.Offset || got.Pages != tt.want.Pages || + got.FolioPages != tt.want.FolioPages { + t.Fatalf("offset %d on %d of %d pages, want %d on %d of %d", + got.Offset, got.Pages, got.FolioPages, + tt.want.Offset, tt.want.Pages, tt.want.FolioPages) + } + if want := float64(tt.want.Pages) / float64(tt.want.FolioPages); got.Support != want { + t.Fatalf("support %v, want %v", got.Support, want) + } + }) + } +} + +// restarting builds the histogram of a document whose folios begin again at 1 in +// each of sections sections, splitting pages between them as evenly as the +// remainder allows. +func restarting(sections, pages int) []FolioOffsetCount { + out := make([]FolioOffsetCount, 0, sections) + for i := range sections { + n := pages / sections + if i < pages%sections { + n++ + } + // Each section starts where the last ended, so each has its own offset. + out = append(out, FolioOffsetCount{Offset: i * (pages / sections), Pages: n}) + } + return out +} diff --git a/internal/registry/regions_fixture_test.go b/internal/registry/regions_fixture_test.go new file mode 100644 index 0000000..6653a70 --- /dev/null +++ b/internal/registry/regions_fixture_test.go @@ -0,0 +1,165 @@ +package registry_test + +import ( + "context" + "os" + "sort" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" + "github.com/gordon2/manualbox/internal/registry" +) + +const fixturesDir = "../../testdata/fixtures" + +// TestColumnManualRegionsSurviveStorage is docs/design/regions.md's ACCEPTANCE +// CRITERION, which is deliberately not "the migration applies": the pipeline must +// store and read back the real parallel-columns manual's five languages across its +// parallel columns. +// +// It runs the whole seam this commit builds -- doc.Analyze on the actual PDF, then +// SaveProbe, then the read-back -- because every other test in this package feeds +// SaveProbe regions typed by hand, and a struct built in a test cannot show that +// what internal/doc really produces for a 68-page manual survives a round trip +// through a STRICT table with an integer primary key. +// +// The eight pages a human compared against their rendered images are the ones held +// to their column count; the manifest's other pages were produced by the detector +// and holding it to those would be circular. See the provenance note in the +// manifest. +func TestColumnManualRegionsSurviveStorage(t *testing.T) { + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixture and run the real-document tests", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFInfo, extern.PDFToText, extern.PDFToHTML} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + + ctx := context.Background() + manifest, err := fixture.Load(fixturesDir, "thomas-drybox-amfibia") + if err != nil { + t.Fatalf("load manifest: %v", err) + } + path, err := manifest.Fetch(ctx) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + + res, err := doc.Analyze(ctx, path) + if err != nil { + t.Fatalf("analyze: %v", err) + } + if res.RegionNote != "" { + t.Fatalf("regions were not read: %s", res.RegionNote) + } + if len(res.Regions) == 0 { + t.Fatal("the column manual produced no regions") + } + + s := newService(t) + docID := newProbedDocument(t, s, "99") + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + stored, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("read back regions: %v", err) + } + + // Nothing may be lost to the integer primary key. Two same-language columns on + // one page are the case that collides under doc_langs' key, and this document + // really sets them, so a short read-back is the failure this table exists to + // prevent rather than a rounding curiosity. + if len(stored) != len(res.Regions) { + t.Errorf("stored %d regions, read back %d: %d were lost, most likely to a "+ + "primary-key collision", len(res.Regions), len(stored), len(res.Regions)-len(stored)) + } + + // The five languages the manifest records as ground truth. + got := make(map[string]bool) + for i := range stored { + if stored[i].Lang != "" { + got[doc.BaseLanguage(stored[i].Lang)] = true + } + } + for _, want := range manifest.Languages { + if !got[want] { + names := make([]string, 0, len(got)) + for l := range got { + names = append(names, l) + } + sort.Strings(names) + t.Errorf("%s did not survive storage; read back %v", want, names) + } + } + + // Characters, not pages, is the unit -- and it must be preserved exactly, since + // it is what the pre-flight gate will price the job from. + var wantChars, gotChars int + for i := range res.Regions { + wantChars += res.Regions[i].Chars + } + for i := range stored { + gotChars += stored[i].Chars + } + if gotChars != wantChars { + t.Errorf("characters survived as %d, want %d", gotChars, wantChars) + } + + // Column for column on the human-verified pages: every region internal/doc put + // on such a page must come back, at the rounded x0 it went in with. + byPage := make(map[int][]registry.Region, len(stored)) + for i := range stored { + byPage[stored[i].Page] = append(byPage[stored[i].Page], stored[i]) + } + for _, fact := range manifest.VerifiedPages() { + var want []doc.Region + for i := range res.Regions { + if res.Regions[i].Page == fact.Page { + want = append(want, res.Regions[i]) + } + } + have := byPage[fact.Page] + if len(have) != len(want) { + t.Errorf("page %d: %d regions computed, %d read back", fact.Page, len(want), len(have)) + continue + } + for i := range want { + if x0 := int(want[i].X0 + 0.5); have[i].X0 != x0 { + t.Errorf("page %d region %d: x0 read back as %d, want %d", + fact.Page, i, have[i].X0, x0) + } + if have[i].Lang != want[i].Lang || have[i].Chars != want[i].Chars { + t.Errorf("page %d region %d: read back %s/%d chars, want %s/%d", + fact.Page, i, have[i].Lang, have[i].Chars, want[i].Lang, want[i].Chars) + } + } + } + + // Re-probing a real document must converge, not accumulate. The idempotency + // tests above use three hand-written regions; this is the same property over the + // document's real region set, where a single colliding pair would show up. + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("re-save probe: %v", err) + } + again, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("read back after re-probe: %v", err) + } + if len(again) != len(stored) { + t.Errorf("re-probing changed the row count from %d to %d", len(stored), len(again)) + } + for i := range stored { + if i < len(again) && again[i] != stored[i] { + t.Errorf("region %d changed on re-probe:\nfirst %+v\nsecond %+v", i, stored[i], again[i]) + } + } + + t.Logf("column manual: %d regions computed, %d stored, %d characters, languages %v", + len(res.Regions), len(stored), gotChars, manifest.Languages) +} diff --git a/internal/registry/regions_test.go b/internal/registry/regions_test.go new file mode 100644 index 0000000..0d61d73 --- /dev/null +++ b/internal/registry/regions_test.go @@ -0,0 +1,408 @@ +package registry_test + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/db" + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// newServiceWithDB is newService plus the handle, for the two assertions that must +// be made against the database rather than through the service: the region summary, +// whose arithmetic is in SQL with nothing in Go protecting it, and the cascade, +// which needs to delete a document row and the service has no method for that. +func newServiceWithDB(t *testing.T) (*registry.Service, *db.DB) { + t.Helper() + database, err := db.Open(context.Background(), db.Options{ + Path: filepath.Join(t.TempDir(), "registry.db"), + }) + if err != nil { + t.Fatalf("open database: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + return registry.New(database, registry.Options{}), database +} + +// newProbedDocument makes a device, a blob and a document to hang regions off, +// because every test here needs one and the ceremony is not what any of them is +// about. The digest is a parameter so two documents in one test cannot collide on +// the unique (device_id, blob_sha256) index. +func newProbedDocument(t *testing.T, s *registry.Service, digest string) string { + t.Helper() + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: "Dry box"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat(digest, 32), Size: 10} + if err := s.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, Filename: "manual.pdf", + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + return document.ID +} + +// resultWith wraps regions in the minimum viable probe result. RegionNote is empty, +// which is what says "the probe did read the positioned text". +func resultWith(regions ...doc.Region) *doc.Result { + return &doc.Result{ + Info: doc.Info{Pages: 4}, + Pages: []doc.Page{{No: 1, Chars: 100}, {No: 2, Chars: 100}, {No: 3, Chars: 100}, {No: 4, Chars: 100}}, + Runs: []doc.Run{}, + HasTextLayer: true, ContentStart: 1, ContentEnd: 4, + Regions: regions, + } +} + +func TestRegionsRoundTrip(t *testing.T) { + // Every field, because a region that survives with the wrong character count or + // the wrong box is worse than one that fails to save: it reads as authoritative. + // The coordinates are the column manual's real page 2 edges from + // testdata/fixtures/thomas-drybox-amfibia.json, with fractions added to check + // that they are rounded rather than truncated on the way in. + s := newService(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "a1") + + res := resultWith( + // A whole page: x0 = 0, x1 = the page width. No box means the whole page. + doc.Region{ + Page: 1, X0: 0, X1: 892.0, + Code: "UA", Lang: "uk", Source: doc.SourcePageTag, + Chars: 1700, Runs: 96, + Note: "the whole page is Ukrainian", + }, + // Three boxed columns on one page, .5 and .49 chosen so rounding and + // truncation give different answers. + doc.Region{ + Page: 2, X0: 42.5, X1: 305.4, + Code: "D", Lang: "de", Source: doc.SourceRepertoire, + Chars: 900, Runs: 50, + Note: "read from the characters the column uses", + }, + doc.Region{ + Page: 2, X0: 322.6, X1: 585.49, + Code: "PL", Lang: "pl", Source: doc.SourceRepertoire, + Chars: 880, Runs: 47, + }, + doc.Region{ + Page: 2, X0: 603.5, X1: 866.5, + Code: "RUS", Lang: "ru", Source: doc.SourceRepertoire, + Chars: 870, Runs: 46, + Conflict: true, + Note: "the printed tag and the alphabet disagreed", + }, + ) + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + got, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions: %v", err) + } + + want := []registry.Region{ + {Page: 1, X0: 0, X1: 892, Source: "page-tag", Code: "UA", Lang: "uk", Name: "Ukrainian", + Chars: 1700, Runs: 96, Note: "the whole page is Ukrainian"}, + {Page: 2, X0: 43, X1: 305, Source: "repertoire", Code: "D", Lang: "de", Name: "German", + Chars: 900, Runs: 50, Note: "read from the characters the column uses"}, + {Page: 2, X0: 323, X1: 585, Source: "repertoire", Code: "PL", Lang: "pl", Name: "Polish", + Chars: 880, Runs: 47}, + {Page: 2, X0: 604, X1: 867, Source: "repertoire", Code: "RUS", Lang: "ru", Name: "Russian", + Chars: 870, Runs: 46, Conflict: true, Note: "the printed tag and the alphabet disagreed"}, + } + if len(got) != len(want) { + t.Fatalf("got %d regions, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("region %d:\n got %+v\nwant %+v", i, got[i], want[i]) + } + } +} + +func TestTwoColumnsOfOneLanguageOnAPageBothSurvive(t *testing.T) { + // THIS IS THE BLOCKER doc_regions EXISTS TO FIX. Under doc_langs' key + // (document_id, source, code, pdf_start) two German columns on one page are the + // same row: same page, same code, same source, nothing to tell them apart, so one + // silently overwrites the other. Keying on geometry is what separates them, and + // the column manual really does set two columns of one language -- its manifest + // calls that out precisely because column count is not language count. + s := newService(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "b2") + + res := resultWith( + doc.Region{Page: 6, X0: 43, X1: 438, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 1200, Runs: 60, Note: "left column"}, + doc.Region{Page: 6, X0: 469, X1: 857, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 1150, Runs: 58, Note: "right column"}, + ) + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + got, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d regions on page 6, want 2 -- a same-language column was lost: %+v", len(got), got) + } + if got[0].X0 != 43 || got[1].X0 != 469 { + t.Errorf("columns are at x0 %d and %d, want 43 and 469", got[0].X0, got[1].X0) + } + if got[0].Note != "left column" || got[1].Note != "right column" { + t.Errorf("the two columns are not distinct: %q and %q", got[0].Note, got[1].Note) + } + // The characters must be summed across both, not taken from whichever was + // written last, or the page's size is half what it is. + if total := got[0].Chars + got[1].Chars; total != 2350 { + t.Errorf("page 6 holds %d characters, want 2350", total) + } +} + +func TestSavingTheSameProbeTwiceLeavesTheSameRegions(t *testing.T) { + // A probe job can run twice: a worker may die after doing the work but before + // recording success, and the reclaimed job runs again. The second run must + // converge on the same rows rather than duplicating them. Values, not just + // counts -- a converging count with a corrupted row would pass a count check. + s := newService(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "c3") + + res := resultWith( + doc.Region{Page: 2, X0: 43, X1: 305, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 900, Runs: 50}, + doc.Region{Page: 2, X0: 323, X1: 585, Code: "PL", Lang: "pl", + Source: doc.SourceRepertoire, Chars: 880, Runs: 47}, + doc.Region{Page: 3, X0: 0, X1: 892, Code: "", Lang: "", Source: "", + Chars: 120, Runs: 12, Note: "no language established for this page"}, + ) + + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save first probe: %v", err) + } + first, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions after first probe: %v", err) + } + + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save second probe: %v", err) + } + second, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions after second probe: %v", err) + } + + if len(first) != 3 { + t.Fatalf("first probe stored %d regions, want 3", len(first)) + } + if len(second) != len(first) { + t.Fatalf("re-probing changed the row count from %d to %d", len(first), len(second)) + } + for i := range first { + if first[i] != second[i] { + t.Errorf("region %d changed on re-probe:\nfirst %+v\nsecond %+v", i, first[i], second[i]) + } + } +} + +func TestAChangedRegionAttributionLeavesNoStaleRow(t *testing.T) { + // THIS IS THE CASE source-IN-THE-KEY WOULD OTHERWISE BREAK, and the reason + // SaveProbe deletes before inserting. + // + // internal/doc produces ONE resolved set of regions in which source merely + // records which signal named each one, and that attribution changes between + // probes: a column named by its alphabet on one run can be named by its printed + // tag on the next, because the tag reader's vocabulary comes from the document's + // own contents table and that parse can improve. Same document, same page, same + // x0, same column -- but a different primary key, so an upsert alone leaves the + // superseded row behind and the page reports itself twice. + // + // If this test still passes with the delete removed from SaveProbe, it is not + // testing what it claims. + s := newService(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "d4") + + named := doc.Region{ + Page: 7, X0: 43, X1: 305, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 900, Runs: 50, + Note: "read from the characters the column uses", + } + if err := s.SaveProbe(ctx, docID, resultWith(named), registry.StateAwaitingScope); err != nil { + t.Fatalf("save first probe: %v", err) + } + + // The same column, at the same x0 on the same page, now named by its printed tag. + reattributed := named + reattributed.Source = doc.SourcePageTag + reattributed.Note = "read from the tag printed on the column" + if err := s.SaveProbe(ctx, docID, resultWith(reattributed), registry.StateAwaitingScope); err != nil { + t.Fatalf("save re-probe: %v", err) + } + + got, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions: %v", err) + } + if len(got) != 1 { + t.Fatalf("got %d regions after re-attribution, want 1 -- the superseded row lingered "+ + "and page 7 now reports itself twice: %+v", len(got), got) + } + if got[0].Source != "page-tag" { + t.Errorf("source = %q, want page-tag: the newer attribution did not win", got[0].Source) + } + if got[0].Note != "read from the tag printed on the column" { + t.Errorf("note = %q, want the re-probe's", got[0].Note) + } +} + +func TestAProbeThatCouldNotReadRegionsLeavesThemIntact(t *testing.T) { + // RegionNote is set when positioned text could not be read at all: pdftohtml is + // absent or failed. poppler is optional at runtime here, so the same document can + // be probed with regions available and then without -- and deleting a good region + // map because a tool went missing from the host would be destructive. + // + // The distinction is RegionNote, not len(Regions): an empty note with no regions + // means the probe read the document and found none, which must replace. + s := newService(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "e5") + + stored := resultWith( + doc.Region{Page: 2, X0: 43, X1: 305, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 900, Runs: 50}, + doc.Region{Page: 2, X0: 323, X1: 585, Code: "PL", Lang: "pl", + Source: doc.SourceRepertoire, Chars: 880, Runs: 47}, + ) + if err := s.SaveProbe(ctx, docID, stored, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe with regions: %v", err) + } + + // A re-probe on a host without pdftohtml. The per-page language map is still + // complete; only the column resolution is missing. + blind := resultWith() + blind.RegionNote = "per-column languages are unavailable: pdftohtml not found" + if err := s.SaveProbe(ctx, docID, blind, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe without regions: %v", err) + } + + got, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d regions, want 2 -- a missing optional tool deleted good rows: %+v", len(got), got) + } + + // And the other half of the distinction: a probe that DID read the document and + // found nothing replaces, even though it also carries zero regions. + silent := resultWith() + if err := s.SaveProbe(ctx, docID, silent, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe that read nothing: %v", err) + } + got, err = s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions: %v", err) + } + if len(got) != 0 { + t.Errorf("got %d regions, want 0 -- a probe that read the document and found none "+ + "must replace a stale map, not hide behind it: %+v", len(got), got) + } +} + +func TestDeletingADocumentRemovesItsRegions(t *testing.T) { + // ON DELETE CASCADE. Without it doc_regions accumulates rows pointing at + // documents that no longer exist, and the FK clause is easy to copy without the + // cascade with nothing else noticing. + s, database := newServiceWithDB(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "f6") + + res := resultWith(doc.Region{ + Page: 2, X0: 43, X1: 305, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 900, Runs: 50, + }) + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + if got, err := s.Regions(ctx, docID); err != nil || len(got) != 1 { + t.Fatalf("regions before delete = %+v, %v; want 1 row", got, err) + } + + if err := gen.New(database.Write()).DeleteDocument(ctx, docID); err != nil { + t.Fatalf("delete document: %v", err) + } + if _, err := s.GetDocument(ctx, docID); !errors.Is(err, registry.ErrNotFound) { + t.Fatalf("document survived deletion: %v", err) + } + + got, err := s.Regions(ctx, docID) + if err != nil { + t.Fatalf("regions after delete: %v", err) + } + if len(got) != 0 { + t.Errorf("%d regions outlived their document: %+v", len(got), got) + } +} + +func TestRegionSummaryCountsCharactersNotPages(t *testing.T) { + // The summary is asserted against the database rather than through the service, + // because the arithmetic is in SQL and nothing in Go protects it. Two German + // columns on one page must sum to one language on one page, not two pages. + s, database := newServiceWithDB(t) + ctx := context.Background() + docID := newProbedDocument(t, s, "07") + + res := resultWith( + doc.Region{Page: 6, X0: 43, X1: 438, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 1200, Runs: 60}, + doc.Region{Page: 6, X0: 469, X1: 857, Code: "D", Lang: "de", + Source: doc.SourceRepertoire, Chars: 1150, Runs: 58}, + doc.Region{Page: 7, X0: 0, X1: 892, Code: "PL", Lang: "pl", + Source: doc.SourcePageTag, Chars: 2000, Runs: 100, Conflict: true}, + ) + if err := s.SaveProbe(ctx, docID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + rows, err := gen.New(database.Read()).SummarizeDocRegions(ctx, docID) + if err != nil { + t.Fatalf("summarize: %v", err) + } + if len(rows) != 2 { + t.Fatalf("got %d summary rows, want 2 (de and pl): %+v", len(rows), rows) + } + for i := range rows { + r := &rows[i] + switch r.Lang { + case "de": + if r.Chars != 2350 || r.Pages != 1 || r.Runs != 118 || r.Disputed != 0 { + t.Errorf("de summary = %+v; want chars 2350, pages 1, runs 118, disputed 0", r) + } + case "pl": + if r.Chars != 2000 || r.Pages != 1 || r.Disputed != 1 { + t.Errorf("pl summary = %+v; want chars 2000, pages 1, disputed 1", r) + } + default: + t.Errorf("unexpected summary row %+v", r) + } + } +} diff --git a/internal/registry/registry.go b/internal/registry/registry.go new file mode 100644 index 0000000..d0cbdf3 --- /dev/null +++ b/internal/registry/registry.go @@ -0,0 +1,263 @@ +// Package registry is the household inventory: where things are, what they are, +// and which documents belong to them. +// +// It deliberately holds no serial numbers and no purchase prices. Those are the +// highest-harm fields manualbox will store, they must be encrypted with a key +// kept outside the data directory, and the keyring is not wired into the schema +// yet. Adding them in the clear now would mean migrating real user data later. +// See docs/design/privacy.md. +package registry + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/gordon2/manualbox/internal/db" + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/id" +) + +var ( + // ErrNotFound is returned when an entity does not exist. + ErrNotFound = errors.New("not found") + // ErrInvalid is returned when input fails validation. + ErrInvalid = errors.New("invalid") +) + +// Service reads and writes the registry. +type Service struct { + db *db.DB + log *slog.Logger + now func() time.Time +} + +// Options configures [New]. +type Options struct { + Logger *slog.Logger + // Now overrides the clock, for tests. + Now func() time.Time +} + +// New returns a registry service. +func New(d *db.DB, opts Options) *Service { + if opts.Logger == nil { + opts.Logger = slog.New(slog.DiscardHandler) + } + if opts.Now == nil { + opts.Now = time.Now + } + return &Service{db: d, log: opts.Logger, now: opts.Now} +} + +// --- locations --- + +// Location is a place something is kept. Locations nest, so "Kitchen" can sit +// under "House". +type Location struct { + ID string `json:"id"` + Name string `json:"name"` + ParentID string `json:"parentId,omitempty"` + Notes string `json:"notes,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// CreateLocation adds a location. +func (s *Service) CreateLocation(ctx context.Context, name, parentID, notes string) (*Location, error) { + if name == "" { + return nil, fmt.Errorf("%w: a location needs a name", ErrInvalid) + } + now := db.Millis(s.now()) + row, err := gen.New(s.db.Write()).CreateLocation(ctx, gen.CreateLocationParams{ + ID: id.New(id.Location), + Name: name, + ParentID: nullString(parentID), + Notes: notes, + CreatedAt: now, + UpdatedAt: now, + }) + if err != nil { + return nil, fmt.Errorf("registry: create location: %w", err) + } + return locationFrom(row), nil +} + +// ListLocations returns every location, by name. +func (s *Service) ListLocations(ctx context.Context) ([]Location, error) { + rows, err := gen.New(s.db.Read()).ListLocations(ctx) + if err != nil { + return nil, fmt.Errorf("registry: list locations: %w", err) + } + out := make([]Location, 0, len(rows)) + for _, r := range rows { + out = append(out, *locationFrom(r)) + } + return out, nil +} + +// --- devices --- + +// Device is a thing the household owns. +type Device struct { + ID string `json:"id"` + Name string `json:"name"` + Brand string `json:"brand,omitempty"` + Model string `json:"model,omitempty"` + Category string `json:"category,omitempty"` + LocationID string `json:"locationId,omitempty"` + Notes string `json:"notes,omitempty"` + PurchasedAt *time.Time `json:"purchasedAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// NewDevice is the input to [Service.CreateDevice]. +type NewDevice struct { + Name string + Brand string + Model string + Category string + LocationID string + Notes string + PurchasedAt *time.Time +} + +// CreateDevice adds a device. +func (s *Service) CreateDevice(ctx context.Context, in NewDevice) (*Device, error) { + if in.Name == "" { + return nil, fmt.Errorf("%w: a device needs a name", ErrInvalid) + } + now := db.Millis(s.now()) + row, err := gen.New(s.db.Write()).CreateDevice(ctx, gen.CreateDeviceParams{ + ID: id.New(id.Device), + Name: in.Name, + Brand: in.Brand, + Model: in.Model, + Category: in.Category, + LocationID: nullString(in.LocationID), + Notes: in.Notes, + PurchasedAt: millisPtr(in.PurchasedAt), + CreatedAt: now, + UpdatedAt: now, + }) + if err != nil { + return nil, fmt.Errorf("registry: create device: %w", err) + } + return deviceFrom(row), nil +} + +// GetDevice returns one device. +func (s *Service) GetDevice(ctx context.Context, deviceID string) (*Device, error) { + row, err := gen.New(s.db.Read()).GetDevice(ctx, deviceID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: device %s", ErrNotFound, deviceID) + } + return nil, fmt.Errorf("registry: get device: %w", err) + } + return deviceFrom(row), nil +} + +// ListDevices returns every device, by name. +func (s *Service) ListDevices(ctx context.Context) ([]Device, error) { + rows, err := gen.New(s.db.Read()).ListDevices(ctx) + if err != nil { + return nil, fmt.Errorf("registry: list devices: %w", err) + } + out := make([]Device, 0, len(rows)) + for i := range rows { + out = append(out, *deviceFrom(rows[i])) + } + return out, nil +} + +// UpdateDevice replaces a device's editable fields. +func (s *Service) UpdateDevice(ctx context.Context, deviceID string, in NewDevice) (*Device, error) { + if in.Name == "" { + return nil, fmt.Errorf("%w: a device needs a name", ErrInvalid) + } + row, err := gen.New(s.db.Write()).UpdateDevice(ctx, gen.UpdateDeviceParams{ + Name: in.Name, + Brand: in.Brand, + Model: in.Model, + Category: in.Category, + LocationID: nullString(in.LocationID), + Notes: in.Notes, + PurchasedAt: millisPtr(in.PurchasedAt), + UpdatedAt: db.Millis(s.now()), + ID: deviceID, + }) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: device %s", ErrNotFound, deviceID) + } + return nil, fmt.Errorf("registry: update device: %w", err) + } + return deviceFrom(row), nil +} + +// DeleteDevice removes a device and, by cascade, its documents' rows. The blobs +// themselves are left alone: another device may reference the same bytes, and an +// original is the one thing that must never be lost by accident. +func (s *Service) DeleteDevice(ctx context.Context, deviceID string) error { + if err := gen.New(s.db.Write()).DeleteDevice(ctx, deviceID); err != nil { + return fmt.Errorf("registry: delete device: %w", err) + } + return nil +} + +// --- conversions --- + +func locationFrom(r gen.Location) *Location { + return &Location{ + ID: r.ID, + Name: r.Name, + ParentID: derefString(r.ParentID), + Notes: r.Notes, + CreatedAt: db.Time(r.CreatedAt), + UpdatedAt: db.Time(r.UpdatedAt), + } +} + +func deviceFrom(r gen.Device) *Device { + return &Device{ + ID: r.ID, + Name: r.Name, + Brand: r.Brand, + Model: r.Model, + Category: r.Category, + LocationID: derefString(r.LocationID), + Notes: r.Notes, + PurchasedAt: db.TimePtr(r.PurchasedAt), + CreatedAt: db.Time(r.CreatedAt), + UpdatedAt: db.Time(r.UpdatedAt), + } +} + +// nullString maps "" to NULL, so an unset optional reference is stored as absent +// rather than as an empty string that satisfies no foreign key. +func nullString(s string) *string { + if s == "" { + return nil + } + return &s +} + +func derefString(s *string) string { + if s == nil { + return "" + } + return *s +} + +func millisPtr(t *time.Time) *int64 { + if t == nil { + return nil + } + ms := db.Millis(*t) + return &ms +} diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go new file mode 100644 index 0000000..29e33ee --- /dev/null +++ b/internal/registry/registry_test.go @@ -0,0 +1,435 @@ +package registry_test + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/gordon2/manualbox/internal/db" + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +func newService(t *testing.T) *registry.Service { + t.Helper() + database, err := db.Open(context.Background(), db.Options{ + Path: filepath.Join(t.TempDir(), "registry.db"), + }) + if err != nil { + t.Fatalf("open database: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + return registry.New(database, registry.Options{}) +} + +func TestDeviceRoundTrip(t *testing.T) { + s := newService(t) + ctx := context.Background() + + purchased := time.Date(2026, 3, 14, 0, 0, 0, 0, time.UTC) + created, err := s.CreateDevice(ctx, registry.NewDevice{ + Name: "Dishwasher", Brand: "Bosch", Model: "SMS4H", PurchasedAt: &purchased, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + got, err := s.GetDevice(ctx, created.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Name != "Dishwasher" || got.Brand != "Bosch" { + t.Errorf("round-tripped device = %+v", got) + } + if got.PurchasedAt == nil || !got.PurchasedAt.Equal(purchased) { + t.Errorf("purchasedAt = %v, want %v", got.PurchasedAt, purchased) + } + if !strings.HasPrefix(got.ID, "dev_") { + t.Errorf("id = %q, want a dev_ prefix", got.ID) + } +} + +func TestDeviceNeedsAName(t *testing.T) { + s := newService(t) + if _, err := s.CreateDevice(context.Background(), registry.NewDevice{}); !errors.Is(err, registry.ErrInvalid) { + t.Errorf("error = %v, want ErrInvalid", err) + } +} + +func TestMissingDeviceIsNotFound(t *testing.T) { + s := newService(t) + if _, err := s.GetDevice(context.Background(), "dev_nope"); !errors.Is(err, registry.ErrNotFound) { + t.Errorf("error = %v, want ErrNotFound", err) + } +} + +func TestUnsetLocationIsNullNotEmptyString(t *testing.T) { + // An empty string satisfies no foreign key, so an unset optional reference has + // to be stored as NULL. Getting this wrong makes device creation fail only + // once a location table exists to violate. + s := newService(t) + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: "Kettle"}) + if err != nil { + t.Fatalf("create device with no location: %v", err) + } + if device.LocationID != "" { + t.Errorf("locationId = %q, want empty", device.LocationID) + } + + location, err := s.CreateLocation(ctx, "Kitchen", "", "") + if err != nil { + t.Fatalf("create location: %v", err) + } + updated, err := s.UpdateDevice(ctx, device.ID, registry.NewDevice{ + Name: "Kettle", LocationID: location.ID, + }) + if err != nil { + t.Fatalf("assign location: %v", err) + } + if updated.LocationID != location.ID { + t.Errorf("locationId = %q, want %q", updated.LocationID, location.ID) + } +} + +func TestDeletingADeviceRemovesItsDocuments(t *testing.T) { + // Cascade is what keeps the document table from accumulating rows pointing at + // devices that no longer exist. The blobs are deliberately left alone. + s := newService(t) + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: "Robot vacuum"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat("ab", 32), Size: 1024} + if err := s.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, Filename: "manual.pdf", + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + + if err := s.DeleteDevice(ctx, device.ID); err != nil { + t.Fatalf("delete device: %v", err) + } + if _, err := s.GetDocument(ctx, document.ID); !errors.Is(err, registry.ErrNotFound) { + t.Errorf("document survived its device: %v", err) + } +} + +func TestDocumentDefaultsToAManual(t *testing.T) { + s := newService(t) + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: "Oven"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat("cd", 32), Size: 10} + if err := s.RecordBlob(ctx, ref, ""); err != nil { + t.Fatalf("record blob: %v", err) + } + + document, created, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + if !created { + t.Error("first insert reported itself as a duplicate") + } + if document.Kind != registry.KindManual { + t.Errorf("kind = %q, want %q", document.Kind, registry.KindManual) + } + if document.State != registry.StateUploaded { + t.Errorf("state = %q, want %q", document.State, registry.StateUploaded) + } + if document.Probed() { + t.Error("a freshly created document claims to have been probed") + } +} + +func TestSaveProbeReplacesRatherThanAccumulates(t *testing.T) { + // Re-probing must converge on one answer. A run the newest probe no longer + // believes in has to disappear, not linger beside its replacement — otherwise a + // corrected boundary shows up as two overlapping claims. + s := newService(t) + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: "Washer"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat("ef", 32), Size: 10} + if err := s.RecordBlob(ctx, ref, ""); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + + first := &doc.Result{ + Info: doc.Info{Pages: 4}, + Pages: []doc.Page{{No: 1, Chars: 100}, {No: 2, Chars: 100}, {No: 3, Chars: 100}, {No: 4, Chars: 100}}, + Runs: []doc.Run{ + {Source: doc.SourceReconciled, Code: "EN", Lang: "en", Start: 1, End: 2}, + {Source: doc.SourceReconciled, Code: "DE", Lang: "de", Start: 3, End: 4}, + }, + HasTextLayer: true, ContentStart: 1, ContentEnd: 4, + } + if err := s.SaveProbe(ctx, document.ID, first, registry.StateAwaitingScope); err != nil { + t.Fatalf("save first probe: %v", err) + } + + // A second probe that finds one language where the first found two. + second := &doc.Result{ + Info: doc.Info{Pages: 4}, + Pages: first.Pages, + Runs: []doc.Run{ + {Source: doc.SourceReconciled, Code: "EN", Lang: "en", Start: 1, End: 4}, + }, + HasTextLayer: true, ContentStart: 1, ContentEnd: 4, + } + if err := s.SaveProbe(ctx, document.ID, second, registry.StateAwaitingScope); err != nil { + t.Fatalf("save second probe: %v", err) + } + + runs, err := s.LanguageRuns(ctx, document.ID, doc.SourceReconciled) + if err != nil { + t.Fatalf("language runs: %v", err) + } + if len(runs) != 1 { + t.Fatalf("got %d runs after re-probing, want 1: %+v", len(runs), runs) + } + if runs[0].Start != 1 || runs[0].End != 4 { + t.Errorf("run = %d-%d, want 1-4", runs[0].Start, runs[0].End) + } + + after, err := s.GetDocument(ctx, document.ID) + if err != nil { + t.Fatalf("get document: %v", err) + } + if !after.Probed() { + t.Error("document is not marked as probed") + } + if after.PageCount == nil || *after.PageCount != 4 { + t.Errorf("pageCount = %v, want 4", after.PageCount) + } +} + +func TestUnplaceableLanguageClaimsAreStored(t *testing.T) { + // A printed index routinely names a language but points at a page that cannot + // be right — a real manual lists Czech at a page that is Arabic. The claim is + // still evidence worth keeping: it is how a user learns their contents table is + // wrong. Such runs carry a start of 0, and several of them can coexist, so + // neither the CHECK nor the primary key may reject them. + // + // Both mistakes were made: a CHECK of pdf_start >= 1 failed the whole probe on + // a real document, and keying on the page alone would have collapsed every + // unplaceable claim into one row. + s := newService(t) + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: "Vacuum"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat("34", 32), Size: 10} + if err := s.RecordBlob(ctx, ref, ""); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + + res := &doc.Result{ + Info: doc.Info{Pages: 2}, + Pages: []doc.Page{{No: 1, Chars: 100}, {No: 2, Chars: 100}}, + HasTextLayer: true, + ContentStart: 1, ContentEnd: 2, + BySource: map[doc.Source][]doc.Run{ + doc.SourceIndex: { + {Source: doc.SourceIndex, Code: "EN", Lang: "en", Start: 1, End: 2}, + // Two different languages the index named but could not place. + {Source: doc.SourceIndex, Code: "CZ", Lang: "cs", Start: 0, End: 0, + Note: "printed index claims page 207, which is Arabic script"}, + {Source: doc.SourceIndex, Code: "PT", Lang: "pt", Start: 0, End: 0, + Note: "printed index claims a page this document does not print"}, + }, + }, + Runs: []doc.Run{{Source: doc.SourceReconciled, Code: "EN", Lang: "en", Start: 1, End: 2}}, + } + + if err := s.SaveProbe(ctx, document.ID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe with unplaceable claims: %v", err) + } + + runs, err := s.LanguageRuns(ctx, document.ID, doc.SourceIndex) + if err != nil { + t.Fatalf("language runs: %v", err) + } + if len(runs) != 3 { + t.Fatalf("got %d index runs, want 3 — an unplaceable claim was dropped: %+v", len(runs), runs) + } + + unplaceable := 0 + for i := range runs { + if runs[i].Start == 0 { + unplaceable++ + if runs[i].Note == "" { + t.Errorf("%s has no boundary and no explanation", runs[i].Code) + } + } + } + if unplaceable != 2 { + t.Errorf("got %d unplaceable claims, want 2 (CZ and PT kept separately)", unplaceable) + } +} + +func TestLanguageRunsCarryADisplayName(t *testing.T) { + // The UI shows "Ukrainian", not "uk", and the manual's own label may be neither. + s := newService(t) + ctx := context.Background() + + device, _ := s.CreateDevice(ctx, registry.NewDevice{Name: "Vacuum"}) + ref := store.Ref{SHA256: strings.Repeat("12", 32), Size: 10} + if err := s.RecordBlob(ctx, ref, ""); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + + res := &doc.Result{ + Info: doc.Info{Pages: 2}, + Pages: []doc.Page{{No: 1, Chars: 100}, {No: 2, Chars: 100}}, + Runs: []doc.Run{ + // The document prints UA; the tag is uk. + {Source: doc.SourceReconciled, Code: "UA", Lang: "uk", Start: 1, End: 2}, + }, + HasTextLayer: true, ContentStart: 1, ContentEnd: 2, + } + if err := s.SaveProbe(ctx, document.ID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + runs, err := s.LanguageRuns(ctx, document.ID, doc.SourceReconciled) + if err != nil { + t.Fatalf("language runs: %v", err) + } + if len(runs) != 1 { + t.Fatalf("got %d runs, want 1", len(runs)) + } + if runs[0].Code != "UA" || runs[0].Lang != "uk" || runs[0].Name != "Ukrainian" { + t.Errorf("run = code %q, lang %q, name %q; want UA/uk/Ukrainian", + runs[0].Code, runs[0].Lang, runs[0].Name) + } + if runs[0].Pages != 2 { + t.Errorf("pages = %d, want 2", runs[0].Pages) + } +} + +func TestAnUnplaceableClaimCoversNoPages(t *testing.T) { + // A run with pdf_start = 0 named a language it could not place, which the schema + // documents as a real state. Both the API and the summary query measured it as + // pdf_end - pdf_start + 1, so GET /documents/{id}/languages?source=index reported + // the fixture's unplaceable Arabic entry as a one-page section spanning 0-0 — a + // section of a language the manual's contents table merely mentions. + dir := t.TempDir() + database, err := db.Open(context.Background(), db.Options{Path: filepath.Join(dir, "registry.db")}) + if err != nil { + t.Fatalf("open database: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + s := registry.New(database, registry.Options{}) + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: "Robot vacuum"}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat("56", 32), Size: 10} + if err := s.RecordBlob(ctx, ref, ""); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + + res := &doc.Result{ + Info: doc.Info{Pages: 2}, + Pages: []doc.Page{{No: 1, Chars: 100}, {No: 2, Chars: 100}}, + HasTextLayer: true, + ContentStart: 1, ContentEnd: 2, + BySource: map[doc.Source][]doc.Run{ + doc.SourceIndex: { + {Source: doc.SourceIndex, Code: "EN", Lang: "en", Start: 1, End: 2}, + {Source: doc.SourceIndex, Code: "AR", Lang: "ar", Start: 0, End: 0, + Note: "printed index claims a page this document does not print"}, + }, + }, + Runs: []doc.Run{{Source: doc.SourceReconciled, Code: "EN", Lang: "en", Start: 1, End: 2}}, + } + if err := s.SaveProbe(ctx, document.ID, res, registry.StateAwaitingScope); err != nil { + t.Fatalf("save probe: %v", err) + } + + runs, err := s.LanguageRuns(ctx, document.ID, doc.SourceIndex) + if err != nil { + t.Fatalf("language runs: %v", err) + } + for i := range runs { + want := 2 + if runs[i].Code == "AR" { + want = 0 + } + if runs[i].Pages != want { + t.Errorf("%s covers %d pages, want %d", runs[i].Code, runs[i].Pages, want) + } + } + + // The summary query does the same arithmetic in SQL, and nothing in Go protects + // it, so it is asserted against the database rather than through the service. + rows, err := gen.New(database.Read()).SummarizeDocLangs(ctx, gen.SummarizeDocLangsParams{ + DocumentID: document.ID, + Source: string(doc.SourceIndex), + }) + if err != nil { + t.Fatalf("summarize: %v", err) + } + if len(rows) != 2 { + t.Fatalf("got %d summary rows, want 2: %+v", len(rows), rows) + } + for i := range rows { + want := int64(2) + if rows[i].Code == "AR" { + want = 0 + } + if rows[i].Pages != want { + t.Errorf("summary says %s covers %d pages, want %d", rows[i].Code, rows[i].Pages, want) + } + } +} diff --git a/internal/registry/search.go b/internal/registry/search.go new file mode 100644 index 0000000..f199e23 --- /dev/null +++ b/internal/registry/search.go @@ -0,0 +1,302 @@ +package registry + +import ( + "context" + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "github.com/gordon2/manualbox/internal/db/gen" + "github.com/gordon2/manualbox/internal/doc" +) + +// trigramMin is the shortest query the index can answer, and it is a property of +// the tokeniser rather than a policy: a trigram index holds no token shorter than +// three characters, so a shorter query matches nothing at all. See +// 00006_block_search.sql for what that costs and why it is still the right +// tokeniser. +const trigramMin = 3 + +// Search modes, reported on every result so a caller can tell which question was +// actually answered. +const ( + // SearchIndex is the FTS5 index: bm25 ranking, substring matching within a + // block, every script the corpus holds. + SearchIndex = "index" + // SearchSubstring is the scan that answers a query the index cannot represent + // -- one shorter than three characters in any of its words. Case folding there + // is SQLite's own lower(), which is ASCII only. + SearchSubstring = "substring" +) + +// Default and maximum result counts. The default is a screenful; the cap is what +// stops a client asking for the whole corpus one query at a time, since every hit +// carries a snippet and a device name. +const ( + DefaultSearchLimit = 25 + MaxSearchLimit = 100 +) + +// SearchQuery is one search: what to look for, and how much of the household to +// look in. +type SearchQuery struct { + // Text is what the user typed, unmodified. Turning it into an FTS5 expression + // is [Service.Search]'s business and deliberately not a caller's: an API that + // accepted FTS5 syntax would make every quote, asterisk and colon in an + // ordinary query a syntax error. + Text string + // DocumentID narrows the search to one manual. Empty searches every one of + // them, which is the question README poses -- "which manual says X". + DocumentID string + // Limit caps the hits. Zero means [DefaultSearchLimit]; anything above + // [MaxSearchLimit] is clamped to it rather than rejected, because a client + // asking for too much wants as much as it can have. + Limit int +} + +// Hit is one match: which manual, which page, which language, and enough text to +// recognise it. +// +// Page, RegionX0 and Index together are the block's natural key, the same one +// doc_blocks stores and conversion.md specifies as a citation -- so a hit can be +// deep-linked to the exact paragraph it came from and will still point there after +// a re-conversion. +type Hit struct { + DocumentID string `json:"documentId"` + // Filename and DeviceName are what a household recognises. "Page 47" without + // them answers a different, useless question. + Filename string `json:"filename,omitempty"` + DeviceID string `json:"deviceId"` + DeviceName string `json:"deviceName"` + // State is the document's pipeline state, so a hit from a manual that is + // mid-re-conversion is visible as such rather than looking stale. + State string `json:"state"` + + Page int `json:"page"` + RegionX0 int `json:"regionX0"` + Index int `json:"index"` + + Kind string `json:"kind"` + Level int `json:"level,omitempty"` + // Lang is the block's language and Name that for a person to read, the same + // pairing [Block] uses: the UI shows "Japanese", not "ja". + Lang string `json:"lang,omitempty"` + Name string `json:"name,omitempty"` + + // Snippet is the text around the match, about 64 characters of it. Chars is + // the whole block's rune count, so a caller can tell a snippet from a complete + // block and fetch the rest through the conversion endpoint. + Snippet string `json:"snippet"` + Chars int `json:"chars"` + + // BM25 is what FTS5 scored the match, and Score is that with the heading bonus + // applied -- the number the results are ordered by. Both are reported because + // the bonus is a judgement: a heading names a section and is a better answer to + // "where does it say this" than a passing mention, and anyone who disagrees can + // see exactly how much it moved. Both are 0 in [SearchSubstring] mode, where + // there is no index term to weigh. + BM25 float64 `json:"bm25"` + Score float64 `json:"score"` +} + +// SearchResults are the hits plus what was actually asked. +type SearchResults struct { + // Query is the text as typed, echoed so a client rendering a result list does + // not have to keep its own copy in step. + Query string `json:"query"` + // Mode is [SearchIndex] or [SearchSubstring]. It is reported rather than + // hidden because the two paths differ in ways a user can see: only the index + // ranks, and only the scan can answer a one or two character query. + Mode string `json:"mode"` + Limit int `json:"limit"` + // Truncated says the limit cut the results off, which is the difference + // between "these are the hits" and "these are the first hits". + Truncated bool `json:"truncated"` + Hits []Hit `json:"hits"` + // Indexed is how many blocks exist to search, and it is filled in only when + // nothing matched. "No results" and "nothing has been converted yet" look + // identical otherwise, and the second is not a search failure -- it is the same + // distinction [Service.Blocks] makes between empty and absent. + Indexed *int `json:"indexed,omitempty"` +} + +// Search answers "which manual says X, and where" across every converted document +// in the household, or within one of them. +// +// # Why the query is not passed through +// +// FTS5 has an expression syntax: unquoted AND, OR, NOT, NEAR, column filters with +// a colon, prefix stars, and quoted phrases. A search box that handed a user's text +// to it directly would fail on an apostrophe-free but perfectly ordinary query like +// `filter: reinigen` and would silently reinterpret `Motor NOT laufen`. So every +// word is quoted as a phrase and the phrases are ANDed: the query means "a block +// containing all of these", which is what a person typing two words means. +// +// # Why a short word sends the whole query to the scan +// +// The index cannot answer a word shorter than three characters at all, so a query +// mixing "Filter" with "ab" would quietly become a search for "Filter" alone -- an +// answer to a question nobody asked, and indistinguishable from a correct one. The +// rule is therefore all or nothing: every word long enough goes to the index, and +// otherwise the whole query, spaces included, is one literal substring for the scan +// to look for. Mode says which happened. +func (s *Service) Search(ctx context.Context, q SearchQuery) (*SearchResults, error) { + text := strings.TrimSpace(q.Text) + if text == "" { + return nil, fmt.Errorf("%w: a search needs something to look for", ErrInvalid) + } + + limit := q.Limit + if limit <= 0 { + limit = DefaultSearchLimit + } + if limit > MaxSearchLimit { + limit = MaxSearchLimit + } + + res := &SearchResults{Query: text, Limit: limit, Mode: SearchSubstring, Hits: []Hit{}} + rq := gen.New(s.db.Read()) + + // One more than asked for, and the extra is thrown away. It is the only way to + // tell "these are the hits" from "these are the first hits": a result set of + // exactly the limit is ambiguous, and reporting it as truncated would put "there + // is more" on every complete answer that happens to fill the page. + fetch := int64(limit) + 1 + + var ( + rows []gen.SearchBlocksRow + err error + ) + if match, ok := matchExpression(text); ok { + res.Mode = SearchIndex + if q.DocumentID != "" { + var narrowed []gen.SearchBlocksInDocumentRow + narrowed, err = rq.SearchBlocksInDocument(ctx, gen.SearchBlocksInDocumentParams{ + Match: match, DocumentID: q.DocumentID, Limit: fetch, + }) + rows = narrowRows(narrowed) + } else { + rows, err = rq.SearchBlocks(ctx, gen.SearchBlocksParams{ + Match: match, Limit: fetch, + }) + } + } else if q.DocumentID != "" { + var scanned []gen.SearchBlocksSubstringInDocumentRow + scanned, err = rq.SearchBlocksSubstringInDocument(ctx, + gen.SearchBlocksSubstringInDocumentParams{ + Needle: text, DocumentID: q.DocumentID, Limit: fetch, + }) + rows = substringInDocumentRows(scanned) + } else { + var scanned []gen.SearchBlocksSubstringRow + scanned, err = rq.SearchBlocksSubstring(ctx, gen.SearchBlocksSubstringParams{ + Needle: text, Limit: fetch, + }) + rows = substringRows(scanned) + } + if err != nil { + return nil, fmt.Errorf("registry: search %q: %w", text, err) + } + + res.Truncated = len(rows) > limit + if res.Truncated { + rows = rows[:limit] + } + res.Hits = hitsFrom(rows) + if len(res.Hits) == 0 { + n, err := rq.CountSearchableBlocks(ctx) + if err != nil { + return nil, fmt.Errorf("registry: count searchable blocks: %w", err) + } + indexed := int(n) + res.Indexed = &indexed + } + return res, nil +} + +// matchExpression turns a user's text into an FTS5 expression, reporting false +// when the index cannot answer it. +// +// Every word becomes a quoted phrase, which is the only FTS5 construct with no +// syntax inside it beyond the quote character itself -- and a quote is escaped by +// doubling. The phrases are ANDed rather than joined into one phrase, so "Filter +// reinigen" finds a block that says both without demanding they be adjacent. +// +// false means at least one word is shorter than [trigramMin] and the whole query +// belongs on the scan. See [Service.Search] for why one short word disqualifies the +// query rather than being dropped from it. +func matchExpression(text string) (string, bool) { + words := strings.FieldsFunc(text, unicode.IsSpace) + if len(words) == 0 { + return "", false + } + quoted := make([]string, 0, len(words)) + for _, w := range words { + if utf8.RuneCountInString(w) < trigramMin { + return "", false + } + quoted = append(quoted, `"`+strings.ReplaceAll(w, `"`, `""`)+`"`) + } + return strings.Join(quoted, " AND "), true +} + +// The four generated row types are structurally identical -- the queries differ in +// their WHERE clause, not in what they return -- but sqlc emits a distinct type per +// statement, so each is converted to the one the mapper reads. Written out rather +// than reached through reflection or an interface: four small functions that the +// compiler checks against the generated types are what catches a column added to +// one statement and not the others. + +func narrowRows(in []gen.SearchBlocksInDocumentRow) []gen.SearchBlocksRow { + out := make([]gen.SearchBlocksRow, 0, len(in)) + for i := range in { + r := &in[i] + out = append(out, gen.SearchBlocksRow(*r)) + } + return out +} + +func substringRows(in []gen.SearchBlocksSubstringRow) []gen.SearchBlocksRow { + out := make([]gen.SearchBlocksRow, 0, len(in)) + for i := range in { + r := &in[i] + out = append(out, gen.SearchBlocksRow(*r)) + } + return out +} + +func substringInDocumentRows(in []gen.SearchBlocksSubstringInDocumentRow) []gen.SearchBlocksRow { + out := make([]gen.SearchBlocksRow, 0, len(in)) + for i := range in { + r := &in[i] + out = append(out, gen.SearchBlocksRow(*r)) + } + return out +} + +func hitsFrom(rows []gen.SearchBlocksRow) []Hit { + out := make([]Hit, 0, len(rows)) + for i := range rows { + r := &rows[i] + out = append(out, Hit{ + DocumentID: r.DocumentID, + Filename: r.Filename, + DeviceID: r.DeviceID, + DeviceName: r.DeviceName, + State: r.State, + Page: int(r.Page), + RegionX0: int(r.RegionX0), + Index: int(r.Idx), + Kind: r.Kind, + Level: int(r.Level), + Lang: r.Lang, + Name: doc.DisplayName(r.Lang), + Snippet: r.Snippet, + Chars: int(r.Chars), + BM25: r.Bm25, + Score: r.Score, + }) + } + return out +} diff --git a/internal/registry/search_fixture_test.go b/internal/registry/search_fixture_test.go new file mode 100644 index 0000000..66e7f76 --- /dev/null +++ b/internal/registry/search_fixture_test.go @@ -0,0 +1,108 @@ +package registry_test + +import ( + "context" + "os" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" + "github.com/gordon2/manualbox/internal/registry" +) + +// TestHebrewIsFoundTypedForwards is the one search question the corpus could not +// answer, and it is a search test only in where it fails: the index was always +// right and what it held was backwards. +// +// docs/design/search.md recorded the hole as measured — the word for "manual" was +// "findable by a query typed backwards (5 blocks) and not by one a Hebrew speaker +// would type (0 blocks)" — and named it as extraction's, not the index's. That is +// exactly what doc/bidi.go fixed, and the measurement is now the exact inverse of +// what that sentence describes: **5 blocks forwards and 0 backwards**. +// +// It got there in two steps and the middle one is worth keeping, because it is why +// this test asserts a page number. When the repair first landed it read 4 and 1: one +// block was still stored backwards, page 188, where the support URL and a Hebrew +// sentence share a line and doc's lineIsRightToLeft gave the line to its Latin +// majority. verify reported the same page from the other side off a comparison that +// shares no code with this one. Letting the region's language decide direction closed +// both at once, and page 188 now reads +// `למדריך אלקטרוני מפורט, יש לעיין בכתובת הבאה: https://…`. +// +// It converts the document for Hebrew alone rather than for the household of 34, +// because one language is all this question needs and it is the whole cost. +func TestHebrewIsFoundTypedForwards(t *testing.T) { + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixture and run the real-document tests", + fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFInfo, extern.PDFToText, extern.PDFToHTML} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + + ctx := context.Background() + manifest, err := fixture.Load(fixturesDir, "dreame-l40-ultra") + if err != nil { + t.Fatalf("load manifest: %v", err) + } + path, err := manifest.Fetch(ctx) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + res, err := doc.Analyze(ctx, path) + if err != nil { + t.Fatalf("analyze: %v", err) + } + conv, err := doc.Convert(ctx, path, res, []string{"he"}, doc.ConvertOptions{}) + if err != nil { + t.Fatalf("convert: %v", err) + } + t.Logf("Hebrew conversion: %s", conv.Summary()) + + s := newService(t) + docID := newDocumentOnDevice(t, s, "Robot vacuum", "dreame-l40-ultra.pdf", "a") + if err := s.SaveConversion(ctx, docID, conv.Blocks, nil, nil, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } + + // מדריך, "manual" — the word search.md measured the hole on. Its own reverse is + // ךירדמ, which is what the index used to hold and what a Hebrew speaker would + // never type. + const ( + forwards = "מדריך" + backwards = "ךירדמ" + ) + fw := search(t, s, registry.SearchQuery{Text: forwards}) + bw := search(t, s, registry.SearchQuery{Text: backwards}) + t.Logf("%q found %d block(s) by %s, %q found %d", forwards, len(fw.Hits), fw.Mode, + backwards, len(bw.Hits)) + for i := range fw.Hits { + t.Logf(" forwards page %d: %s", fw.Hits[i].Page, fw.Hits[i].Snippet) + } + for i := range bw.Hits { + t.Logf(" backwards page %d: %s", bw.Hits[i].Page, bw.Hits[i].Snippet) + } + + // The number that matters is that this is not zero. It was zero, for every Hebrew + // word, and search.md said so. + if len(fw.Hits) != 5 { + t.Errorf("%q found %d block(s) of the Hebrew section, want 5 — it found 0 "+ + "before doc/bidi.go and 4 while a line decided its own direction, and it "+ + "is the hole search.md records", forwards, len(fw.Hits)) + } + // And nothing is stored backwards any more. Naming the page in the failure is the + // point: page 188 was the last one, so if this comes back it says whether the same + // line lost ground or a new one did. + if len(bw.Hits) != 0 { + for i := range bw.Hits { + t.Errorf("still backwards on page %d: %s", bw.Hits[i].Page, bw.Hits[i].Snippet) + } + t.Errorf("the word typed backwards finds %d block(s), want 0 — it found 5 "+ + "before doc/bidi.go and 1, on page 188, while a line's own characters "+ + "decided its direction", len(bw.Hits)) + } +} diff --git a/internal/registry/search_test.go b/internal/registry/search_test.go new file mode 100644 index 0000000..3859b10 --- /dev/null +++ b/internal/registry/search_test.go @@ -0,0 +1,571 @@ +package registry_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/registry" + "github.com/gordon2/manualbox/internal/store" +) + +// These tests are about the index, not about conversion: they store blocks +// directly and then ask the questions a household asks. The text is real text from +// the two measured manuals wherever the script matters, because the whole tokeniser +// decision turns on scripts that a made-up ASCII fixture cannot represent. + +// newDocumentOnDevice is newProbedDocument with the device named, because a search +// hit has to say WHICH manual and the device's name is half of that. +func newDocumentOnDevice(t *testing.T, s *registry.Service, deviceName, filename, digest string) string { + t.Helper() + ctx := context.Background() + + device, err := s.CreateDevice(ctx, registry.NewDevice{Name: deviceName}) + if err != nil { + t.Fatalf("create device: %v", err) + } + ref := store.Ref{SHA256: strings.Repeat(digest, 32), Size: 10} + if err := s.RecordBlob(ctx, ref, "application/pdf"); err != nil { + t.Fatalf("record blob: %v", err) + } + document, _, err := s.CreateDocument(ctx, registry.NewDocument{ + DeviceID: device.ID, BlobSHA256: ref.SHA256, Filename: filename, + }) + if err != nil { + t.Fatalf("create document: %v", err) + } + return document.ID +} + +// block is one stored block with only the fields search reads set. +func block(page int, x0 float64, index int, kind doc.BlockKind, lang, text string) doc.Block { + return doc.Block{ + Page: page, RegionX0: x0, Index: index, + Kind: kind, Text: text, Lang: lang, + X0: x0, X1: x0 + 200, Y0: 100, Y1: 118, + Lines: 1, Chars: len([]rune(text)), + } +} + +func heading(page int, x0 float64, index int, lang, text string) doc.Block { + b := block(page, x0, index, doc.BlockHeading, lang, text) + b.Level = 2 + return b +} + +// save stores blocks as a conversion, which is the only way they ever arrive. +func save(t *testing.T, s *registry.Service, docID string, blocks ...doc.Block) { + t.Helper() + if err := s.SaveConversion(context.Background(), docID, blocks, nil, nil, + registry.StateReady); err != nil { + t.Fatalf("save conversion: %v", err) + } +} + +func search(t *testing.T, s *registry.Service, q registry.SearchQuery) *registry.SearchResults { + t.Helper() + res, err := s.Search(context.Background(), q) + if err != nil { + t.Fatalf("search %q: %v", q.Text, err) + } + return res +} + +// TestASearchHitSaysWhichManualAndWhere is the acceptance criterion, in the words +// README uses: the paper pile is unsearchable, and knowing that something matched +// is not the answer. A hit must name the document, the device, the page and the +// language, and carry enough text to recognise. +func TestASearchHitSaysWhichManualAndWhere(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "thomas-drybox.pdf", "a") + + save(t, s, docID, + heading(48, 43, 0, "de", "Ausblasfilter austauschen"), + block(48, 43, 1, doc.BlockParagraph, "de", + "Tauschen Sie den Spezial-Hygiene-Filter alle zwei Jahre aus."), + ) + + res := search(t, s, registry.SearchQuery{Text: "Ausblasfilter"}) + if res.Mode != registry.SearchIndex { + t.Errorf("mode = %q, want %q", res.Mode, registry.SearchIndex) + } + if len(res.Hits) != 1 { + t.Fatalf("got %d hits, want 1: %+v", len(res.Hits), res.Hits) + } + got := res.Hits[0] + if got.DocumentID != docID { + t.Errorf("documentId = %q, want %q", got.DocumentID, docID) + } + if got.Filename != "thomas-drybox.pdf" || got.DeviceName != "Vacuum cleaner" { + t.Errorf("hit names %q on %q; a household recognises the file and the device", + got.Filename, got.DeviceName) + } + if got.Page != 48 { + t.Errorf("page = %d, want 48", got.Page) + } + // The citation conversion.md specifies: the page, the region's left edge and the + // index within it. Without these a hit cannot be deep-linked to the paragraph. + if got.RegionX0 != 43 || got.Index != 0 { + t.Errorf("regionX0/index = %d/%d, want 43/0", got.RegionX0, got.Index) + } + if got.Lang != "de" || got.Name != "German" { + t.Errorf("lang/name = %q/%q, want de/German", got.Lang, got.Name) + } + if got.Kind != "heading" || got.Level != 2 { + t.Errorf("kind/level = %q/%d, want heading/2", got.Kind, got.Level) + } + if !strings.Contains(got.Snippet, "Ausblasfilter") { + t.Errorf("snippet %q does not contain the word searched for", got.Snippet) + } + if got.State != registry.StateReady { + t.Errorf("state = %q, want %q", got.State, registry.StateReady) + } + if res.Indexed != nil { + t.Errorf("indexed = %d on a search that matched; it is only for an empty result", + *res.Indexed) + } +} + +// TestSearchSpansDocumentsAndCanBeNarrowedToOne is the scope decision. "Which +// manual says X" is a question about the household, so the default is every +// document; a reader already inside one asks the narrower question. +func TestSearchSpansDocumentsAndCanBeNarrowedToOne(t *testing.T) { + s := newService(t) + vacuum := newDocumentOnDevice(t, s, "Vacuum cleaner", "vacuum.pdf", "a") + washer := newDocumentOnDevice(t, s, "Washing machine", "washer.pdf", "b") + + save(t, s, vacuum, block(12, 43, 0, doc.BlockParagraph, "de", + "Den Filter alle drei Monate reinigen.")) + save(t, s, washer, block(7, 0, 0, doc.BlockParagraph, "de", + "Den Flusenfilter nach jedem Waschgang reinigen.")) + + all := search(t, s, registry.SearchQuery{Text: "Filter"}) + if len(all.Hits) != 2 { + t.Fatalf("searching every manual got %d hits, want 2: %+v", len(all.Hits), all.Hits) + } + seen := map[string]bool{} + for i := range all.Hits { + seen[all.Hits[i].DeviceName] = true + } + if !seen["Vacuum cleaner"] || !seen["Washing machine"] { + t.Errorf("hits came from %v, want both devices", seen) + } + + one := search(t, s, registry.SearchQuery{Text: "Filter", DocumentID: washer}) + if len(one.Hits) != 1 || one.Hits[0].DocumentID != washer { + t.Fatalf("narrowed search got %+v, want the one washer hit", one.Hits) + } + + // An unknown document is a search of nothing rather than an error: this + // parameter scopes a search, and turning it into an existence check would make + // it a way to probe for ids. + none := search(t, s, registry.SearchQuery{Text: "Filter", DocumentID: "doc_nope"}) + if len(none.Hits) != 0 { + t.Errorf("unknown document returned %d hits", len(none.Hits)) + } +} + +// TestAWordWithNoSpacesAroundItIsFound is why the tokeniser is trigram and not +// unicode61, and it is the test that would fail on the obvious choice. +// +// Japanese and Thai do not separate words with spaces, so unicode61 indexes a whole +// run as one token and finds a real word in neither: measured on the sequential +// manual, 0 hits for the Japanese "instruction manual" and 0 for the Thai "manual" +// against 6 stored blocks each. Every string here is real text from that manual. +func TestAWordWithNoSpacesAroundItIsFound(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Robot vacuum", "dreame-l40.pdf", "a") + + save(t, s, docID, + block(539, 0, 0, doc.BlockParagraph, "ja", + "本製品の不適切な使用による感電、火災、またはケガを回避するために、"+ + "本製品を使用する前に取扱説明書をよくお読みになり、大切に保管してください。"), + block(473, 0, 0, doc.BlockParagraph, "th", + "เพื่อหลีกเลี่ยงการเกิดไฟฟ้าช็อต ไฟไหม้ "+ + "หรือการบาดเจ็บที่เกิดจากการใช้เครื่องอย่างไม่เหมาะสม โปรดอ่านคู่มือการใช้งาน"), + block(517, 0, 0, doc.BlockParagraph, "ru", + "Во избежание поражения электрическим током перед использованием "+ + "устройства прочитайте руководство по эксплуатации."), + ) + + for _, tc := range []struct { + script, query, lang string + }{ + // "Instruction manual", inside a Japanese sentence with no spaces anywhere. + {"Japanese", "取扱説明書", "ja"}, + // "Manual", inside a Thai sentence whose words are not separated either. + {"Thai", "คู่มือ", "th"}, + // Cyrillic, which unicode61 would also have found -- here to show the one + // index serves both kinds of script rather than trading one for the other. + {"Cyrillic", "устройства", "ru"}, + } { + res := search(t, s, registry.SearchQuery{Text: tc.query}) + if len(res.Hits) != 1 { + t.Errorf("%s %q: got %d hits, want 1. A word-boundary tokeniser finds "+ + "none of these", tc.script, tc.query, len(res.Hits)) + continue + } + if res.Hits[0].Lang != tc.lang { + t.Errorf("%s %q matched the %s block", tc.script, tc.query, res.Hits[0].Lang) + } + if res.Mode != registry.SearchIndex { + t.Errorf("%s %q was answered by %s, not the index", tc.script, tc.query, res.Mode) + } + } +} + +// TestDiacriticsAreFoldedForLatinOnly pins both halves of that decision, and the +// second half is the one that was measured rather than assumed: FTS5's folding +// table reaches precomposed Latin and does not touch Cyrillic or Greek, so turning +// it on costs those scripts nothing. +func TestDiacriticsAreFoldedForLatinOnly(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Coffee machine", "manual.pdf", "a") + + save(t, s, docID, + block(3, 0, 0, doc.BlockParagraph, "de", "Zubehör für das Gerät alle drei Monate reinigen."), + block(4, 0, 0, doc.BlockParagraph, "ru", "Ещё раз проверьте фильтр."), + block(5, 0, 0, doc.BlockParagraph, "el", "Διαβάστε τις οδηγίες χρήσης."), + ) + + // A German household on any keyboard types "Zubehor" and must find "Zubehör". + // Without remove_diacritics -- which trigram, unlike unicode61, leaves OFF by + // default -- this is 0 hits. + if res := search(t, s, registry.SearchQuery{Text: "Zubehor"}); len(res.Hits) != 1 { + t.Errorf("the umlaut-free spelling found %d blocks, want the one holding it", + len(res.Hits)) + } + if res := search(t, s, registry.SearchQuery{Text: "Zubehör"}); len(res.Hits) != 1 { + t.Errorf("the word as printed found %d blocks, want 1", len(res.Hits)) + } + + // Cyrillic and Greek are NOT folded, so a query must be written as the text is. + // That is the measured behaviour and the reason the fold was free: it never had + // the chance to merge two Russian or Greek words. + if res := search(t, s, registry.SearchQuery{Text: "Ещё"}); len(res.Hits) != 1 { + t.Errorf("the Russian word as printed found %d blocks, want 1", len(res.Hits)) + } + if res := search(t, s, registry.SearchQuery{Text: "Еще"}); len(res.Hits) != 0 { + t.Errorf("the Russian word with its diaeresis dropped found %d blocks; "+ + "FTS5 was measured not to fold Cyrillic, so this must be 0", len(res.Hits)) + } + if res := search(t, s, registry.SearchQuery{Text: "οδηγίες"}); len(res.Hits) != 1 { + t.Errorf("the Greek word as printed found %d blocks, want 1", len(res.Hits)) + } + if res := search(t, s, registry.SearchQuery{Text: "οδηγιες"}); len(res.Hits) != 0 { + t.Errorf("the Greek word without its tonos found %d blocks; FTS5 was measured "+ + "not to fold Greek, so this must be 0", len(res.Hits)) + } +} + +// TestAHeadingOutranksAParagraphOfEqualStanding is the ranking judgement, held to +// what it claims. A heading names a section, so it is a better answer to "where +// does it say this" than a sentence mentioning the word in passing -- and the bonus +// is visible in the response, as the gap between bm25 and score, so it can be +// argued with. +func TestAHeadingOutranksAParagraphOfEqualStanding(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "manual.pdf", "a") + + // Same length, same one occurrence, so bm25 alone would rank them together and + // the tie would break on page order -- which would put the paragraph first. + save(t, s, docID, + block(12, 0, 0, doc.BlockParagraph, "de", "Der Wasserfilter sitzt hinten."), + heading(48, 0, 0, "de", "Der Wasserfilter wird getauscht"), + ) + + res := search(t, s, registry.SearchQuery{Text: "Wasserfilter"}) + if len(res.Hits) != 2 { + t.Fatalf("got %d hits, want 2", len(res.Hits)) + } + if res.Hits[0].Kind != "heading" { + t.Errorf("first hit is a %s from page %d; a heading of equal standing should "+ + "lead", res.Hits[0].Kind, res.Hits[0].Page) + } + // The bonus is 1.0 and it is applied to the heading only. + h := res.Hits[0] + if h.Score >= h.BM25 { + t.Errorf("heading score %v is not better than its bm25 %v; bm25 is negative "+ + "and the bonus is subtracted", h.Score, h.BM25) + } + if got := h.BM25 - h.Score; got < 0.99 || got > 1.01 { + t.Errorf("heading bonus = %v, want 1.0", got) + } + if p := res.Hits[1]; p.Score != p.BM25 { + t.Errorf("a %s got a bonus of %v; only headings do", p.Kind, p.BM25-p.Score) + } +} + +// TestAQueryTooShortForTheIndexIsAnsweredByAScan is the named limitation and its +// mitigation. A trigram index holds no token under three characters, so a +// two-character query -- an ordinary word in Chinese and Japanese -- matches +// nothing in it. Measured on the sequential manual: the two characters for "power" +// occur in 27 stored blocks and the index finds 0. +func TestAQueryTooShortForTheIndexIsAnsweredByAScan(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Robot vacuum", "dreame-l40.pdf", "a") + + save(t, s, docID, + block(541, 0, 0, doc.BlockParagraph, "ja", "電源を入れる前に取扱説明書をお読みください。"), + heading(542, 0, 0, "ja", "電源について"), + ) + + res := search(t, s, registry.SearchQuery{Text: "電源"}) + if res.Mode != registry.SearchSubstring { + t.Fatalf("mode = %q, want %q: two characters cannot be a trigram", + res.Mode, registry.SearchSubstring) + } + if len(res.Hits) != 2 { + t.Fatalf("got %d hits, want 2; the scan is what makes a two-character word "+ + "findable at all", len(res.Hits)) + } + // The same heading judgement applies, since it is a judgement about answers and + // not about scoring. + if res.Hits[0].Kind != "heading" { + t.Errorf("first hit is a %s; the heading should lead here too", res.Hits[0].Kind) + } + // No bm25 exists on this path, and saying 0 is honest where inventing a number + // would not be. + for i := range res.Hits { + if res.Hits[i].BM25 != 0 || res.Hits[i].Score != 0 { + t.Errorf("hit %d carries bm25 %v score %v; there is no index term to weigh", + i, res.Hits[i].BM25, res.Hits[i].Score) + } + } + if !strings.Contains(res.Hits[0].Snippet, "電源") { + t.Errorf("snippet %q does not show the match", res.Hits[0].Snippet) + } + + // One short word sends the WHOLE query to the scan, rather than being dropped + // from it: a search for two words that quietly became a search for one is + // indistinguishable from a correct answer. + mixed := search(t, s, registry.SearchQuery{Text: "取扱説明書 を"}) + if mixed.Mode != registry.SearchSubstring { + t.Errorf("a query mixing a long word with a short one ran as %q", mixed.Mode) + } +} + +// TestTwoWordsMeanBothOfThem: the phrases are ANDed, so a query is a conjunction +// rather than a phrase that must appear verbatim. +func TestTwoWordsMeanBothOfThem(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "manual.pdf", "a") + + save(t, s, docID, + block(1, 0, 0, doc.BlockParagraph, "de", "Den Filter regelmaessig reinigen."), + block(2, 0, 0, doc.BlockParagraph, "de", "Reinigen Sie das Gehaeuse feucht."), + block(3, 0, 0, doc.BlockParagraph, "de", "Der Filter sitzt hinten."), + ) + + res := search(t, s, registry.SearchQuery{Text: "Filter reinigen"}) + if len(res.Hits) != 1 || res.Hits[0].Page != 1 { + t.Fatalf("got %+v, want only the block holding both words", res.Hits) + } +} + +// TestAQueryIsNeverFTS5Syntax. FTS5 has an expression language, so an ordinary +// query containing a colon, a quote, a star or the word NOT would otherwise be a +// syntax error or, worse, silently mean something else. +func TestAQueryIsNeverFTS5Syntax(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "manual.pdf", "a") + save(t, s, docID, block(1, 0, 0, doc.BlockParagraph, "de", + `Fehler: "Motor laeuft NICHT" - Filter pruefen.`)) + + for _, q := range []string{ + `Fehler:`, + `"Motor`, + `Motor NOT laeuft`, + `Filter*`, + `Motor OR Filter`, + `NEAR(Motor Filter)`, + `^Fehler`, + `{Motor}`, + } { + res, err := s.Search(context.Background(), registry.SearchQuery{Text: q}) + if err != nil { + t.Errorf("searching %q failed: %v. A search box has no query language", q, err) + continue + } + _ = res + } + + // And a quote inside a word is escaped rather than opening a phrase. + if res := search(t, s, registry.SearchQuery{Text: `"Motor laeuft NICHT"`}); len(res.Hits) != 1 { + t.Errorf("quoted text found %d hits, want the one block holding it", len(res.Hits)) + } +} + +func TestSearchRejectsAnEmptyQuery(t *testing.T) { + s := newService(t) + for _, q := range []string{"", " ", "\t\n"} { + if _, err := s.Search(context.Background(), registry.SearchQuery{Text: q}); !errors.Is(err, registry.ErrInvalid) { + t.Errorf("searching %q gave %v, want ErrInvalid", q, err) + } + } +} + +// TestNothingMatchedSaysHowMuchWasIndexed. "No manual says that" and "no manual has +// been converted yet" are the same empty list, and the second is not a search +// failure -- it is the same distinction Blocks makes between empty and absent. +func TestNothingMatchedSaysHowMuchWasIndexed(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "manual.pdf", "a") + + fresh := search(t, s, registry.SearchQuery{Text: "Saugkraft"}) + if fresh.Indexed == nil || *fresh.Indexed != 0 { + t.Errorf("indexed = %v before any conversion, want 0", fresh.Indexed) + } + + save(t, s, docID, block(1, 0, 0, doc.BlockParagraph, "de", "Den Filter reinigen.")) + after := search(t, s, registry.SearchQuery{Text: "Saugkraft"}) + if after.Indexed == nil || *after.Indexed != 1 { + t.Errorf("indexed = %v with one block stored, want 1", after.Indexed) + } +} + +func TestSearchTruncatesAtTheLimitAndSaysSo(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "manual.pdf", "a") + + blocks := make([]doc.Block, 0, 5) + for i := range 5 { + blocks = append(blocks, block(1+i, 0, 0, doc.BlockParagraph, "de", "Den Filter reinigen.")) + } + save(t, s, docID, blocks...) + + res := search(t, s, registry.SearchQuery{Text: "Filter", Limit: 3}) + if len(res.Hits) != 3 || !res.Truncated { + t.Errorf("got %d hits truncated=%v, want 3 and true", len(res.Hits), res.Truncated) + } + full := search(t, s, registry.SearchQuery{Text: "Filter", Limit: 5}) + if len(full.Hits) != 5 || full.Truncated { + t.Errorf("got %d hits truncated=%v, want 5 and false", len(full.Hits), full.Truncated) + } + // Above the cap is clamped rather than refused, because a client asking for too + // much wants as much as it can have. + if capped := search(t, s, registry.SearchQuery{ + Text: "Filter", Limit: registry.MaxSearchLimit + 1000, + }); capped.Limit != registry.MaxSearchLimit { + t.Errorf("limit = %d, want it clamped to %d", capped.Limit, registry.MaxSearchLimit) + } +} + +// TestReconvertingADocumentDoesNotDuplicateItsHits is the idempotency requirement +// every derived table here carries: a worker can die after doing the work and +// before recording success, so the same conversion runs twice. +func TestReconvertingADocumentDoesNotDuplicateItsHits(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "manual.pdf", "a") + + blocks := []doc.Block{ + heading(48, 43, 0, "de", "Ausblasfilter austauschen"), + block(48, 43, 1, doc.BlockParagraph, "de", "Den Filter alle zwei Jahre tauschen."), + } + save(t, s, docID, blocks...) + first := search(t, s, registry.SearchQuery{Text: "Filter"}) + + for range 3 { + save(t, s, docID, blocks...) + } + again := search(t, s, registry.SearchQuery{Text: "Filter"}) + + if len(again.Hits) != len(first.Hits) { + t.Fatalf("after three more conversions the index returns %d hits, first "+ + "returned %d", len(again.Hits), len(first.Hits)) + } + if len(again.Hits) != 2 { + t.Fatalf("got %d hits, want 2", len(again.Hits)) + } + for i := range again.Hits { + if again.Hits[i] != first.Hits[i] { + t.Errorf("hit %d changed:\n first %+v\n again %+v", i, first.Hits[i], again.Hits[i]) + } + } +} + +// TestAReconversionThatDropsABlockDropsItFromTheIndex. The wholesale replace exists +// because a re-conversion can produce FEWER blocks, and an index that kept the old +// text would answer with a paragraph that no longer exists -- worse than a stale +// row in a reader, because search is how it would be found. +func TestAReconversionThatDropsABlockDropsItFromTheIndex(t *testing.T) { + s := newService(t) + docID := newDocumentOnDevice(t, s, "Vacuum cleaner", "manual.pdf", "a") + + save(t, s, docID, + block(48, 43, 0, doc.BlockParagraph, "de", "Den Ausblasfilter tauschen."), + block(48, 43, 1, doc.BlockParagraph, "de", "Den Motorschutzfilter waschen."), + block(48, 43, 2, doc.BlockParagraph, "de", "Den Hygienefilter entsorgen."), + ) + if res := search(t, s, registry.SearchQuery{Text: "Hygienefilter"}); len(res.Hits) != 1 { + t.Fatalf("setup: got %d hits for the third block", len(res.Hits)) + } + + // A better paragraph rule merges the first two and drops the third: two blocks + // where there were three, and the surviving indices are 0 and 1. + save(t, s, docID, + block(48, 43, 0, doc.BlockParagraph, "de", + "Den Ausblasfilter tauschen. Den Motorschutzfilter waschen."), + block(48, 43, 1, doc.BlockParagraph, "de", "Den Staubbehaelter leeren."), + ) + + if res := search(t, s, registry.SearchQuery{Text: "Hygienefilter"}); len(res.Hits) != 0 { + t.Errorf("the dropped block is still findable: %+v", res.Hits) + } + // And the block that was updated in place -- same key, new text -- is findable by + // its new text and not by its old. + if res := search(t, s, registry.SearchQuery{Text: "Staubbehaelter"}); len(res.Hits) != 1 { + t.Errorf("the replacement block at index 1 is not findable: %+v", res.Hits) + } + if res := search(t, s, registry.SearchQuery{Text: "Motorschutzfilter"}); len(res.Hits) != 1 { + t.Errorf("the merged text found %d hits, want the one block it merged into", + len(res.Hits)) + } +} + +// TestDeletingADeviceRemovesItsManualsFromTheIndex is the path NO GO CODE OBSERVES, +// and it is why the index is maintained by triggers rather than by statements next +// to each write. Deleting a device cascades twice -- device to documents to blocks +// -- and nothing in Go touches doc_blocks on the way. Without the delete trigger, +// search hands a household a manual it deleted; TestBlockSearchIndexSurvivesTheCascade +// in internal/db is the same claim held against FTS5's own integrity check. +// +// There is no Service.DeleteDocument and no DELETE route for a document: a document +// is removed today only with its device. Declining one keeps it deliberately. So +// this is the whole of the delete surface, and the document-level cascade is pinned +// in internal/db where a raw handle can exercise it. +func TestDeletingADeviceRemovesItsManualsFromTheIndex(t *testing.T) { + s := newService(t) + ctx := context.Background() + kept := newDocumentOnDevice(t, s, "Washing machine", "washer.pdf", "b") + doomed := newDocumentOnDevice(t, s, "Vacuum cleaner", "vacuum.pdf", "a") + + save(t, s, kept, block(7, 0, 0, doc.BlockParagraph, "de", "Den Flusenfilter reinigen.")) + save(t, s, doomed, block(12, 0, 0, doc.BlockParagraph, "de", "Den Saugfilter reinigen.")) + + if res := search(t, s, registry.SearchQuery{Text: "Saugfilter"}); len(res.Hits) != 1 { + t.Fatalf("setup: got %d hits", len(res.Hits)) + } + + if err := s.DeleteDevice(ctx, mustDeviceOf(t, s, doomed)); err != nil { + t.Fatalf("delete device: %v", err) + } + + if res := search(t, s, registry.SearchQuery{Text: "Saugfilter"}); len(res.Hits) != 0 { + t.Errorf("the deleted device's manual is still findable: %+v", res.Hits) + } + if res := search(t, s, registry.SearchQuery{Text: "Flusenfilter"}); len(res.Hits) != 1 { + t.Errorf("the other device's manual lost its hit: %+v", res.Hits) + } +} + +func mustDeviceOf(t *testing.T, s *registry.Service, documentID string) string { + t.Helper() + document, err := s.GetDocument(context.Background(), documentID) + if err != nil { + t.Fatalf("get document: %v", err) + } + return document.DeviceID +} diff --git a/internal/store/store.go b/internal/store/store.go index 98d8e89..ab7db9f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -273,6 +273,29 @@ func Digest(b []byte) string { // Root returns the store's root directory. func (s *Store) Root() string { return s.root } +// Path returns the filesystem path of a stored blob. +// +// Handing out a path rather than a reader exists for one reason: the document +// pipeline shells out to poppler, and an external process needs a real file. It +// is safe because blobs are immutable and stored mode 0400 — the callee can read +// the bytes but cannot alter them, so the digest the filename asserts stays true. +// +// Prefer [Store.Open] for anything in-process. The digest is validated, so a +// caller-supplied value cannot escape the store root. +func (s *Store) Path(digest string) (string, error) { + path, err := s.pathFor(digest) + if err != nil { + return "", err + } + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("%w: %s", ErrNotFound, digest) + } + return "", fmt.Errorf("store: stat blob %s: %w", digest, err) + } + return path, nil +} + // CleanTemp removes leftover temporary uploads, which is how a crash mid-upload // is reclaimed. Safe to call at startup. func (s *Store) CleanTemp() error { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 3439915..76b195a 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -168,6 +168,49 @@ func TestPathTraversalRejected(t *testing.T) { if s.Exists(bad) { t.Errorf("Exists(%q) should be false", truncate(bad)) } + // Path is the one method that hands a real filesystem path to an external + // process, so it matters most here — and it was the one this test did not + // cover, while the #nosec justification in store.go named this test. + if _, err := s.Path(bad); !errors.Is(err, ErrBadDigest) { + t.Errorf("Path(%q) should fail with ErrBadDigest, got %v", truncate(bad), err) + } + } +} + +// TestPathIsInsideRootAndReadOnly checks the two properties the document +// pipeline relies on when it hands a blob to poppler: the path is inside the +// store, and the file cannot be modified through it, so the digest its name +// asserts stays true. +func TestPathIsInsideRootAndReadOnly(t *testing.T) { + s := newStore(t) + + ref, err := s.Put(context.Background(), strings.NewReader("a manual")) + if err != nil { + t.Fatalf("put: %v", err) + } + + path, err := s.Path(ref.SHA256) + if err != nil { + t.Fatalf("path: %v", err) + } + if !strings.HasPrefix(path, s.Root()+string(filepath.Separator)) { + t.Errorf("path %q escapes the store root %q", path, s.Root()) + } + // Absolute, so an external tool can never read it as a command-line flag. + if !filepath.IsAbs(path) { + t.Errorf("path %q is not absolute", path) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat: %v", err) + } + if perm := info.Mode().Perm(); perm&0o222 != 0 { + t.Errorf("permissions are %04o — a blob handed to another process must not be writable", perm) + } + + if _, err := s.Path(strings.Repeat("a", 64)); !errors.Is(err, ErrNotFound) { + t.Errorf("Path on an absent blob should report ErrNotFound, got %v", err) } } diff --git a/internal/testpdf/testpdf.go b/internal/testpdf/testpdf.go new file mode 100644 index 0000000..b85239d --- /dev/null +++ b/internal/testpdf/testpdf.go @@ -0,0 +1,312 @@ +// Package testpdf builds small, valid PDFs in memory for tests. +// +// It exists because of two constraints that meet awkwardly. The document pipeline +// can only be tested against a real PDF read by real poppler — a hand-made fake +// would test nothing. But no PDF may be committed to this repository: CI's hygiene +// job rejects every .pdf outright, because a committed document is either someone's +// copyrighted manual or someone's private paperwork. +// +// So the test corpus is generated. A few hundred bytes of PDF per page is enough +// to exercise page counting, text extraction, printed language tags, folios and a +// contents table, with no network access and nothing to commit. +// +// The text is written with a standard Type 1 font, so only Latin-1 characters are +// representable. Non-Latin script behaviour is unit-tested directly against +// strings instead, where no PDF is involved. +package testpdf + +import ( + "bytes" + "fmt" + "strings" +) + +// Page is one page of a generated document. +type Page struct { + // Headings are drawn above Lines in a second, heavier face at a larger size. + // + // They exist because a document written wholly in one font cannot exercise the + // font a run comes back with, and that is what separates a heading from a + // paragraph. One face and one size was enough while only geometry was read + // back; it is not enough now. + // + // Two things measured on what poppler actually reports for these, neither of + // them guessable. It names the family "Helvetica" for the bold face as well as + // the regular one — a standard font is not embedded, so there is no subset name + // to read a weight out of — and marks these runs regardless. And it scales + // the size by the same 1.5 as the coordinates, so headingSize of 17 arrives as + // 26 and bodySize of 11 as 17. See runs_test.go, which asserts both. + Headings []string + // Lines are drawn top to bottom, and come back out of pdftotext in the same + // order. The first line is therefore where a language tag goes, which is what + // makes the printed-tag signal testable. + // + // They span the full measure from the left margin, so a long line here is how + // a heading set across several columns is generated. + Lines []string + // Columns are blocks of text at their own horizontal offsets, drawn below + // Lines and all starting at the same height. + Columns []Column + // Drawings are vector illustrations, drawn before any text so the text sits + // over them the way a caption does on a real page. + // + // They exist because a document with no vector graphics cannot exercise the + // figure reader at all, and that turned out to be the only kind of picture the + // two real manuals contain: `pdfimages` yields not one illustration over their + // 628 pages, and every diagram in both is drawn. See internal/doc/figures.go. + // A raster is deliberately not generated here — embedding a JPEG would test a + // path this project has no document for. + Drawings []Drawing +} + +// Drawing is a scribble in a box: a frame with a fixed number of strokes inside +// it, at a position in PDF points on the 612 by 792 page. +// +// Strokes rather than a shape, because what separates a picture from page +// furniture in internal/doc/figures.go is how many shapes an area holds, and a +// generator that draws one rectangle can only produce furniture. The strokes are +// laid out deterministically so a test can assert an exact count. +type Drawing struct { + // X and Y are the lower-left corner in PDF points, W and H the size. Poppler + // reports coordinates at 1.5 times this and measures Y down from the top of the + // page, so a drawing at Y=400 on a 792-point page arrives with its top edge at + // 1.5*(792-400-H). + X, Y, W, H int + // Strokes is how many lines to draw inside the frame, over and above the frame + // itself. Each is a separate subpath, so the shape count internal/doc reads back + // is Strokes plus one. + Strokes int +} + +// Column is a block of lines set at its own horizontal offset. +// +// It exists because a single-column generator cannot exercise the column +// detector, and geometry is the one input the detector has: it needs a real +// gutter in a real PDF read by real poppler, not hand-written coordinates. Every +// other test of it supplies runs directly, which cannot catch a disagreement +// between what this package writes and what poppler reports back. +type Column struct { + // X is the left edge in PDF points, on the 612 by 792 page this package + // writes. Poppler's XML reports coordinates at 1.5 times this — 108 dpi + // against the PDF's 72 — so a column at X=60 arrives as left=90 and the page + // as 918 by 1188. Measured on both real fixtures: 918/612.283 and 892/595.276. + X int + // Lines are drawn top to bottom, as [Page.Lines] are. + Lines []string +} + +// Doc describes a document to generate. +type Doc struct { + Pages []Page +} + +// TaggedSections builds a multi-language document of the shape real appliance +// manuals take: a contents table, then one section per language, every page +// carrying its own language code. +// +// codes are the language codes in document order, pagesPerSection how many pages +// each occupies. When withContents is true a contents page precedes the sections, +// listing each code with a title and its printed page — including, deliberately, +// the printed page rather than the PDF page, so the offset between them has to be +// resolved rather than assumed. +func TaggedSections(codes []string, pagesPerSection int, withContents bool) Doc { + var d Doc + + if withContents { + lines := []string{"Contents"} + printed := 1 + for _, code := range codes { + lines = append(lines, code, code+" User Manual", fmt.Sprint(printed)) + printed += pagesPerSection + } + d.Pages = append(d.Pages, Page{Lines: lines}) + } + + folio := 1 + for _, code := range codes { + for i := range pagesPerSection { + body := fmt.Sprintf("Section %s page %d. ", code, i+1) + + strings.Repeat("Maintenance information for this appliance. ", 3) + d.Pages = append(d.Pages, Page{Lines: []string{ + code, + fmt.Sprintf("%s Safety Information", code), + body, + fmt.Sprint(folio), + }}) + folio++ + } + } + return d +} + +// Blank builds a document of n pages with no extractable text, standing in for a +// scan. It is what exercises the "no text layer" branch of the pipeline. +func Blank(n int) Doc { + d := Doc{Pages: make([]Page, n)} + return d +} + +// Build renders the document to PDF bytes. +func (d Doc) Build() []byte { + var buf bytes.Buffer + // Offsets are byte positions of each object, needed for the cross-reference + // table. Index 0 is the free head entry, so object numbering starts at 1. + offsets := []int{0} + + addObject := func(body string) { + offsets = append(offsets, buf.Len()) + fmt.Fprintf(&buf, "%d 0 obj\n%s\nendobj\n", len(offsets)-1, body) + } + + buf.WriteString("%PDF-1.4\n") + // A binary comment marks the file as binary for tools that sniff it. + buf.WriteString("%\xe2\xe3\xcf\xd3\n") + + // Object numbers are laid out in advance so references can be written before + // the objects they point at exist. + const catalogObj, pagesObj, fontObj, boldFontObj = 1, 2, 3, 4 + firstPageObj := 5 + + kids := make([]string, 0, len(d.Pages)) + for i := range d.Pages { + kids = append(kids, fmt.Sprintf("%d 0 R", firstPageObj+i*2)) + } + + addObject(fmt.Sprintf("<< /Type /Catalog /Pages %d 0 R >>", pagesObj)) + addObject(fmt.Sprintf("<< /Type /Pages /Kids [%s] /Count %d >>", + strings.Join(kids, " "), len(d.Pages))) + addObject("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>") + // The second face is one of the standard 14, so it needs no embedded file and + // no metrics of its own. Poppler does not report it as a distinct family — + // see Page.Headings, it calls both of them "Helvetica" — but it does mark its + // runs bold, which is what makes a weight readable end to end from here. + addObject("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding >>") + + for i, page := range d.Pages { + contentObj := firstPageObj + i*2 + 1 + addObject(fmt.Sprintf( + "<< /Type /Page /Parent %d 0 R /MediaBox [0 0 612 792] /Contents %d 0 R "+ + "/Resources << /Font << /F1 %d 0 R /F2 %d 0 R >> >> >>", + pagesObj, contentObj, fontObj, boldFontObj)) + + stream := pageStream(page) + addObject(fmt.Sprintf("<< /Length %d >>\nstream\n%s\nendstream", len(stream), stream)) + } + + // Cross-reference table. Every entry is exactly 20 bytes, which the format + // requires and which is the easiest thing to get subtly wrong. + xrefOffset := buf.Len() + fmt.Fprintf(&buf, "xref\n0 %d\n", len(offsets)) + buf.WriteString("0000000000 65535 f \n") + for _, off := range offsets[1:] { + fmt.Fprintf(&buf, "%010d 00000 n \n", off) + } + fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root %d 0 R >>\nstartxref\n%d\n%%%%EOF\n", + len(offsets), catalogObj, xrefOffset) + + return buf.Bytes() +} + +// Where text is placed on the 612 by 792 page, in PDF points. +const ( + marginX = 72 + firstY = 720 + lineStep = 24 + // bodySize and headingSize are the two sizes written. They differ enough that + // poppler reports two distinct fontspecs rather than rounding them together. + bodySize = 11 + headingSize = 17 + // lastY is the floor. Lines pile up on it rather than running off the page, + // because a run outside the page box is dropped by the column detector as + // off-page and would silently vanish from a test's expectations. + lastY = 40 +) + +// pageStream renders one page's text as a content stream. +func pageStream(p Page) string { + if len(p.Headings) == 0 && len(p.Lines) == 0 && len(p.Columns) == 0 && + len(p.Drawings) == 0 { + return "" + } + var b strings.Builder + + for _, d := range p.Drawings { + drawDrawing(&b, d) + } + + y := firstY + for _, line := range p.Headings { + drawIn(&b, "F2", headingSize, marginX, y, line) + y = nextY(y) + } + for _, line := range p.Lines { + drawLine(&b, marginX, y, line) + y = nextY(y) + } + // Every column starts where the full-measure lines left off, so a heading in + // Lines sits above all of them rather than beside one. + for _, col := range p.Columns { + cy := y + for _, line := range col.Lines { + drawLine(&b, col.X, cy, line) + cy = nextY(cy) + } + } + return b.String() +} + +// drawDrawing writes a frame and its strokes as one stroked path per subpath. +// +// Each stroke is its own `m`/`l`/`S` rather than one path with many subpaths, +// because cairo re-emits a multi-subpath stroke as a single element and +// internal/doc counts shapes off those elements' subpaths — separate paths keep +// the two counts equal and the test's arithmetic honest. +func drawDrawing(b *strings.Builder, d Drawing) { + fmt.Fprintf(b, "0.5 w 0 0 0 RG\n%d %d %d %d re S\n", d.X, d.Y, d.W, d.H) + for i := range d.Strokes { + // A diagonal from the left edge to the right, stepped down the frame, so + // every stroke is inside it and none is axis-aligned — an axis-aligned one + // would also be read as a ruled line by rules.go, which would make a test + // of one file depend on the other. + dy := d.H * (i + 1) / (d.Strokes + 1) + fmt.Fprintf(b, "%d %d m %d %d l S\n", + d.X+2, d.Y+dy, d.X+d.W-2, d.Y+dy-d.H/(2*(d.Strokes+1))-1) + } +} + +func drawLine(b *strings.Builder, x, y int, line string) { + drawIn(b, "F1", bodySize, x, y, line) +} + +func drawIn(b *strings.Builder, font string, size, x, y int, line string) { + fmt.Fprintf(b, "BT /%s %d Tf %d %d Td (%s) Tj ET\n", font, size, x, y, escapeString(line)) +} + +func nextY(y int) int { + if y-lineStep < lastY { + return lastY + } + return y - lineStep +} + +// escapeString escapes the characters that would otherwise end a PDF string +// literal. Non-Latin-1 runes are replaced rather than mangled, since a standard +// Type 1 font cannot represent them and a silently corrupted glyph would make a +// test failure hard to read. +func escapeString(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r == '(' || r == ')' || r == '\\': + b.WriteByte('\\') + b.WriteRune(r) + case r < 32: + b.WriteByte(' ') + case r > 255: + b.WriteByte('?') + default: + b.WriteRune(r) + } + } + return b.String() +} diff --git a/internal/testpdf/testpdf_test.go b/internal/testpdf/testpdf_test.go new file mode 100644 index 0000000..a4d6549 --- /dev/null +++ b/internal/testpdf/testpdf_test.go @@ -0,0 +1,62 @@ +package testpdf_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/testpdf" +) + +// The generated PDFs are only useful if real poppler can read them, so that is +// what this asserts. Without poppler the check cannot be made and skips. +func TestGeneratedPDFIsValid(t *testing.T) { + if !extern.Available(extern.PDFInfo) || !extern.Available(extern.PDFToText) { + t.Skip("poppler is not installed") + } + + d := testpdf.TaggedSections([]string{"EN", "DE", "FR"}, 2, true) + path := filepath.Join(t.TempDir(), "gen.pdf") + if err := os.WriteFile(path, d.Build(), 0o600); err != nil { + t.Fatal(err) + } + + info, err := exec.CommandContext(t.Context(), "pdfinfo", path).CombinedOutput() + if err != nil { + t.Fatalf("pdfinfo rejected the generated file: %v\n%s", err, info) + } + // 1 contents page + 3 sections x 2 pages. + if !strings.Contains(string(info), "Pages: 7") { + t.Errorf("expected 7 pages, pdfinfo said:\n%s", info) + } + + text, err := exec.CommandContext(t.Context(), "pdftotext", "-enc", "UTF-8", path, "-").CombinedOutput() + if err != nil { + t.Fatalf("pdftotext rejected the generated file: %v\n%s", err, text) + } + for _, want := range []string{"Contents", "EN User Manual", "Section DE page 1"} { + if !strings.Contains(string(text), want) { + t.Errorf("extracted text is missing %q", want) + } + } +} + +func TestBlankHasNoText(t *testing.T) { + if !extern.Available(extern.PDFToText) { + t.Skip("poppler is not installed") + } + path := filepath.Join(t.TempDir(), "blank.pdf") + if err := os.WriteFile(path, testpdf.Blank(3).Build(), 0o600); err != nil { + t.Fatal(err) + } + text, err := exec.CommandContext(t.Context(), "pdftotext", "-enc", "UTF-8", path, "-").CombinedOutput() + if err != nil { + t.Fatalf("pdftotext failed: %v\n%s", err, text) + } + if strings.TrimSpace(strings.ReplaceAll(string(text), "\f", "")) != "" { + t.Errorf("a blank document yielded text: %q", text) + } +} diff --git a/internal/verify/callouts_test.go b/internal/verify/callouts_test.go new file mode 100644 index 0000000..42f56fc --- /dev/null +++ b/internal/verify/callouts_test.go @@ -0,0 +1,74 @@ +package verify_test + +import ( + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/verify" +) + +// The two things a figure's callout labels change about these checks, and they pull +// in OPPOSITE directions — which is the point, and why one flag could not have served +// both. See doc.Callouts. + +// TestACalloutIsCountedByCoverage is the difference between a callout and page +// furniture, stated as a test because the two are one line apart in checkCoverage and +// it would be natural to "tidy" them together. +// +// Furniture is DISCARDED, so coverage does not count it: counting it would leave every +// ratio where it was and hide a rule that wrongly claimed a paragraph. A callout label +// is RELOCATED — the reader draws it beside the picture — so its characters must stay +// in the numerator. Skip them and coverage falls by exactly the labels on every +// illustrated page, and nothing could tell that from a page that lost text. +func TestACalloutIsCountedByCoverage(t *testing.T) { + claimed := block(7, 0, 40, 300, 100, prose) + claimed.Callout = true + in := verify.Input{ + Blocks: []doc.Block{claimed}, + Text: []doc.Page{page(7, prose)}, + } + if got := count(t, in, verify.KindCoverage); got != 0 { + t.Errorf("a page whose only block is a callout label reported %d coverage "+ + "finding(s); the label is still shown to the reader, so its characters "+ + "are not missing and coverage must count them", got) + } + + // The control, and the contrast: the SAME block marked furniture does fire, + // because furniture is not shown at all. + in.Blocks[0].Callout = false + in.Blocks[0].Furniture = true + if got := count(t, in, verify.KindCoverage); got != 1 { + t.Errorf("the same block marked furniture reported %d coverage finding(s), "+ + "expected 1; if these two flags behave alike, one of them is wrong", got) + } +} + +// TestACalloutIsNotJudgedForReadingOrder is the other direction. A callout label has +// no place in reading order — doc.RegionBlocks appends the labels after a region's +// content, exactly as it appends the furniture — so asking where in the reading order +// one comes is asking a question with no answer. +// +// The geometry below is page 521 of the sequential manual: two labels either side of +// its underside drawing, disjoint in x and descending in y, which is this check's +// violation shape exactly. Judged, it reports interleaving on a page that is read +// correctly. Both texts clear [minOrderChars] on their own, so the exemption and not a +// length guard is what has to do the work. +func TestACalloutIsNotJudgedForReadingOrder(t *testing.T) { + left := block(521, 0, 40, 200, 300, "Зажимы защиты щетки") + right := block(521, 1, 600, 760, 320, "Всенаправленное колесо") + left.Callout, right.Callout = true, true + + in := verify.Input{Blocks: []doc.Block{left, right}} + if got := count(t, in, verify.KindReadingOrder); got != 0 { + t.Errorf("two callout labels either side of a drawing reported %d reading-order "+ + "finding(s); they are not in the reading flow and cannot interleave", got) + } + + // The control: the same two blocks as ordinary content DO fire, which is what says + // the geometry is a real violation shape and the exemption is doing the work. + in.Blocks[0].Callout, in.Blocks[1].Callout = false, false + if got := count(t, in, verify.KindReadingOrder); got != 1 { + t.Errorf("the same two blocks as content reported %d reading-order finding(s), "+ + "expected 1; without that this test would pass for the wrong reason", got) + } +} diff --git a/internal/verify/figures.go b/internal/verify/figures.go new file mode 100644 index 0000000..a7e6bc8 --- /dev/null +++ b/internal/verify/figures.go @@ -0,0 +1,299 @@ +package verify + +import ( + "bytes" + "fmt" + "image" + "image/png" + "math" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Bounds on a figure's geometry, in the 1.5-scaled space [doc.PageRuns] and a +// `pdftoppm -r 108` raster share, so a unit here is a pixel on a render a person +// can look at. Measured on both fixtures; the numbers are at each constant. +const ( + // maxBlankBand is how much empty space a figure's render may carry on one side + // of the picture, in units. + // + // The cause is the clip-path limitation conversion.md records: a figure's box is a + // path's UNCLIPPED extent, so a drawing clipped to a smaller window reports a box + // bigger than anything painted in it, and the crop carries the difference as a + // blank band. + // + // Measured on the RENDERED PIXELS of every figure of both manuals — see + // [paintedMargins] for why the ink boxes cannot answer this — as the largest of + // the four blank margins: + // + // column manual 46 figures: 35 at 0.0, 11 over 2, 9 over 4, 4 over 12, 3 over 16, 1 over 40 + // sequential 163 figures: 17 over 2, 12 over 4, 7 over 8, 6 over 12, 4 over 16, 1 over 40 + // + // The largest are page 46 figure 1 of the column manual (34 units blank at the + // right, 64 at the foot) and page 530 figure 0 of the sequential one (33 left, 49 + // right) — which is the fault the user reports, of the size they report it at. + // + // 12 is chosen because pdftoppm rounds the crop outwards by up to a unit on each + // side, a hairline's own stroke width is one or two, and half a line of body text + // on either manual is 8: below 12 the check would report rounding. Above it every + // case seen is a real band. It is not a gap in a distribution — there is no gap; + // the counts above are a smooth tail, the same shape doc/figures.go records for + // its own size guard. + maxBlankBand = 12.0 + + // whiteCutoff is how light a pixel may be and still count as background, out of + // 65535 per channel. + // + // pdftoppm renders on opaque white, so the background is 0xffff exactly and any + // cutoff below that would work for a solid drawing. It is lower than that for the + // anti-aliased edge of a hairline, which is the only thing painted in the outer + // pixels of a line drawing. Measured over both manuals, moving it from 0xffff to + // 0xf000 changes no figure's verdict; see the sweep in the fixture test. + whiteCutoff = 0xf000 + + // clipSlack is how far a shape may reach past the figure's box before it counts + // as clipped rather than as touching it. + // + // A stroke has width, and a box derived from path extents sits within a unit of + // the strokes that made it, so an exact comparison reports every figure. 1.0 is + // the same one unit [doc.Convert] allows a figure against a region's edge, and + // for the same reason: this is comparing one measurement of a drawing against + // another. + clipSlack = 1.0 + + // minClipOverlap is how much of a shape must fall inside the figure's box before + // the shape is treated as part of that figure at all. + // + // Both ends of this range are degenerate, which is what fixes the value in the + // middle. Swept over both manuals, as figures reported clipped (column of 46 / + // sequential of 163): + // + // overlap >= 0.00 46 / 163 — every figure: a page-sized background path + // "crosses the edge" of all of them + // overlap >= 0.25 29 / 92 + // overlap >= 0.50 22 / 74 + // overlap >= 0.75 16 / 39 + // overlap >= 1.00 0 / 0 — containment cannot detect clipping at all, + // since a contained shape crosses nothing by definition + // + // 0.5 is the midpoint of the usable range: a shape more than half inside the box + // is the figure's, one mostly outside belongs to whatever else is on the page. + // There is no plateau to sit on, which is stated rather than hidden. + // + // The verdict was cross-checked against a signal it shares no code with: whether + // the render's own paint reaches the crop's edge, which is what being cut off + // looks like. Of the column manual's 46 figures, all 22 flagged clipped have paint + // at the edge and none is flagged without it; on the sequential manual 73 of 74 + // do. The converse does not hold and should not — the crop is derived from the + // ink, so a picture's paint routinely reaches its own edge — and that is exactly + // what the ink comparison adds: it says the drawing CONTINUES past the crop. + minClipOverlap = 0.5 +) + +// checkFigures reports the two distinct faults a figure's box can have. +// +// Both come from one cause — conversion.md's "clip paths are not read" — and they +// are opposite failures of the same box, which is why they are counted separately: +// +// [KindFigureBand] the box is bigger than the drawing, so the picture arrives +// with an empty band around it +// [KindFigureClipped] shapes drawn inside the box cross its edge, so part of the +// picture is cut off by the crop +// +// # What this has to work around +// +// [doc.Figure] carries how many shapes it holds and not which, so the shapes have +// to be matched to the figure here, by geometry. That is the one thing this package +// wanted from `internal/doc` and did not have; see the report. Matching is by area +// overlap ([minClipOverlap]) rather than by containment, because a containment test +// would define away the clipped case it is looking for. +func checkFigures(in Input) []Finding { + var out []Finding + for i := range in.Figures { + f := &in.Figures[i] + out = append(out, blankBand(f)...) + out = append(out, clipped(f, in.Ink[f.Page])...) + } + return out +} + +// blankBand reports a figure whose render is mostly margin on one side. +// +// It reads the PNG the conversion already carries and finds the box of pixels that +// are not the background. Nothing is decoded that a reader will not see: this is +// the same bytes the reader is served, which is what makes the finding a statement +// about the picture rather than about the geometry behind it. +func blankBand(f *doc.ConvertedFigure) []Finding { + l, r, t, b, ok := paintedMargins(f) + if !ok { + return nil + } + worst := math.Max(math.Max(l, r), math.Max(t, b)) + if worst <= maxBlankBand { + return nil + } + return []Finding{{ + Kind: KindFigureBand, Page: f.Page, Index: f.Index, + Got: worst, Want: maxBlankBand, + Count: f.Ink, Total: f.PixelWidth * f.PixelHeight, + Detail: fmt.Sprintf("page %d figure %d: its %.0fx%.0f box renders with blank "+ + "margins of %.0f left, %.0f right, %.0f top and %.0f bottom units "+ + "(want at most %.0f) — the box is bigger than the picture in it", + f.Page, f.Index, f.Rect.Width(), f.Rect.Height(), l, r, t, b, maxBlankBand), + }} +} + +// paintedMargins is how much blank space each side of a figure's render carries, +// in the units its box is in, and whether the render could be read at all. +// +// # Why the pixels and not the ink boxes +// +// The obvious measurement is the bounding box of [doc.Ink] against the figure's +// box, and it does not work, because the figure's box IS that bounding box: +// doc.FindFigures clusters the ink and takes its extent. Measured on the column +// manual, comparing the two gives 0.0 for 38 of its 46 figures and a negative +// number for several more, and page 14's two photographs — where a band was +// reported by eye — come out at 0.0 and 2.5. The comparison cannot see the fault +// because the fault is in the ink: a clipped path reports an extent larger than +// anything it paints, and both sides of that comparison are built from the same +// inflated extent. +// +// The pixels are downstream of the clip. Whatever poppler painted is what a reader +// sees, so a band in the render is a band, and the same measurement on the same 46 +// figures finds the four the eye finds. +func paintedMargins(f *doc.ConvertedFigure) (left, right, top, bottom float64, ok bool) { + if len(f.PNG) == 0 || f.PixelWidth <= 0 || f.Rect.Width() <= 0 { + return 0, 0, 0, 0, false + } + img, err := png.Decode(bytes.NewReader(f.PNG)) + if err != nil { + // A figure whose bytes will not decode is a different fault, and it is + // [doc.PageFigures]'s to report: it read the size out of the same bytes. + return 0, 0, 0, 0, false + } + box, painted := paintedBox(img) + if !painted { + return 0, 0, 0, 0, false + } + // Pixels per unit, read off the render rather than assumed: doc renders at twice + // the coordinate space's dpi, and taking the ratio means this stays right if that + // changes. + scale := float64(f.PixelWidth) / f.Rect.Width() + return float64(box.Min.X) / scale, float64(f.PixelWidth-box.Max.X) / scale, + float64(box.Min.Y) / scale, float64(f.PixelHeight-box.Max.Y) / scale, true +} + +// paintedBox is the bounding box of pixels that are not the background. +func paintedBox(img image.Image) (image.Rectangle, bool) { + b := img.Bounds() + minX, minY, maxX, maxY := b.Max.X, b.Max.Y, b.Min.X, b.Min.Y + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + r, g, bl, _ := img.At(x, y).RGBA() + if r > whiteCutoff && g > whiteCutoff && bl > whiteCutoff { + continue + } + if x < minX { + minX = x + } + if y < minY { + minY = y + } + if x >= maxX { + maxX = x + 1 + } + if y >= maxY { + maxY = y + 1 + } + } + } + if maxX <= minX || maxY <= minY { + return image.Rectangle{}, false + } + return image.Rect(minX, minY, maxX, maxY), true +} + +// clipped reports a figure whose drawn shapes cross the box the crop was taken +// from, so the picture is cut off at the edge. +// +// [doc.Figure] carries how many shapes it holds and not which, so the shapes are +// matched to the figure here, by geometry — that is the one thing this package +// wanted from internal/doc and did not have; see the report. Matching is by area +// overlap ([minClipOverlap]) rather than by containment, because a containment test +// would define away the case it is looking for. +// +// Which box answers which question is not interchangeable, and getting it wrong +// makes this check report the opposite of the truth. A shape belongs to the figure +// by how much of it falls inside the DRAWN extent, and it is cut by whether it +// leaves the RENDERED one. Asking both of the rendered box makes a crop grown onto +// its labels adopt whatever of the neighbouring drawing it now reaches over and +// then report that as its own picture being cut: measured, it took the sequential +// manual from 25 clipped figures to 27 the moment [doc.growToLabels] landed, with +// page 521 figure 2 "cut" by six units of a leader belonging to the drawing above +// it. Split this way the metric can only fall as the crop grows, which is what it +// is for. +func clipped(f *doc.ConvertedFigure, ink []doc.Ink) []Finding { + var inside, crossing int + var worstOver float64 + var worstShape doc.CellRect + own := f.DrawnExtent() + for j := range ink { + r := ink[j].Rect + if overlapFraction(r, own) < minClipOverlap { + continue + } + inside++ + if over := outside(r, f.Rect); over > clipSlack { + crossing++ + if over > worstOver { + worstOver, worstShape = over, r + } + } + } + if crossing == 0 { + return nil + } + return []Finding{{ + Kind: KindFigureClipped, Page: f.Page, Index: f.Index, + Got: worstOver, Want: clipSlack, Count: crossing, Total: inside, + Detail: fmt.Sprintf("page %d figure %d: %d of %d shapes cross the box "+ + "x=%.0f-%.0f y=%.0f-%.0f, the worst by %.0f units "+ + "(x=%.0f-%.0f y=%.0f-%.0f), so the crop cuts the picture", + f.Page, f.Index, crossing, inside, + f.Rect.X0, f.Rect.X1, f.Rect.Y0, f.Rect.Y1, worstOver, + worstShape.X0, worstShape.X1, worstShape.Y0, worstShape.Y1), + }} +} + +// overlapFraction is how much of inner falls inside outer, 1 for a shape wholly +// inside it, measured per axis and multiplied. +// +// Per axis because a drawn shape is routinely degenerate: a horizontal rule has +// zero height and a vertical one zero width, and an area comparison divides by +// zero on both. On a degenerate axis the question becomes containment, which is the +// same question asked of a shape with no thickness. +func overlapFraction(inner, outer doc.CellRect) float64 { + return overlap1D(inner.X0, inner.X1, outer.X0, outer.X1) * + overlap1D(inner.Y0, inner.Y1, outer.Y0, outer.Y1) +} + +// overlap1D is the share of [a0,a1] lying inside [b0,b1]. +func overlap1D(a0, a1, b0, b1 float64) float64 { + if a1 <= a0 { + if a0 >= b0 && a0 <= b1 { + return 1 + } + return 0 + } + in := math.Min(a1, b1) - math.Max(a0, b0) + if in <= 0 { + return 0 + } + return in / (a1 - a0) +} + +// outside is how far a shape reaches past a box, on its worst side. +func outside(inner, outer doc.CellRect) float64 { + return math.Max(math.Max(outer.X0-inner.X0, inner.X1-outer.X1), + math.Max(outer.Y0-inner.Y0, inner.Y1-outer.Y1)) +} diff --git a/internal/verify/joins.go b/internal/verify/joins.go new file mode 100644 index 0000000..23a71af --- /dev/null +++ b/internal/verify/joins.go @@ -0,0 +1,179 @@ +package verify + +import ( + "fmt" + "strings" + "unicode" + + "github.com/gordon2/manualbox/internal/doc" +) + +// Bounds on what reads as a bad join. Measured on both fixtures, quoted below. +const ( + // minGluedPart is how long each half of a suspected glued word must be before + // the split is believed. + // + // Without a floor the check reads any two-letter prefix as a word. Swept over + // both manuals, as words reported: + // + // floor 2 5 column / 3 sequential + // floor 3 0 / 3 + // floor 4 0 / 1 + // + // The five the column manual reports at 2 are the check running backwards: the + // block holds "трубка" as one word and it is `pdftotext` that split it, into + // "труб" and "ка", so the split it finds is the other tool's rather than ours. 3 + // removes all five and keeps both of the sequential manual's Thai cases; 4 loses + // two of the three. + minGluedPart = 3 +) + +// checkJoins reports text that reads as a typo, and fixes nothing. +// +// Three shapes, all mechanical: +// +// a hyphen followed by a space mid-word — "Gehäusede- ckel" +// two words glued with no space between them — "imGerät" +// a doubled space inside a block +// +// The first is deliberate in `doc` and stays deliberate. conversion.md records +// that hyphenation is not undone because German legitimately ends a line with a +// hyphen, and this check does not argue with it: it counts the cost, so a later +// tier that can afford a judgement knows which pages to read. +// +// # What the hyphen sub-check cannot separate, measured +// +// German elides a shared stem with exactly the same characters: "Ein- und +// Ausschalten" is correct prose and "Gehäusede- ckel" is a broken word, and both +// are a letter, a hyphen, a space and a lowercase letter. Nothing on the page +// separates them without a lexicon — a line-break hyphen is recognisable only from +// where the line broke, and a block has deliberately removed that. So this fires on +// both, the excerpt says which, and no filter pretends otherwise. +// +// Measured: 276 blocks of the column manual carrying 313 such hyphens, and 72 blocks +// of the sequential one carrying 79. Of the first 25 read by eye, 22 are line-break +// hyphenation ("Polster- möbel", "эксклю- зивное") and 3 are elision ("Vor- oder", +// "Elektro- und"), so the shape is mostly right and cannot be made entirely right. +// Requiring a lowercase letter after the space is what keeps the punctuation dash out: +// without it the same pass reports "230 V - 50 Hz" on every specification page. +// +// The glued sub-check needs the second opinion and is skipped without it: a word +// is believed to be two words only when the page prints both of them separately. +func checkJoins(in Input) []Finding { return checkJoinsWith(in, minGluedPart) } + +func checkJoinsWith(in Input, glueFloor int) []Finding { + printed := make(map[int]map[string]bool, len(in.Text)) + for i := range in.Text { + printed[in.Text[i].No] = tokenSet(in.Text[i].Text) + } + + var out []Finding + for i := range in.Blocks { + b := &in.Blocks[i] + out = append(out, hyphenJoins(b)...) + out = append(out, doubleSpaces(b)...) + if have := printed[b.Page]; have != nil { + out = append(out, gluedWords(b, have, glueFloor)...) + } + } + return out +} + +// hyphenJoins finds a hyphen followed by a space and a lowercase letter. +// +// Lowercase and letter on purpose: "230 V - 50 Hz" and "Amfibia 788/M - Modell" +// are a dash used as punctuation, and a capital, a digit or another dash after the +// space says so. The rune before the hyphen must be a letter for the same reason. +func hyphenJoins(b *doc.Block) []Finding { + r := []rune(b.Text) + var hits []string + for i := 1; i+2 < len(r); i++ { + if r[i] != '-' || !unicode.IsLetter(r[i-1]) { + continue + } + if r[i+1] != ' ' || !unicode.IsLower(r[i+2]) { + continue + } + hits = append(hits, excerpt(window(r, i, 12))) + } + if len(hits) == 0 { + return nil + } + return []Finding{{ + Kind: KindJoinHyphen, Page: b.Page, RegionX0: b.RegionX0, Index: b.Index, + Count: len(hits), Total: b.Chars, + Sample: excerpt(strings.Join(hits, " | ")), + Detail: fmt.Sprintf("page %d block %d at x=%.0f: %d hyphen(s) followed by a "+ + "space mid-word", b.Page, b.Index, b.RegionX0, len(hits)), + }} +} + +// doubleSpaces finds two or more spaces in a row. +func doubleSpaces(b *doc.Block) []Finding { + n := 0 + r := []rune(b.Text) + for i := 1; i < len(r); i++ { + if r[i] == ' ' && r[i-1] == ' ' { + n++ + } + } + if n == 0 { + return nil + } + return []Finding{{ + Kind: KindJoinSpace, Page: b.Page, RegionX0: b.RegionX0, Index: b.Index, + Count: n, Total: b.Chars, + Sample: excerpt(b.Text), + Detail: fmt.Sprintf("page %d block %d at x=%.0f: %d doubled space(s)", + b.Page, b.Index, b.RegionX0, n), + }} +} + +// gluedWords finds a word the page never printed whose two halves it did. +// +// This is the one join sub-check with evidence behind it rather than a shape: the +// word is absent from `pdftotext`'s reading of the page, and a split point exists +// where both halves are words that page printed. A word absent for any other +// reason — a ligature, a reversed right-to-left line — has no such split and is +// not reported here. +func gluedWords(b *doc.Block, printed map[string]bool, floor int) []Finding { + var hits []string + for _, tok := range tokens(b.Text) { + if printed[tok] { + continue + } + r := []rune(tok) + if len(r) < 2*floor { + continue + } + for i := floor; i <= len(r)-floor; i++ { + if printed[string(r[:i])] && printed[string(r[i:])] { + hits = append(hits, string(r[:i])+"|"+string(r[i:])) + break + } + } + } + if len(hits) == 0 { + return nil + } + return []Finding{{ + Kind: KindJoinGlued, Page: b.Page, RegionX0: b.RegionX0, Index: b.Index, + Count: len(hits), Total: b.Chars, + Sample: excerpt(strings.Join(hits, " ")), + Detail: fmt.Sprintf("page %d block %d at x=%.0f: %d word(s) glued from two the "+ + "page prints separately", b.Page, b.Index, b.RegionX0, len(hits)), + }} +} + +// window is the runes around an index, for an excerpt a person can find on the +// page. +func window(r []rune, at, radius int) string { + lo, hi := at-radius, at+radius + if lo < 0 { + lo = 0 + } + if hi > len(r) { + hi = len(r) + } + return string(r[lo:hi]) +} diff --git a/internal/verify/order.go b/internal/verify/order.go new file mode 100644 index 0000000..8873845 --- /dev/null +++ b/internal/verify/order.go @@ -0,0 +1,211 @@ +package verify + +import ( + "fmt" + "math" + "sort" + + "github.com/gordon2/manualbox/internal/doc" +) + +// orderSlack is how far up the page the next block may sit and still count as +// advancing down it, in the 1.5-scaled space. +// +// Two blocks of one column legitimately share a top to within rounding — a list +// marker folded into its text, a heading and the run beside it — and a strict +// comparison would report those. 2.0 is the baseline tolerance columns.go measures +// at 15% of a median run height, which is about 2.5 units against either manual's +// line pitch, rounded down so that this never accepts a real backwards jump: the +// smallest one on the column manual's page 62, the interleaving case conversion.md +// describes, is 16 units. +const ( + orderSlack = 2.0 + + // minOrderGap is how far apart two blocks' x-ranges must be before they are + // believed to be in different columns, in the same units. + // + // Disjointness alone is not enough, and the measurement says why: a folio at + // x=43-47 and the paragraph above it at x=49-293 are disjoint by two units and + // are plainly the same column. A real gutter is an order wider — the narrowest + // on the column manual is 13 units, which columns.go measures on its page 68 — + // so the guard sits just under that. + minOrderGap = 12.0 + + // minOrderChars is how much text each block must hold before a switch between + // them is read as interleaving. + // + // This is the page-furniture guard, and without it the check reports furniture + // and little else. The sequential manual prints a two-letter language badge at + // x=27-41 below the running head on 110 pages, and that badge is a block: it is + // disjoint from the heading above it and lower down the page, which is the + // violation's exact shape. The column manual's parts pages do the same with + // numbered callouts scattered around a diagram. + // + // Swept over both manuals, as findings (column / sequential): + // + // chars>=0 chars>=8 chars>=16 chars>=24 + // gap>=0 67 / 687 0 / 71 0 / 37 0 / 11 + // gap>=12 61 / 662 0 / 70 0 / 37 0 / 11 + // gap>=20 58 / 326 0 / 69 0 / 36 0 / 11 + // + // A floor of 8 runes already removes every one of the column manual's 67, all of + // which are a callout number or a folio. 16 is chosen over 8 because the 34 the + // sequential manual loses between them are the short interval labels of the same + // grid its 37 remaining findings name, so nothing new is lost, and because a + // block of interleaved prose is a printed line or more — the page-62 case + // conversion.md describes runs 40 to 80 runes. + // + // What survives at the defaults is one real class, and its concentration is what + // makes it believable: 37 findings on 26 pages, the routine-maintenance page of + // one language section after another, where an unruled grid of intervals — + // invisible to the table detector by conversion.md's own account — is read in + // columns. + minOrderChars = 16 +) + +// orderGuards are the three bounds, taken as a value so a test can sweep them +// over both whole documents. That is how every threshold in this project is set. +type orderGuards struct { + slack, minGap float64 + minChars int +} + +var defaultOrderGuards = orderGuards{ + slack: orderSlack, minGap: minOrderGap, minChars: minOrderChars, +} + +// checkOrder answers "would a person read this in the order it is stored", and it +// is the check that catches the failure conversion.md spends the most words on. +// +// # What a violation is, and why it is not simply "y increases" +// +// A region of several columns is read column by column, so y going backwards is +// CORRECT at every column boundary — from the foot of one column to the head of the +// next. The wrong thing is the opposite: switching column while continuing DOWN the +// page, which is what sorting a whole-page region's runs by y then x produces and +// what conversion.md records on the column manual's page 62 ("rial bitte +// umweltgerecht. sich bei gewerblicher Benutzung…"). +// +// So a finding is two consecutive blocks of one page and region whose x-ranges do +// not overlap at all — they are in different columns — where the second does not +// start above the first. Horizontal disjointness rather than a column id because +// [doc.Block] carries no column, only its own box; that is the second thing this +// package wanted from `internal/doc` and worked around. +// +// # Table cells are excluded, and must be +// +// [doc.BlockTable] cells are emitted row-major, deliberately: conversion.md records +// that reading down every question and then down every answer was the limitation +// row-major reading fixed. Row-major is exactly this check's violation shape — cell +// (r,c) to (r,c+1) is disjoint and level — so a table page would report one finding +// per cell. Measured: including table cells takes the column manual from 0 findings +// to 207 and the sequential one from 686 to 3,158, and every added one is correct +// row-major reading. +// +// Excluded from the comparison, though, is not the same as invisible, and reading them +// as invisible is a blind spot this check had. Two prose blocks with a table between +// them are not consecutive in reading order at all, so "the second does not start above +// the first" says nothing about either. The measured case is the column manual's four +// troubleshooting pages: each prints two side-by-side tables whose header rows sit above +// a top border that is not drawn, so those headers are the page's only prose, and the +// left table's header is followed — across twenty-six cells the check cannot see — by +// the right table's header at the same y. That is the two columns read in the right +// order, and it has the exact shape of interleaving once the cells are dropped. So a +// table between two blocks now breaks the chain rather than closing over it. +// # A figure's callout labels are excluded, and must be +// +// A block [doc.Block.Callout] marks is one printed line of a label a reader is shown +// BESIDE a picture, not in the prose. [doc.RegionBlocks] appends those after the +// content for the same reason it appends the furniture — they have no place in reading +// order to be put back into — so judging them here is asking where in the reading order +// something that is not in the reading order comes. +// +// It is not hypothetical. Page 521 of the sequential manual sets its labels in two +// columns either side of a drawing, so two consecutive label blocks are disjoint in x +// and descending in y, which is this check's violation shape exactly. Including them +// reported one finding there and moved another on page 522. +// +// Page furniture is NOT excluded here, and that is worth stating because it looks +// inconsistent. It escapes by position rather than by rule: a tab, a folio and a +// running head sit at the top or the bottom of a page, so the block after one is almost +// always further UP the page and the check skips it before it can fire. That is luck +// holding rather than a decision, and a document whose furniture sat mid-page would +// find it. It is left alone because changing it would move a number this change has +// nothing to do with. +func checkOrder(blocks []doc.Block) []Finding { + return checkOrderWith(blocks, defaultOrderGuards) +} + +func checkOrderWith(blocks []doc.Block, g orderGuards) []Finding { + type key struct { + page int + x0 float64 + } + groups := make(map[key][]int, 16) + for i := range blocks { + if blocks[i].Callout { + continue + } + k := key{blocks[i].Page, blocks[i].RegionX0} + groups[k] = append(groups[k], i) + } + + keys := make([]key, 0, len(groups)) + for k := range groups { + keys = append(keys, k) + } + sort.Slice(keys, func(a, b int) bool { + if keys[a].page != keys[b].page { + return keys[a].page < keys[b].page + } + return keys[a].x0 < keys[b].x0 + }) + + var out []Finding + for _, k := range keys { + idx := groups[k] + sort.Slice(idx, func(a, b int) bool { return blocks[idx[a]].Index < blocks[idx[b]].Index }) + judged := 0 + for j := range idx { + if blocks[idx[j]].Kind != doc.BlockTable { + judged++ + } + } + var prev *doc.Block + for j := range idx { + cur := &blocks[idx[j]] + if cur.Kind == doc.BlockTable { + prev = nil + continue + } + last := prev + prev = cur + if last == nil { + continue + } + if gapX(last, cur) < g.minGap || cur.Y0 < last.Y0-g.slack { + continue + } + if last.Chars < g.minChars || cur.Chars < g.minChars { + continue + } + out = append(out, Finding{ + Kind: KindReadingOrder, Page: cur.Page, RegionX0: cur.RegionX0, Index: cur.Index, + Got: cur.Y0, Want: last.Y0, Count: 1, Total: judged, + Sample: excerpt(last.Text + " → " + cur.Text), + Detail: fmt.Sprintf("page %d region x=%.0f: block %d at x=%.0f-%.0f y=%.0f is "+ + "read after block %d at x=%.0f-%.0f y=%.0f — a different column, no "+ + "further up the page, which is what interleaving looks like", + cur.Page, cur.RegionX0, cur.Index, cur.X0, cur.X1, cur.Y0, + last.Index, last.X0, last.X1, last.Y0), + }) + } + } + return out +} + +// gapX is the horizontal distance between two blocks, negative when they overlap. +// A banner set across the measure overlaps every column, so it is never a switch. +func gapX(a, b *doc.Block) float64 { + return math.Max(b.X0-a.X1, a.X0-b.X1) +} diff --git a/internal/verify/text.go b/internal/verify/text.go new file mode 100644 index 0000000..11322a9 --- /dev/null +++ b/internal/verify/text.go @@ -0,0 +1,530 @@ +package verify + +import ( + "fmt" + "strings" + "unicode" +) + +// Bounds on what counts as dropped content and as an invented word. +// +// Every one is measured against both fixtures — the 68-page parallel-columns +// manual (internal/doc/testdata/fixtures/thomas-drybox-amfibia.json) and the +// 560-page sequential one (dreame-l40-ultra.json) — converted for EVERY language +// each holds, so that a page carrying five languages is compared against all five. +// The measurements are quoted at each constant, in the style columns.go set. +const ( + // minCoverage is how little of `pdftotext`'s text a page's blocks may hold + // before the page is reported as having dropped content. + // + // The honest baseline is below 1 and that is not a defect. Blocks are built from + // doc's usableRuns, which drops sub-legible production artifacts — the column + // manual's text layer carries an InDesign filename slug and an export timestamp + // 260 times each, 8% of its runs — and `pdftotext` reports every one of them. It + // also reports rotated text, which that filter drops, and furniture outside any + // region. So the question is not "is it 1" but "is it what a correct conversion + // scores". + // + // Measured per page over every language of both manuals: + // + // column manual 66 pages judged: median 0.974, min 0.801 (page 5), then + // 0.802, 0.858, 0.859, 0.871; 8 pages under 0.90, 0 under 0.80 + // sequential 552 pages judged: median 1.000, min 0.952 (page 189), + // 0 pages under 0.95 + // + // So a correct conversion of these two documents floors at 0.80, and the pages + // that get there are the artifact-heavy front matter the filter is for. 0.75 + // leaves that floor about six points of headroom while still reporting a page + // that lost a quarter of itself. Set at 0.80 it would report page 5 of the column + // manual on a thousandth of a point, which is a threshold pinned to one page. + // + // A ratio slightly ABOVE 1 is also normal — the sequential manual's maximum is + // 1.003 — because a block joins a hyphenated word that `pdftotext` leaves broken + // across two lines, and the join is one character shorter than the break. + minCoverage = 0.75 + + // minCoverageText is how much text a page needs before its ratio is judged. + // A page holding a folio and a language badge scores whatever those two runs + // happen to do; one page of the sequential manual is that page. The same floor + // [doc.MinTextChars] sets, and for the same reason. + minCoverageText = 50 + + // minTokenRunes is how long a word must be to enter the comparison. + // + // It barely moves the numbers and it is still worth having. Measured over the + // left-to-right pages of both manuals, the share of block words absent from + // `pdftotext` runs 1.84% / 1.95% / 1.98% on the column manual at a floor of 1, 2 + // and 3 runes, and 0.47% / 0.45% / 0.48% on the sequential one. What the floor + // removes is 1,756 of the column manual's 31,450 tokens, and they are bullets, + // folios, list markers and unit letters — tokens on which the two tools + // legitimately disagree (a printed bullet arrives as U+2022 from one and a hyphen + // from the other) and which carry no evidence either way. 2 keeps every word. + minTokenRunes = 2 + + // maxInventedShare and minInventedTokens are how much of a block may be absent + // from `pdftotext` before the block is reported. + // + // Not zero, and the measurement says why: 1.95% of the column manual's words and + // 0.45% of the sequential one's are absent from a CORRECT conversion, because the + // two tools break lines and normalise combining marks differently. Reporting + // every one of them is 280 and 322 blocks of noise. + // + // Swept together over both manuals, as blocks reported: + // + // share > 0.00 231 column / 190 sequential + // share > 0.10 113 / 179 + // share > 0.20 17 / 169 + // share > 0.34 4 / 153 + // share > 0.50 0 / 118 + // + // 0.34 is where the column manual stops reporting anything but real faults: its + // remaining 4 are table cells where the two tools disagree about where a Cyrillic + // or Kazakh word divides. It is deliberately not pushed to 0.50, because the + // sequential manual's 153 findings at 0.34 are real — its Thai section arrives + // with words broken at a vowel — and a threshold chosen to silence one document + // would hide a defect in the other. + // + // The floor of 2 absent words is what keeps a one-word block from reporting + // itself at 100%: a unit symbol or a bullet is a block, and one absent word out + // of one is not evidence. + maxInventedShare = 0.34 + minInventedTokens = 2 + + // rtlShare is how much of a page's words must be right-to-left before the page is + // reported as [KindRightToLeft] rather than block by block. + // + // Measured: 32 pages of the sequential manual are 0.65 to 1.00 right-to-left by + // word — its Hebrew and Arabic sections, the rest of each page being Latin part + // numbers — and every other page of either manual is exactly 0.000. 0.5 sits in + // the middle of that, and nothing between 0.05 and 0.6 changes the answer. + // + // Still 32 after doc/bidi.go, because reversing a line does not change which + // script its characters are in. What changed is that 7 of the 32 now hold no + // absent word at all, so only 25 reach [minReversibleWords] to be judged, and + // none of those 25 is judged against it any more: see that constant. + rtlShare = 0.5 + + // minReversibleWords is how many of a right-to-left page's words must be absent + // from `pdftotext` AND present in it reversed before the page is reported as + // [KindRightToLeft] rather than block by block. + // + // # Why the check needed this at all + // + // It used to fire on a right-to-left page with any absent word whatsoever, which + // was the same question as "is this page Hebrew or Arabic" for as long as every + // such page arrived backwards. Once doc/bidi.go put the order right, the two + // questions came apart and the check went on answering the first while its name + // and its Detail string claimed the second: 25 pages of the sequential manual + // still fired, on 220 absent words in 6,834, three of them on one page of 510. + // A finding that reports pages which are not reversed cannot reach zero, and + // says nothing on the way there. + // + // The evidence for reversal was already being counted and not used: a word that + // is absent from the reference and present in it BACKWARDS was not extracted + // wrong in some general way, it was extracted in visual order. That is the + // signature, and nothing else this pipeline does produces it. + // + // # Why a count and not a share, which is the part worth keeping + // + // Three measurements over the sequential manual, absent words present reversed: + // + // before bidi.go 32 pages, 8,120 absent, 7,938 reversible; per-page + // share 0.913 (page 188) to 1.000, 8 pages at 1.000 + // majority direction 25 pages, 220 absent, 18 reversible; per-page + // share .600 .538 .125 .100 .091 .059, nineteen 0.000 + // region direction 25 pages, 202 absent, 0 reversible; ALL 0.000 + // + // A share of the absent words is the obvious rule, it had a real gap to sit in at + // the middle measurement — nothing between 0.600 and 0.913 — and it was the wrong + // rule, because those 18 words were not noise. Every one was a genuine word still + // reversed: `תבותכב` where the page prints `בכתובת`, `ليلد` for `دليل`. A share of + // 0.65 would have reported zero while six pages were reversed, and measured, it + // would not merely have renamed them: pages 188, 204 and 207 fell through to a + // [KindInvented] block, but 189, 191 and 205 held one reversed word in a block + // that was otherwise right — under both [maxInventedShare] and + // [minInventedTokens] — and vanished entirely. + // + // That is why the rule is a count, and it is worth keeping the argument even + // though the corpus can no longer make it: the third row is zero, so a sweep over + // this document would now choose anything. The argument is kept where it stays + // falsifiable instead — TestAShareOfAbsentWordsWouldHideAReversal builds the + // measured page-191 shape by hand and fails if the rule is ever changed back. + // + // # There is nothing under the floor, and 1 is still the only value + // + // The sweep chose the RULE, never the VALUE. 1 is not fitted to anything: it is + // the statement that one word of evidence is evidence. 0 restores the defect this + // constant was added to remove, since it requires no evidence at all, and any + // value above 1 asserts that some quantity of reversed text is acceptable, which + // nothing here would defend and which the corpus gives no basis for. So it stays + // at 1 with its measurements recorded as history, and the load-bearing test moved + // from "which threshold" to "is it zero" — see [TestNoTextIsStoredReversed]. + // + // The one number that did fit the data is gone with it: the floor was 1 rather + // than 2 because over the nineteen right-to-left pages holding no reversal, 140 + // absent words produced not one coincidental match. `שי` for `יש` was the only + // two-rune match in the whole corpus and it sat among five unambiguous ones. + // + // # What this can and cannot see + // + // It sees a page holding at least one word that this pipeline read backwards and + // `pdftotext` did not. It is named per page, which overstated the extent while + // there was anything to overstate: the residual was one LINE on each page. + // + // It cannot see a reversal both tools make — they share no code, so this has no + // example, but it is not ruled out. It cannot see a reversed word whose reverse is + // missing from the reference for a second reason, which is what Arabic costs it: + // `pdftohtml` returns unshaped letter forms, so a word can be both reversed and + // unshaped and then only the shaping shows. It cannot see a reversed PALINDROME. + // + // AND IT CANNOT SEE A REORDERING THAT PRESERVES THE WORD SET, which is the + // limitation worth knowing about, because it is the one that has actually cost + // something. A zero from [KindRightToLeft] means no word is stored as its own + // reverse. It does not mean the words are in the right order. + // + // Set membership per page is what [checkText] compares, for the reasons given there, + // so word ORDER is outside it by construction — and all three of doc/bidi.go's + // run-ORDER defects were invisible here, every token still present in the reference + // each time: + // + // page 204's support URL, its seventeen runs reversed — surfaced sideways as a + // [KindJoinHyphen], the side effect rather than the defect + // page 211's list marker `1.` stored as `. 1` — DID NOT SURFACE AT ALL, and was + // caught only because a pinned block count moved by 43 + // page 204's laser standard, `EN1:2014/ 60825-` for `EN 60825- 1:2014/` — sideways + // again, as a [KindJoinGlued] on the one space the transposition also lost + // + // Two were caught by another check reacting to a side effect and one by an + // unexplained pinned count, which is the argument conversion.md makes for pinning + // counts before you can explain them. This check reported 0 throughout all three. + // + // The contrast is what says the sharpening above was worth doing rather than merely + // tidy: bidi.go's FOURTH defect, the direction rule that left six lines unrepaired, + // really did store words as their own reverse — and this check named those six pages + // and 18 words exactly, which the version that fired on any right-to-left page could + // not have distinguished from noise. + // + // [checkOrder] asks the order question of blocks and nothing asks it of words. That + // gap is deliberately open, and conversion.md carries the design: what the + // comparison would be against — `pdftotext`'s byte order already IS reading order + // once the bidi controls are stripped, which the tokeniser does anyway — and what + // makes it real work, which is matching lines to a per-page reference and not + // reporting the reflow and column interleaving that are already reported elsewhere. + minReversibleWords = 1 +) + +// checkCoverage answers "did we drop content", by comparing the blocks of a page +// against `pdftotext`'s reading of the same page. +// +// The comparison is non-space runes on both sides. Runes for the reason the whole +// project counts runes; non-space because `pdftotext` preserves the printed line +// breaks and column padding a block deliberately removes, so counting whitespace +// would compare a layout against a reflow. +// +// The ratio is expected to be below 1 for real reasons, which is why [minCoverage] +// is 0.75 and not 1: see its measurement. +// +// # Page furniture is NOT counted, on purpose, and it lowers every ratio +// +// A block [doc.Furniture] claimed is a language tab, a folio or a running head. It +// is really printed on the page, so `pdftotext` reports it and counting it would +// leave this ratio exactly where it was before that pass existed. It is skipped +// anyway, and the reason is that this check is the only thing that can refute the +// furniture rule. Count furniture and a rule that wrongly claims a paragraph is +// invisible here, because the paragraph is still in the sum. Skip it and the same +// mistake reads as a page that dropped a paragraph, which is what a coverage +// finding is for. The cost is a permanently lower floor, measured at +// [minCoverage]. +func checkCoverage(in Input, scope []int) ([]PageCoverage, []Finding) { + blocks := make(map[int]int, len(scope)) + for i := range in.Blocks { + if in.Blocks[i].Furniture { + continue + } + blocks[in.Blocks[i].Page] += countGraphemes(in.Blocks[i].Text) + } + text := make(map[int]int, len(in.Text)) + for i := range in.Text { + text[in.Text[i].No] = countGraphemes(in.Text[i].Text) + } + + cov := make([]PageCoverage, 0, len(scope)) + var out []Finding + for _, p := range scope { + c := PageCoverage{Page: p, Blocks: blocks[p], Text: text[p]} + if c.Text > 0 { + c.Ratio = float64(c.Blocks) / float64(c.Text) + } + cov = append(cov, c) + if c.Text < minCoverageText || c.Ratio >= minCoverage { + continue + } + out = append(out, Finding{ + Kind: KindCoverage, Page: p, + Got: c.Ratio, Want: minCoverage, + Count: c.Blocks, Total: c.Text, + Detail: fmt.Sprintf("page %d: blocks hold %d of pdftotext's %d characters "+ + "(%.2f, want at least %.2f)", p, c.Blocks, c.Text, c.Ratio, minCoverage), + }) + } + return cov, out +} + +// checkText answers "did we invent text", which coverage cannot. +// +// Interleaved columns keep every character of a page and destroy every word, so +// the count matches and the reading does not. Comparing words catches it: a word +// in a converted block that appears nowhere in `pdftotext`'s reading of the same +// page was assembled by this pipeline rather than printed on the paper. +// +// # The normalisation, which is the whole of the check's precision +// +// A token is a maximal run of letters, digits and combining marks, lowercased, +// with Unicode format characters stripped first — the bidi controls `pdftotext` +// wraps a right-to-left line in, which CONTRIBUTING.md records as having silently +// lost whole sections once already. Everything else is a separator, so punctuation, +// the soft hyphen and the printed bullet never enter the comparison, and neither +// does a difference of opinion about them. Tokens shorter than [minTokenRunes] are +// dropped, measured. +// +// Set membership per page, not a multiset and not a sequence. A multiset would +// report a legitimate difference of one occurrence, and a sequence would report +// the reading order this check is not about — [checkOrder] is. +// +// # Right-to-left is a known defect and gets its own finding +// +// conversion.md records that `pdftohtml -xml` returns a right-to-left line in +// visual order, and doc/bidi.go now repairs it. Where the repair does not reach, +// a page would report hundreds of invented words; a page that is more than +// [rtlShare] right-to-left by token AND carries at least [minReversibleWords] +// words absent from `pdftotext` but present in it reversed gets one +// [KindRightToLeft] finding instead of one per block. +// +// Both halves of that are needed and the second is the one measured hardest: being +// Hebrew is not being backwards, so a right-to-left page whose absent words are +// ordinary disagreement is judged block by block like any other. The reversal +// itself is the evidence, it is what the finding's name claims, and it is what +// makes the count able to reach zero. See [minReversibleWords]. +func checkText(in Input, scope []int) []Finding { + return checkTextWith(in, scope, defaultTextGuards) +} + +// textGuards are the bounds, taken as a value so a test can sweep them over both +// whole documents. Every threshold in this project is set that way; see +// doc/figures.go's figureGuards. +type textGuards struct { + minToken int + maxInvented float64 + minAbsent int + rtl float64 + reversible int +} + +var defaultTextGuards = textGuards{ + minToken: minTokenRunes, maxInvented: maxInventedShare, + minAbsent: minInventedTokens, rtl: rtlShare, reversible: minReversibleWords, +} + +func checkTextWith(in Input, scope []int, g textGuards) []Finding { + inScope := make(map[int]bool, len(scope)) + for _, p := range scope { + inScope[p] = true + } + printed := make(map[int]map[string]bool, len(in.Text)) + for i := range in.Text { + if inScope[in.Text[i].No] { + printed[in.Text[i].No] = tokenSetMin(in.Text[i].Text, g.minToken) + } + } + + type pageState struct { + tokens, rtl, absent, reversible int + reversed string + } + state := make(map[int]*pageState, len(scope)) + byPage := make(map[int][]Finding, len(scope)) + + for i := range in.Blocks { + b := &in.Blocks[i] + if !inScope[b.Page] { + continue + } + have := printed[b.Page] + st := state[b.Page] + if st == nil { + st = &pageState{} + state[b.Page] = st + } + + var absent []string + toks := tokensMin(b.Text, g.minToken) + for _, t := range toks { + st.tokens++ + if isRightToLeft(t) { + st.rtl++ + } + if have[t] { + continue + } + absent = append(absent, t) + st.absent++ + if have[reverse(t)] { + st.reversible++ + // The word as stored and as the page prints it, side by side. This is + // the readable proof and it is why the finding exists at all, so it is + // collected here rather than reconstructed from the page later. + if len([]rune(st.reversed)) < sampleRunes { + st.reversed += t + " for " + reverse(t) + "; " + } + } + } + if len(absent) == 0 { + continue + } + if len(absent) < g.minAbsent || + float64(len(absent))/float64(len(toks)) <= g.maxInvented { + continue + } + byPage[b.Page] = append(byPage[b.Page], Finding{ + Kind: KindInvented, Page: b.Page, RegionX0: b.RegionX0, Index: b.Index, + Count: len(absent), Total: len(toks), + Got: float64(len(absent)) / float64(len(toks)), + Want: g.maxInvented, + Sample: excerpt(strings.Join(absent, " ")), + Detail: fmt.Sprintf("page %d block %d at x=%.0f: %d of %d words appear "+ + "nowhere in pdftotext's reading of the page", + b.Page, b.Index, b.RegionX0, len(absent), len(toks)), + }) + } + + var out []Finding + for _, p := range scope { + st := state[p] + if st == nil || st.tokens == 0 { + continue + } + if float64(st.rtl)/float64(st.tokens) <= g.rtl { + out = append(out, byPage[p]...) + continue + } + if st.absent == 0 || st.reversible < g.reversible { + // Absent words with no reversal behind them are ordinary disagreement + // between the two extractions, whatever direction the page reads in, so + // they are judged block by block like every other page. See + // [minReversibleWords] for what happens to a page that is judged the + // other way round. + out = append(out, byPage[p]...) + continue + } + out = append(out, Finding{ + Kind: KindRightToLeft, Page: p, + Count: st.absent, Total: st.tokens, + // Got is the reversed words counted, Want the fewest that raise this at + // all — the same "measurement and the bound it failed" every other finding + // carries, where this one used to put the absent count in Want and so read + // as though every absent word were expected to reverse. + Got: float64(st.reversible), + Want: float64(g.reversible), + // The excerpt is the reversed words with the printed spelling beside each, + // which is the readable proof of the cause. It was the absent words, which + // was the same list while whole pages were reversed and is mostly ordinary + // disagreement now. + Sample: excerpt(strings.TrimSuffix(st.reversed, "; ")), + Detail: fmt.Sprintf("page %d reads right to left: %d of %d words are absent "+ + "from pdftotext, %d of them present when reversed — the known "+ + "pdftohtml visual-order defect, see docs/design/conversion.md", + p, st.absent, st.tokens, st.reversible), + }) + } + return out +} + +// tokens splits text the way both extractions can agree on. See [checkText] for +// why this is the normalisation and not another. +func tokens(s string) []string { return tokensMin(s, minTokenRunes) } + +func tokensMin(s string, minRunes int) []string { + var out []string + var cur []rune + flush := func() { + if len(cur) >= minRunes { + out = append(out, string(cur)) + } + cur = cur[:0] + } + for _, r := range strings.ToLower(s) { + switch { + case unicode.Is(unicode.Cf, r): + // A bidi control is not a separator: dropping it joins the runes either + // side, which is what they are on the page. + case unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.Is(unicode.Mn, r) || + unicode.Is(unicode.Mc, r): + cur = append(cur, r) + default: + flush() + } + } + flush() + return out +} + +func tokenSet(s string) map[string]bool { return tokenSetMin(s, minTokenRunes) } + +func tokenSetMin(s string, minRunes int) map[string]bool { + toks := tokensMin(s, minRunes) + out := make(map[string]bool, len(toks)) + for _, t := range toks { + out[t] = true + } + return out +} + +// countGraphemes counts non-space runes. Runes, not bytes — half of a real manual +// is Cyrillic, Greek, Hebrew, Arabic or CJK. +func countGraphemes(s string) int { + n := 0 + for _, r := range s { + if !unicode.IsSpace(r) && !unicode.Is(unicode.Cf, r) { + n++ + } + } + return n +} + +// rightToLeftScripts are the scripts these manuals actually print in that +// direction. Hebrew and Arabic are the two the sequential manual has; the other +// three cost nothing and stop the check being wrong about a document nobody here +// has seen. +var rightToLeftScripts = []*unicode.RangeTable{ + unicode.Hebrew, unicode.Arabic, unicode.Syriac, unicode.Thaana, unicode.Nko, +} + +// isRightToLeft reports whether a token is written in a right-to-left script, +// decided by its first letter. First letter and not a majority vote: a Hebrew word +// with a Latin unit suffix is still a Hebrew word, and it is the line's direction +// this stands in for. +func isRightToLeft(tok string) bool { + for _, r := range tok { + if !unicode.IsLetter(r) { + continue + } + for _, tab := range rightToLeftScripts { + if unicode.Is(tab, r) { + return true + } + } + return false + } + return false +} + +// reverse reverses a string's runes. Used only as evidence for the right-to-left +// finding — conversion.md is explicit that reversing in the view would be wrong +// twice over, and nothing here repairs anything. +func reverse(s string) string { + r := []rune(s) + for i, j := 0, len(r)-1; i < j; i, j = i+1, j-1 { + r[i], r[j] = r[j], r[i] + } + return string(r) +} diff --git a/internal/verify/text_internal_test.go b/internal/verify/text_internal_test.go new file mode 100644 index 0000000..6b7cdf6 --- /dev/null +++ b/internal/verify/text_internal_test.go @@ -0,0 +1,255 @@ +package verify + +import ( + "context" + "os" + "sort" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" +) + +// TestNoTextIsStoredReversed is the acceptance criterion for doc/bidi.go, asserted +// from the outside: over the only document either fixture holds that reads right to +// left, not one word is absent from `pdftotext` and present in it backwards. +// +// It began as a threshold sweep for [minReversibleWords], on the model of +// doc/figures_internal_test.go's TestGuardSweep, and it is not one any more because +// there is nothing left to sweep. Three measurements, the first two recorded at that +// constant and the third printed by this test every time it runs: +// +// before bidi.go 32 pages, 8,120 absent, 7,938 reversible +// majority direction 25 pages, 220 absent, 18 reversible on 6 pages +// region direction 25 pages, 202 absent, 0 reversible +// run-level islands 27 pages, 235 absent, 0 reversible +// islands anchored 25 pages, 202 absent, 0 reversible +// printed order kept 25 pages, 200 absent, 0 reversible +// +// The third row was neither an improvement nor a reversal: a run-level island test that +// swallowed neutral runs turned six Arabic and Hebrew list markers round, `. 1` for +// `1.`, which added absent words and brought two more pages over [rtlShare] without any +// of them being backwards. Anchoring the island at both ends put it back, exactly, and +// the last two absent words went when page 204's laser standard stopped being repaired +// twice over. The reversible column is 0 through all of it, which is what this test is +// for; the rest is [KindInvented]'s business and conversion.md has the story. +// +// A sweep over an empty population would accept any value and prove nothing, so the +// assertion moved from "which threshold" to "is it zero" — which is stronger, and is +// what would actually break if direction handling regressed. What the sweep decided, +// a count of reversed words rather than a share of the absent ones, is pinned by +// [TestAShareOfAbsentWordsWouldHideAReversal] instead, hermetically, because this +// corpus can no longer demonstrate it. +// +// The distribution is still printed. Someone changing doc's direction handling wants +// to see what this document does, not to be told that it is fine. +func TestNoTextIsStoredReversed(t *testing.T) { + in := sequentialInput(t) + scope := pageScope(in) + rows := rightToLeftPages(t, in, scope) + + t.Logf("%d right-to-left page(s), by how much of their absent text is present reversed:", + len(rows)) + var absent, reversible int + var flagged []int + for _, r := range rows { + absent += r.absent + reversible += r.reversible + if r.reversible > 0 { + flagged = append(flagged, r.page) + } + t.Logf(" page %3d: %3d of %4d words absent, %3d present reversed (%.3f) %s", + r.page, r.absent, r.tokens, r.reversible, r.share, r.sample) + for _, w := range reversibleWords(in, r.page) { + // Every word behind a non-zero verdict, printed, because the whole + // question is whether such a word is a real reversal. Last time all 18 + // were, and naming them is what found the cause. + t.Logf(" %s", w) + } + } + t.Logf(" %d page(s), %d absent word(s), %d present reversed", len(rows), absent, reversible) + + // The assertion. 200 absent words remain and none is backwards: Arabic shaping and + // combining-mark disagreement, [KindInvented]'s business, and none of it a reversal. + if reversible != 0 { + t.Errorf("%d word(s) on page(s) %v are absent from pdftotext and present in it "+ + "reversed; doc/bidi.go is meant to leave none, and each word is logged "+ + "above with the block it sits in", reversible, flagged) + } + + // With the population empty the constant decides nothing, and that is worth + // showing rather than asserting: every value gives the same report. + for _, v := range []int{1, 2, 5, 50} { + g := defaultTextGuards + g.reversible = v + pages, blocks := countKinds(checkTextWith(in, scope, g)) + t.Logf(" minReversibleWords=%-2d -> %d right-to-left page(s), %d invented-text block(s)", + v, pages, blocks) + } +} + +// TestAShareOfAbsentWordsWouldHideAReversal keeps the one design decision the +// fixture measurement can no longer defend, now that the residual it was measured on +// is zero. +// +// The obvious rule for [minReversibleWords] is a share of the page's absent words +// rather than a count of the reversed ones, and when it was chosen that share had a +// real gap to sit in: nothing between 0.600 and 0.913. This is the shape that rules +// it out, taken from the measured page 191 — one line reversed on a page that is +// otherwise right, so one reversed word among ordinary disagreement. A count reports +// it. A share buries it under the noise on its own page, and it does not even fall +// through to [KindInvented], because one absent word in a block is under +// [minInventedTokens]. +// +// So a share would have called page 191 clean while `אפלקציית` was stored as +// `תייצקלפא`, and the check would have read zero for the wrong reason. +func TestAShareOfAbsentWordsWouldHideAReversal(t *testing.T) { + // The page as pdftotext reads it, wrapped in the bidi controls it uses. Written + // as escapes because they are invisible, the reason verify_test.go's rtlEmbed + // gives — a test whose input cannot be seen in the source is one nobody can check. + const printed = "\u202b" + "אפלקציית Dreamehome תואמתמ הוראות בטמפרטורה גבוהה " + + "ובלחות רבה יש להימנע משימוש" + "\u202c" + in := Input{ + Blocks: []doc.Block{ + // One reversed word, in a short block of its own, as page 191 has it. + {Page: 191, Index: 0, Text: "Dreamehome תייצקלפא", Chars: 19, Lines: 1, + X0: 700, X1: 860, Y0: 100, Y1: 118}, + // The rest of the page, correctly ordered but with one word the + // reference spells differently — the ordinary disagreement that raises + // the absent count a share would be divided by. + {Page: 191, Index: 1, Text: "תואמתמ הוראות בטמפרטורת גבוהה ובלחות רבה " + + "יש להימנע משימוש", Chars: 57, Lines: 1, + X0: 500, X1: 860, Y0: 130, Y1: 148}, + }, + Text: []doc.Page{{No: 191, Text: printed, Chars: len([]rune(printed))}}, + } + + // The rule as it stands: the reversed word is found and the page is named. + rep := Inspect(in) + if got := rep.Count(KindRightToLeft); got != 1 { + t.Fatalf("the count rule missed a page with a reversed word on it: %+v", + rep.Findings) + } + if got := int(rep.Findings[0].Got); got != 1 { + t.Errorf("the finding claims %d reversed word(s), want 1", got) + } + + // The rejected rule, standing in for any share a single reversal cannot reach. + // Nothing is reported at all — not renamed, gone — which is the whole argument. + g := defaultTextGuards + g.reversible = 1 << 30 + if found := checkTextWith(in, pageScope(in), g); len(found) != 0 { + t.Fatalf("this fixture no longer demonstrates the trap: judged block by "+ + "block it reports %+v, so a share rule would have renamed the reversal "+ + "rather than hidden it, and the test needs rebuilding", found) + } +} + +func countKinds(found []Finding) (pages, blocks int) { + for i := range found { + switch found[i].Kind { + case KindRightToLeft: + pages++ + case KindInvented: + blocks++ + } + } + return pages, blocks +} + +type rtlPage struct { + page int + absent, reversible, tokens int + share float64 + sample string +} + +// rightToLeftPages is every page the direction test claims, whatever its evidence, +// which is the population a threshold would be chosen over. +func rightToLeftPages(t *testing.T, in Input, scope []int) []rtlPage { + t.Helper() + g := defaultTextGuards + g.reversible = 0 + var out []rtlPage + for _, f := range checkTextWith(in, scope, g) { + if f.Kind != KindRightToLeft { + continue + } + r := rtlPage{page: f.Page, absent: f.Count, reversible: int(f.Got), + tokens: f.Total, sample: f.Sample} + if r.absent > 0 { + r.share = float64(r.reversible) / float64(r.absent) + } + out = append(out, r) + } + sort.Slice(out, func(a, b int) bool { + if out[a].share != out[b].share { + return out[a].share > out[b].share + } + return out[a].page < out[b].page + }) + return out +} + +// reversibleWords is the words of one page that are absent from `pdftotext` and +// present in it reversed, with the block they sit in. Exactly what [checkTextWith] +// counts, printed so a reader can judge whether it is a reversal or a coincidence. +func reversibleWords(in Input, page int) []string { + var have map[string]bool + for i := range in.Text { + if in.Text[i].No == page { + have = tokenSet(in.Text[i].Text) + } + } + var out []string + seen := make(map[string]bool) + for i := range in.Blocks { + if in.Blocks[i].Page != page { + continue + } + for _, t := range tokens(in.Blocks[i].Text) { + if !have[t] && have[reverse(t)] && !seen[t] { + seen[t] = true + out = append(out, t+" for "+reverse(t)+", in ["+excerpt(in.Blocks[i].Text)+"]") + } + } + } + return out +} + +// sequentialInput converts the 560-page fixture for every language and reads it a +// second time with `pdftotext`, which is what the text checks compare. +func sequentialInput(t *testing.T) Input { + t.Helper() + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixture and run this", fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFInfo, extern.PDFToText, extern.PDFToHTML} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + m, err := fixture.Load("../../testdata/fixtures", "dreame-l40-ultra") + if err != nil { + t.Fatalf("load manifest: %v", err) + } + ctx := context.Background() + path, err := m.Fetch(ctx) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + res, err := doc.Analyze(ctx, path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + conv, err := ConvertAll(ctx, path, res) + if err != nil { + t.Fatalf("ConvertAll: %v", err) + } + text, err := doc.ExtractText(ctx, path, conv.Scope.TotalPages) + if err != nil { + t.Fatalf("ExtractText: %v", err) + } + return Input{Blocks: conv.Blocks, Text: text, Pages: conv.Pages} +} diff --git a/internal/verify/verify.go b/internal/verify/verify.go new file mode 100644 index 0000000..481fd66 --- /dev/null +++ b/internal/verify/verify.go @@ -0,0 +1,448 @@ +// Package verify checks a conversion against a second, independent extraction of +// the same bytes, and reports what it finds as data rather than as log lines. +// +// # Why this can be free +// +// docs/design/conversion.md records five defects, and every one of them is +// arithmetic rather than judgement. The reason arithmetic is enough is that the +// document has already been extracted twice by different code: every block, +// column and region in this project comes from `pdftohtml -xml` through +// [doc.ExtractRuns], while [doc.ExtractText] reads the same file with +// `pdftotext`. So for every page there is a second opinion that cost nothing to +// obtain and that shares no code with the first. Comparing them is a diff. +// +// That is the whole design. No model is called, nothing is sampled, and the +// checks run in CI so a regression cannot come back. A later tier can spend +// tokens on the pages this one flags. +// +// # What it does not claim +// +// A finding is evidence, not a verdict. Two of the five checks fire on defects +// this project has deliberately accepted — a hyphen followed by a space is +// recorded in conversion.md as the smaller error, and right-to-left text was a +// known extraction defect with its own named finding so that fixing it later +// would turn off one [KindRightToLeft] rather than thousands of [KindInvented]. +// It was fixed, that is exactly what happened, and the finding then had to be +// sharpened before it would go quiet on the pages that were no longer wrong: see +// [minReversibleWords], which is the clearest example here of a check outliving +// the shape of the defect it was written for. A report with no findings would mean +// the checks are broken, not that the conversion is perfect. +package verify + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" +) + +// Kind is what a finding is. A string for the same reason [doc.BlockKind] is one: +// it reaches a report a person reads and a test that asserts on it, where +// "coverage" survives a reordering of this list and 0 does not. +type Kind string + +const ( + // KindCoverage says a page's blocks hold materially less text than + // `pdftotext` found on the same page, so content was dropped. + KindCoverage Kind = "coverage" + // KindInvented says a converted block holds words that do not appear anywhere + // in `pdftotext`'s text for that page. Characters in the wrong order produce + // this and not [KindCoverage], which is why both checks exist: interleaved + // columns preserve every character and destroy every word. + KindInvented Kind = "invented-text" + // KindRightToLeft says a page reads right to left AND still holds text this + // pipeline read backwards: words absent from `pdftotext` that are present in it + // reversed. Named apart from [KindInvented] because the cause is known and + // recorded — see [minReversibleWords] and conversion.md — and reported once per + // page rather than once per word, so a Hebrew section costs the report a line + // instead of a thousand. Being right to left is not enough on its own: that made + // this fire on pages that were correct. + // + // It reports nothing on either manual now, and that is the one finding here of + // which a zero is the goal rather than a suspicion — the defect it names was + // fixed in doc/bidi.go. verify.TestNoTextIsStoredReversed is what holds it there. + KindRightToLeft Kind = "right-to-left-reversed" + // KindJoinHyphen, KindJoinGlued and KindJoinSpace are the three shapes of a + // suspicious join: a hyphen followed by a space mid-word, two words glued with + // no space between them, a doubled space. Three kinds and not one because they + // have three different causes and only one of them is deliberate — see + // [checkJoins]. Reported, never fixed. + KindJoinHyphen Kind = "join-hyphen-space" + KindJoinGlued Kind = "join-glued-words" + KindJoinSpace Kind = "join-double-space" + // KindFigureBand says a figure's box is materially bigger than the shapes + // drawn inside it, so the picture arrives with an empty band around it. + KindFigureBand Kind = "figure-blank-band" + // KindFigureClipped says shapes crossing the figure's box are drawn from + // inside it, so part of the picture is cut off. + KindFigureClipped Kind = "figure-clipped" + // KindReadingOrder says two consecutive blocks of one region switch column + // without going back up the page, which is what interleaving looks like from + // the outside. + KindReadingOrder Kind = "reading-order" +) + +// AllKinds is every kind, in report order — coverage first because it is the +// question a reader asks first, the deliberate defects last. +var AllKinds = []Kind{KindCoverage, KindInvented, KindRightToLeft, + KindJoinGlued, KindJoinHyphen, KindJoinSpace, + KindFigureBand, KindFigureClipped, KindReadingOrder} + +// Finding is one thing that is wrong, with the numbers behind it. +// +// Every field is a number or a short string, and nothing here is a formatted +// sentence except Detail: a test asserts on Kind, Page and the counts, and a +// person reads Detail. That split is deliberate — a check whose only output is +// prose cannot be regression-tested. +type Finding struct { + // Kind is what is wrong. + Kind Kind + // Page is the 1-based PDF page, 0 for a finding about the document. + Page int + // RegionX0 and Index locate the block, matching [doc.Block]'s natural key. + // Both are 0 for a finding about a page rather than a block. + RegionX0 float64 + Index int + + // Got and Want are the measurement and the bound it failed, in the check's own + // units: runes for [KindCoverage], units of the 1.5-scaled page space for the + // figure checks. Both 0 for a check that counts rather than measures. + Got, Want float64 + // Count is how many things are wrong, and Total how many were examined — + // tokens for [KindInvented], shapes for [KindFigureClipped]. + Count, Total int + // Sample is a short excerpt of the offending text, at most [sampleRunes] + // runes. Present so a person can find the page; never the whole block, because + // a report is not a copy of the manual. + Sample string + // Detail is the finding in one sentence, numbers included. + Detail string +} + +// sampleRunes bounds an excerpt. Long enough to recognise a line on the page, +// short enough that a report naming 400 findings is still a page of text. +const sampleRunes = 60 + +// Report is everything one pass found, plus what it measured on the way. +type Report struct { + // Findings are the problems, sorted by page then kind. + Findings []Finding + // Coverage is the per-page measurement behind [KindCoverage], kept for every + // page examined and not only for the ones that failed. This is what a + // threshold is chosen against, so a report that dropped it would make the + // next threshold a guess. + Coverage []PageCoverage + // Pages is how many pages were examined, Figures how many figures. + Pages, Figures int + // Notes say what could not be checked, in the caller's terms — a missing + // optional tool costs a check and does not fail a document, the stance + // [doc.Convert] takes one level down. + Notes []string +} + +// PageCoverage is one page's text accounting. +type PageCoverage struct { + // Page is the 1-based PDF page. + Page int + // Blocks is the non-space rune count summed over the page's converted blocks, + // Text the same count from `pdftotext`. + Blocks, Text int + // Ratio is Blocks over Text, 0 when the page has no `pdftotext` text. + Ratio float64 +} + +// Input is everything a check needs, already gathered. +// +// Taking it as data rather than as a path is what makes every check testable +// without poppler and without a fixture: a hermetic test hands it three blocks +// and one page of text. [Check] is the only thing here that spawns a process, and +// all it does is fill this in. +type Input struct { + // Blocks are the converted blocks under test. + Blocks []doc.Block + // Figures are the converted figures under test. + Figures []doc.ConvertedFigure + // Text is `pdftotext`'s reading of the same document, by page. The second + // opinion; without it the text checks are skipped and said to be skipped. + Text []doc.Page + // Ink is every shape each page draws, keyed by page, as [doc.ExtractInk] + // reports it. Needed only for the pages carrying figures. + Ink map[int][]doc.Ink + // Pages bounds which pages are examined at all. Empty means every page a + // block or a figure appears on. This is what keeps a conversion of 22 pages + // from being judged against a 560-page document's other 538 blank ones. + Pages []int +} + +// Check gathers the second opinion and runs every check. +// +// It calls `pdftotext` once over the whole document and `pdftocairo` once per +// page that carries a figure — the same costs [doc.Analyze] and [doc.Convert] +// already pay, paid again because neither hands back what it read. Losing either +// tool costs the checks that need it and is written into Notes rather than +// returned as an error. +// +// conv must be a conversion of EVERY language the document holds, not one +// household's. Coverage compares a page's blocks against all the text on that +// page, and a page of the column manual holds five languages, so judging one +// language's conversion against it would report a correct conversion as having +// dropped four fifths of the page. See [ConvertAll]. +func Check(ctx context.Context, path string, conv *doc.Conversion) (*Report, error) { + if conv == nil { + return nil, errors.New("verify: Check needs a conversion") + } + in := Input{Blocks: conv.Blocks, Figures: conv.Figures, Pages: conv.Pages} + rep := &Report{} + + pageCount := conv.Scope.TotalPages + if pageCount <= 0 { + pageCount = maxPage(in) + } + text, err := doc.ExtractText(ctx, path, pageCount) + switch { + case err == nil: + in.Text = text + case errors.Is(err, extern.ErrNotFound): + rep.note("no text comparison: " + err.Error()) + default: + return nil, fmt.Errorf("verify: %w", err) + } + + withFigures := make(map[int]bool, len(in.Figures)) + for i := range in.Figures { + withFigures[in.Figures[i].Page] = true + } + if len(withFigures) > 0 { + in.Ink = make(map[int][]doc.Ink, len(withFigures)) + for _, p := range sortedPages(withFigures) { + ink, err := doc.ExtractInk(ctx, path, p) + switch { + case err == nil: + in.Ink[p] = ink + case errors.Is(err, extern.ErrNotFound): + rep.note("no figure geometry: " + err.Error()) + in.Ink = nil + default: + return nil, fmt.Errorf("verify: %w", err) + } + if in.Ink == nil { + break + } + } + } + + got := Inspect(in) + got.Notes = append(rep.Notes, got.Notes...) + return got, nil +} + +// ConvertAll converts a document for every language it holds, which is the input +// coverage has to be measured against. +// +// One [doc.Convert] call with every language as the household, not one call per +// language: the union is what is wanted and one call produces it, where 34 calls +// would re-read the document 34 times. Measured on the sequential manual, a +// per-language loop is about eight minutes against 25 s for this. +func ConvertAll(ctx context.Context, path string, res *doc.Result) (*doc.Conversion, error) { + if res == nil { + return nil, errors.New("verify: ConvertAll needs the probe's result") + } + summaries := res.Languages() + langs := make([]string, 0, len(summaries)) + for i := range summaries { + if summaries[i].Lang != "" { + langs = append(langs, summaries[i].Lang) + } + } + // Every language, and deliberately NOT the pages no language owns. + // + // Those are an opt-in scope a household chooses per document, so a checker that + // took them unconditionally would be measuring a conversion nobody asked for, and + // every count pinned in docs/design/conversion.md would move for a reason that is + // not a change in the pipeline. The consequence is stated rather than hidden: the + // checks do not cover the neutral pages' figures, so the sequential manual's + // plates on pages 5 and 6 are converted-but-unchecked until this takes an option. + return doc.Convert(ctx, path, res, langs, doc.ConvertOptions{}) +} + +// Inspect runs every check over gathered input. Pure: it spawns nothing, reads no +// file, and is the entry point every hermetic test uses. +func Inspect(in Input) *Report { + rep := &Report{} + scope := pageScope(in) + rep.Pages = len(scope) + rep.Figures = len(in.Figures) + + if len(in.Text) == 0 { + rep.note("coverage and word checks were skipped: no pdftotext reading was supplied") + } else { + cov, findings := checkCoverage(in, scope) + rep.Coverage = cov + rep.Findings = append(rep.Findings, findings...) + rep.Findings = append(rep.Findings, checkText(in, scope)...) + } + rep.Findings = append(rep.Findings, checkJoins(in)...) + if len(in.Figures) > 0 { + // The two figure faults need different evidence, so they are skipped + // separately: a blank band is read off the rendered bytes, a clipped picture + // off the page's shapes. + if len(in.Ink) == 0 { + rep.note("clipped figures were not checked: no page ink was supplied") + } + if !anyRendered(in.Figures) { + rep.note("blank bands were not checked: the figures carry no rendered bytes") + } + } + rep.Findings = append(rep.Findings, checkFigures(in)...) + rep.Findings = append(rep.Findings, checkOrder(in.Blocks)...) + + sort.SliceStable(rep.Findings, func(a, b int) bool { + if rep.Findings[a].Page != rep.Findings[b].Page { + return rep.Findings[a].Page < rep.Findings[b].Page + } + if rep.Findings[a].Kind != rep.Findings[b].Kind { + return rep.Findings[a].Kind < rep.Findings[b].Kind + } + return rep.Findings[a].Index < rep.Findings[b].Index + }) + return rep +} + +// Count is how many findings of one kind there are. +func (r *Report) Count(k Kind) int { + n := 0 + for i := range r.Findings { + if r.Findings[i].Kind == k { + n++ + } + } + return n +} + +// Kinds is the count of every kind present, for a report a person skims. +func (r *Report) Kinds() map[Kind]int { + out := make(map[Kind]int, 4) + for i := range r.Findings { + out[r.Findings[i].Kind]++ + } + return out +} + +// PagesFlagged is how many distinct pages carry a finding of one kind. +func (r *Report) PagesFlagged(k Kind) int { + seen := make(map[int]bool) + for i := range r.Findings { + if r.Findings[i].Kind == k { + seen[r.Findings[i].Page] = true + } + } + return len(seen) +} + +// MedianCoverage is the middle page's block-to-text ratio, over the pages that +// carry text. The median and not the mean, for the reason +// [doc.Result.MedianChars] gives: one page of front matter with two words on it +// would otherwise move the number more than a whole section. +func (r *Report) MedianCoverage() float64 { + ratios := make([]float64, 0, len(r.Coverage)) + for i := range r.Coverage { + if r.Coverage[i].Text > 0 { + ratios = append(ratios, r.Coverage[i].Ratio) + } + } + if len(ratios) == 0 { + return 0 + } + sort.Float64s(ratios) + return ratios[len(ratios)/2] +} + +// Summary describes a report in one line. Counts only — no text and no filename — +// so it is safe in a log line, the stance [doc.Conversion.Summary] takes. +func (r *Report) Summary() string { + kinds := r.Kinds() + parts := make([]string, 0, len(kinds)) + for _, k := range AllKinds { + if n := kinds[k]; n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", n, k)) + } + } + s := fmt.Sprintf("%d finding(s) over %d page(s) and %d figure(s)", + len(r.Findings), r.Pages, r.Figures) + if len(parts) > 0 { + s += ": " + strings.Join(parts, ", ") + } + if len(r.Coverage) > 0 { + s += fmt.Sprintf("; median coverage %.2f", r.MedianCoverage()) + } + if len(r.Notes) > 0 { + s += fmt.Sprintf("; %d note(s)", len(r.Notes)) + } + return s +} + +func (r *Report) note(s string) { r.Notes = append(r.Notes, s) } + +// anyRendered reports whether any figure carries its PNG. A conversion read back +// out of the database does not — the bytes live in the blob store — and the band +// check has to say so rather than reporting every figure as clean. +func anyRendered(figs []doc.ConvertedFigure) bool { + for i := range figs { + if len(figs[i].PNG) > 0 { + return true + } + } + return false +} + +// pageScope is the pages to examine, ascending. +func pageScope(in Input) []int { + if len(in.Pages) > 0 { + seen := make(map[int]bool, len(in.Pages)) + for _, p := range in.Pages { + seen[p] = true + } + return sortedPages(seen) + } + seen := make(map[int]bool) + for i := range in.Blocks { + seen[in.Blocks[i].Page] = true + } + for i := range in.Figures { + seen[in.Figures[i].Page] = true + } + return sortedPages(seen) +} + +func maxPage(in Input) int { + last := 0 + for i := range in.Blocks { + if in.Blocks[i].Page > last { + last = in.Blocks[i].Page + } + } + return last +} + +func sortedPages(m map[int]bool) []int { + out := make([]int, 0, len(m)) + for p := range m { + out = append(out, p) + } + sort.Ints(out) + return out +} + +// excerpt trims text to something a report can print. +func excerpt(s string) string { + s = strings.Join(strings.Fields(s), " ") + r := []rune(s) + if len(r) <= sampleRunes { + return s + } + return string(r[:sampleRunes]) + "…" +} diff --git a/internal/verify/verify_fixture_test.go b/internal/verify/verify_fixture_test.go new file mode 100644 index 0000000..bbcff5e --- /dev/null +++ b/internal/verify/verify_fixture_test.go @@ -0,0 +1,580 @@ +package verify_test + +import ( + "context" + "os" + "sort" + "testing" + "time" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/extern" + "github.com/gordon2/manualbox/internal/fixture" + "github.com/gordon2/manualbox/internal/verify" +) + +// These run every check over the two real manuals, and their log IS the report: +// the counts per kind, the coverage distribution, and a sample of each finding. +// The assertions pin what was measured while the thresholds were chosen, so that a +// change in `doc` that alters any of it fails here with the number that moved. +// +// The documents are fetched on demand and are not committed. Without +// MANUALBOX_TEST_FIXTURES=1 these skip, so the default suite stays hermetic. + +const fixturesDir = "../../testdata/fixtures" + +// checked converts a fixture for every language it holds and verifies it. Every +// language, not one household's, for the reason [verify.Check] gives: a page of +// the column manual holds five languages and coverage is measured against all the +// text on the page. +func checked(t *testing.T, name string) (*doc.Conversion, *verify.Report) { + t.Helper() + if os.Getenv(fixture.EnableEnv) == "" { + t.Skipf("set %s=1 to download the fixtures and run the real-document tests", + fixture.EnableEnv) + } + for _, tool := range []extern.Tool{extern.PDFInfo, extern.PDFToText, extern.PDFToHTML, + extern.PDFToCairo, extern.PDFToPPM} { + if !extern.Available(tool) { + t.Skipf("%s is not installed", tool.Name) + } + } + m, err := fixture.Load(fixturesDir, name) + if err != nil { + t.Fatalf("load manifest: %v", err) + } + path, err := m.Fetch(context.Background()) + if err != nil { + t.Fatalf("fetch fixture: %v", err) + } + + ctx := context.Background() + res, err := doc.Analyze(ctx, path) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + start := time.Now() + conv, err := verify.ConvertAll(ctx, path, res) + if err != nil { + t.Fatalf("ConvertAll: %v", err) + } + t.Logf("%v converting every language: %s", time.Since(start).Round(time.Millisecond), + conv.Summary()) + + start = time.Now() + rep, err := verify.Check(ctx, path, conv) + if err != nil { + t.Fatalf("Check: %v", err) + } + t.Logf("%v checking: %s", time.Since(start).Round(time.Millisecond), rep.Summary()) + report(t, rep) + return conv, rep +} + +// report logs everything one pass found, which is the point of these tests. +func report(t *testing.T, rep *verify.Report) { + t.Helper() + for _, n := range rep.Notes { + t.Logf(" note: %s", n) + } + kinds := rep.Kinds() + for _, k := range verify.AllKinds { + if kinds[k] > 0 { + t.Logf(" %-24s %4d finding(s) over %d page(s)", k, kinds[k], rep.PagesFlagged(k)) + } + } + + cov := make([]verify.PageCoverage, len(rep.Coverage)) + copy(cov, rep.Coverage) + sort.Slice(cov, func(a, b int) bool { return cov[a].Ratio < cov[b].Ratio }) + t.Logf(" median coverage %.3f", rep.MedianCoverage()) + for i := 0; i < len(cov) && i < 6; i++ { + t.Logf(" least covered: page %d at %.3f (%d block characters against %d)", + cov[i].Page, cov[i].Ratio, cov[i].Blocks, cov[i].Text) + } + + // Up to three examples of each kind, so the log is readable on a document that + // reports a thousand findings. + shown := make(map[verify.Kind]int, len(kinds)) + for i := range rep.Findings { + f := &rep.Findings[i] + if shown[f.Kind] >= 3 { + continue + } + shown[f.Kind]++ + t.Logf(" %s", f.Detail) + if f.Sample != "" { + t.Logf(" %s", f.Sample) + } + } +} + +// figurePages is how many distinct pages carry a figure. Counted rather than +// derived from the figure count because the two move independently: a geometry +// change that splits one drawing into two raises the figures and not the pages, +// while one that admits page furniture raises both. +func figurePages(conv *doc.Conversion) int { + seen := make(map[int]bool, len(conv.Figures)) + for i := range conv.Figures { + seen[conv.Figures[i].Page] = true + } + return len(seen) +} + +// TestCheckTheColumnManual is the parallel-columns fixture: 68 pages, five +// languages sharing most of them, 59 figures. +func TestCheckTheColumnManual(t *testing.T) { + conv, rep := checked(t, "thomas-drybox-amfibia") + + // 2,336 blocks, of which 111 are page furniture — the three language tabs this + // manual prints in its columns, and 41 folios. It was 2,180 before doc's + // furniture pass existed, and the rise of 76 to 2,256 was not text appearing: 35 + // content blocks lost a tab that was glued to them and 111 furniture blocks took + // its place. Counted apart because a change to the furniture rule must move the + // second number and not the first. + // + // 2,336 since the contents pages came apart. This document prints its table of + // contents once per language, 17 entries each, and each was one run-together + // block of dot leaders: +16 per language over five languages is exactly the 80. + // Coverage did not move — the dots are still in the text, only grouped + // differently — and neither did the figures. + // + // 2,345 since the running-head clause, and the +9 is one page rather than a + // spread: furniture went 111 -> 172 without the total rising by 61, because 61 + // of the 61 were already blocks of their own or were the whole of one. The nine + // are all on page 44, where taking the head off the Polish column changed the + // line pitch that page's paragraph rule measures, and two run-together blocks + // resolved into ten discrete printed instructions. That is the same second-order + // effect the tab had when it lifted the column manual's level-1 headings by 29. + // + // 2,407 since reading order got its own strips for the pages the column detector + // declines to call two-column. +62 over seven pages and no others, every one of + // them a page that was welding two columns onto one line: + // + // page 11 +27 the parts list. Its 39 numbered items were arriving as two + // run-together blocks of 7 and 19 lines, each with the diagram's + // callout numbers spliced into the middle of the words — + // `"17 Staubbehälter für Grobschmutz und Feinstaub 7 18 Saugschlauch*"`. + // They are now 39 list items and the callouts are their own blocks. + // page 12 +25 the same page in the other four languages' overview + // pages 57-61 + // +10 two per troubleshooting page. Each prints two side-by-side + // tables whose header row sits above a top border the document + // does not draw, so the headers read as prose, and the two tables' + // headers were one block: `"Aufgetretene Störungen/ Grund / Abhilfe + // Aufgetretene Störungen/ Grund / Abhilfe Fehlfunktionen + // Fehlfunktionen"`. Each is now its own table's header. + // + // No word is gained or lost on any page of the document — checked as a multiset + // per page — so this is grouping and order, not text. + if len(conv.Blocks) != 2407 || len(conv.Figures) != 59 { + t.Errorf("the conversion under test moved: %d blocks and %d figures, "+ + "was 2407 and 59 (2345 before the columns of a one-column page came apart, "+ + "2336 before the running-head clause)", len(conv.Blocks), len(conv.Figures)) + } + if got := len(conv.FurnitureBlocks()); got != 172 { + t.Errorf("%d furniture block(s), was 172 (111 before the running-head clause)", got) + } + + // No page loses text. The lowest score is page 5 at 0.80, which is a page of + // framed illustrations whose captions the run filter drops, and the median is + // 0.97 — so the floor of 0.75 leaves headroom and reports nothing here. + // + // Coverage now excludes the furniture it counted before, and on this manual that + // costs almost nothing: the median moved from 0.974 to 0.973 and the floor stayed + // at 0.801, because a tab and a folio are four characters against a page of three + // thousand. checkCoverage records why it is excluded anyway. + // + // A running head is not four characters, so the median fell again, 0.973 -> 0.965. + // That fall is the mechanism working and not a page losing text: the head is still + // in `pdftotext`'s reading, so it stays in the denominator while leaving the + // numerator. NO page is reported, which is the assertion that would catch a rule + // claiming a paragraph, and the floor is 8.5 points above 0.75 either way. + if got := rep.Count(verify.KindCoverage); got != 0 { + t.Errorf("coverage reported %d page(s) on a manual that drops none", got) + } + if m := rep.MedianCoverage(); m < 0.95 || m > 1.0 { + t.Errorf("median coverage %.3f, was 0.965 (0.973 before the running-head clause, "+ + "0.974 before furniture was excluded)", m) + } + + // Four blocks hold words the page never printed, and all four are table cells + // where the two tools disagree about where a Cyrillic or Kazakh word divides. + if got := rep.Count(verify.KindInvented); got != 4 { + t.Errorf("invented text: %d finding(s), was 4", got) + } + // Nothing here reads right to left. + if got := rep.Count(verify.KindRightToLeft); got != 0 { + t.Errorf("right-to-left: %d finding(s) on a manual with no such script", got) + } + + // Hyphenation is deliberate in doc, and this is its cost: 276 blocks carrying + // 313 hyphens followed by a space. Reported, not fixed. + if got := rep.Count(verify.KindJoinHyphen); got != 276 { + t.Errorf("hyphen joins: %d block(s), was 276", got) + } + if got := rep.Count(verify.KindJoinGlued) + rep.Count(verify.KindJoinSpace); got != 0 { + t.Errorf("glued words or doubled spaces: %d, was 0", got) + } + + // The figure geometry, in two steps. Both of these were the clip-path limitation + // conversion.md recorded; reading the clip took them from 4 blank bands and 22 + // clipped to 0 and 15 while the figure count rose from 46 to 59, and narrowing + // trimToPicture to lines the box had reached over took the 15 to 3. + // + // Zero is asserted on the band because that check reads the RENDERED PIXELS and + // so is independent of the geometry that produced them: it is the one number + // here that cannot improve by the box and the ink agreeing with each other. + // ONE, AND IT IS THE BAND'S DOING. Page 1's cover figure is the columns manual's + // only crop that widens: its single claim is the book title, printed well clear of + // the art, so the band spans the gap and 23 units of paper come with it against a + // 12-unit allowance. It is the honest reading of a crop that takes in what the page + // set beside the drawing, and the check is doing its job by saying so — this reads + // the RENDERED PIXELS, so it is the one number here that cannot improve by the box + // and the ink agreeing with each other. 0 before the band, 4 before the clip. + if got := rep.Count(verify.KindFigureBand); got != 1 { + t.Errorf("blank bands: %d figure(s), was 4 before the clip, 0 after, and 1 once "+ + "the crop became the band the page prints", got) + } + // The 3 that remain are three different things and none is the trim cutting a + // drawing away from its own label. Pages 11 and 12 report one and two shapes + // crossing out of 2,741 — a page-sized path this package's geometric matching + // cannot attribute, and the same 2 that stand with trimming switched off + // entirely. Page 1 is the cover, whose art really does continue behind the + // title block the trim excludes; that one is a genuine trade and it is taken + // deliberately, because the alternative is a cover crop full of headline type. + if got := rep.Count(verify.KindFigureClipped); got != 2 { + t.Errorf("clipped figures: %d of 59, was 22 of 46 before the clip, 15 while the "+ + "trim cut labels off, and 3 before the crop became a band", got) + } + // The pages carrying figures, which is what says a change to the geometry split + // or merged pictures rather than admitting or losing them. 27 since the clip was + // read, and the trim change did not move it. + if got := figurePages(conv); got != 27 { + t.Errorf("figures land on %d page(s), was 27", got) + } + + // Reading order is clean, including on the parts pages whose callouts scatter + // across the measure and on the ten table pages. + // + // One finding, and it is the check's shape rather than a defect, which is why it + // is pinned with its explanation instead of being tuned away. Page 58 prints its + // right-hand troubleshooting table's header row — `Usterki / Wadliwe działanie` + // and `Przyczyna / Środki zaradcze` — above a top border the document does not + // draw, so the two cells are prose rather than [doc.BlockTable] and are read left + // to right, level, which is exactly what interleaving looks like. It is the same + // row-major reading [checkOrder] excludes table cells for; these two are simply + // not inside the table. Before the columns of a one-column page came apart they + // were ONE block, welded across the gutter, and the check could not see them at + // all: zero here used to be worth less than one is now. + if got := rep.Count(verify.KindReadingOrder); got != 1 { + t.Errorf("reading order reported %d finding(s) on a manual read correctly, was 1", got) + } +} + +// TestCheckTheSequentialManual is the 560-page, 34-language fixture, and it is +// where the checks find defects nothing had recorded: the Thai section's words +// arrive broken, and its Hebrew and Arabic used to arrive backwards. That second one +// is fixed and this test is where it stays fixed. +func TestCheckTheSequentialManual(t *testing.T) { + conv, rep := checked(t, "dreame-l40-ultra") + + // 16,055 blocks, of which 1,105 are page furniture: the 34 language tabs, one on + // every page of every section, and 552 folios. + // + // This number has been up and back down, for reasons that are all in doc/bidi.go + // after the first two. Every move is on a Hebrew or Arabic page, measured page by + // page against each previous conversion; the furniture count and the figures have + // never moved. + // + // 15,951 before the furniture pass + // 16,055 after it, the tab un-glued from a running head on 104 pages + // 16,097 right-to-left lines read in logical order: +42 over ten pages, because + // a list marker leads its line only in logical order, so + // `– يجب إزالة البطارية` became the list item it is printed as + // 16,098 +1 on page 191, when the region's language took over deciding + // direction and gave that page's `Dreamehome אפלקציית` line to the repair + // 16,055 −43 over six pages: a REGRESSION, and the only thing that caught it + // 16,098 the same 43 back, page for page identical to the reading above it + // + // THE SEQUENCE IS THE POINT, because the number alone lies twice. 16,055 appears + // twice and means opposite things: once as the honest total before any right-to-left + // repair, and once as a regression that had cost six pages their list structure — + // page 194 was 7 blocks BELOW its original at that point, so even the distribution + // was not the same document. Anyone moving this number should read the sequence + // before deciding which direction is good. Higher has meant better every time so + // far, because every rise has been a printed list becoming list blocks. + // + // The regression and its repair are both in bidi.go's run-level island test, and + // conversion.md carries them. `hasRightToLeft` asks whether a run holds a + // right-to-left LETTER, so a run of only digits, punctuation or spaces answered no + // and was held in printed order as though it were left-to-right content: page 211's + // marker is two such runs, the digit at x=859 and the period at x=850, and it came + // out `. 1 مستشعر المسافة بالليزر ( )LDS` where the page prints + // `1. مستشعر المسافة بالليزر (LDS)`. An island is now the part BETWEEN the outermost + // runs carrying a left-to-right letter, so those two reverse with the Arabic beside + // them, `leadingMarker` sees `1.` again, and the six list items are six blocks. + // 16,097 −1 on one page, when column detection stopped depending on which runs + // the furniture pass had taken out and started reading the page as + // printed. That change is in readingGroups and it is what lets the line + // below cost nothing: with it, the running-head clause moves no block + // total and no finding count at all. + // 16,132 +35 over 28 pages, when reading order got its own strips for the pages + // the column detector declines to call two-column. This one moves BOTH + // ways and both directions are the same repair. 25 pages gain, the + // two-column maintenance and disposal pages of one language section + // after another, where a banner and a step were welded across the gutter + // — page 530 read `"Мешок для сбора пыли Основная щетка"`, two section + // titles spliced. 3 pages LOSE blocks and that is the same fix seen from + // the other side: page 216's Arabic warning was arriving as five + // fragments because each of its lines was cut where the left column's + // step began, and it is now one seven-line paragraph (−7). Page 53's + // French spec page is the same shape (−4), page 537's Russian one (−1). + // No word is gained or lost on any page of the document, checked as a + // multiset per page. + // + // The furniture count moved 1,105 -> 1,289 on the same commit, and the total did + // not, because a running head was already a block of its own on every page it was + // claimed from. That is the shape to expect: a clause 3 that moved the total would + // be splitting or merging content, which is not what it is for. + // 16,132 -> 16,201 when a figure's callout labels left the block flow, and the + // total went UP by 69 while content went DOWN. Both halves are the same fact: 157 + // runs became Callout blocks, and 66 of the 89 on the Russian pages had been + // arriving INSIDE a bigger paragraph rather than as blocks of their own, so + // splitting them out adds blocks. Nothing left conv.Blocks — that is what keeps + // coverage flat at a median of 0.996 — so this total is still every character the + // pages print. See doc.Callouts. + // + // 16,201 -> 16,203 when a wrapped label stopped losing its later lines. 8 runs across + // four languages joined their own label — page 521's `держателя насадки для швабры`, + // two on 522, one Japanese one on 542 — and 6 of the 8 were already inside a bigger + // paragraph, so only 2 new blocks come of it. That ratio is the 66-of-89 measurement + // above holding at a much smaller sample, and it is the shape to expect: a run leaving + // the flow moves the total by 1 only when it was a block of its own. See + // doc.claimLabels. + // + // THE BLOCK TOTAL DID NOT MOVE WHEN THE CROP BECAME THE BAND, and that is the check + // on this whole change rather than a footnote: 16,203 before and after, over all 34 + // languages, with the furniture and callout splits identical too. The crop is a + // picture and the blocks are text, and nothing was supposed to cross between them. + // The FIGURE count did move, 134 -> 128, which is doc.ServedFigures dropping six + // crops that lay wholly inside another once the band widened them; the 166 labels + // they carry are unchanged, because an absorbed crop's labels go to the one that + // swallowed it. + if len(conv.Blocks) != 16203 || len(conv.Figures) != 128 { + t.Errorf("the conversion under test moved: %d blocks and %d figures, "+ + "was 16203 and 128 (134 before the crop became a band, 16201 before a "+ + "wrapped label kept its tail, 16132 before "+ + "the callout labels left the flow) — read the sequence above before deciding "+ + "which way is better, because 16055 has been both the honest total and a "+ + "regression that cost six pages their lists", + len(conv.Blocks), len(conv.Figures)) + } + if got := len(conv.FurnitureBlocks()); got != 1289 { + t.Errorf("%d furniture block(s), was 1289 (1105 before the running-head clause)", got) + } + + // Excluding the furniture moved the median from 1.000 to 0.997 and the worst + // judged page from 0.952 to 0.949, against a floor of 0.75. The page that moves + // furthest is page 558, which holds nothing but a tab and a folio and so scores + // 0.500 — it is under minCoverageText and is not judged, which is the page that + // constant was written for. + // + // 0.997 -> 0.996 with the running-head clause, and that fall is the mechanism + // rather than a loss: the head is still in `pdftotext`'s reading, so it stays in + // the denominator while leaving the numerator. Nothing is reported, which is the + // assertion that would catch a rule claiming a paragraph instead of a head. + if got := rep.Count(verify.KindCoverage); got != 0 { + t.Errorf("coverage reported %d page(s); its worst judged page scores 0.949", got) + } + if m := rep.MedianCoverage(); m < 0.99 { + t.Errorf("median coverage %.3f, was 0.997 (1.000 before furniture was excluded)", m) + } + + // The right-to-left defect is GONE, and this is where that is asserted as a + // number. Three measurements over this whole document: + // + // before majority region + // pages reported right-to-left-reversed 32 6 0 + // words absent from pdftotext on them 8,120 80 — + // ...of those, present when reversed 7,938 18 0 + // + // The last row is the one that means it: a word absent from the reference but + // present in it backwards is the signature of visual order, and there are none. + // The "before" column was re-measured rather than quoted, by putting joinRuns back + // at doc/blocks.go's one call site; the middle column is what a line deciding its + // own direction by majority left behind, six lines whose Latin outweighed their + // Hebrew. + // + // Zero here is not zero absent words: 202 remain over 25 right-to-left pages, and + // they are Arabic shaping and combining-mark disagreement, reported block by block + // as [verify.KindInvented]. The whole-document assertion and the distribution + // behind this number live in verify.TestNoTextIsStoredReversed. + if got := rep.Count(verify.KindRightToLeft); got != 0 { + for i := range rep.Findings { + if rep.Findings[i].Kind == verify.KindRightToLeft { + t.Errorf("%s | %s", rep.Findings[i].Detail, rep.Findings[i].Sample) + } + } + t.Errorf("right-to-left: %d page(s), want 0 — was 6 while a line's own "+ + "characters decided its direction, and 32 before the lines were read in "+ + "order at all", got) + } + + // A defect nothing had recorded, and this check is how it was found: 142 of these + // blocks are on pages 473-488, the Thai section, where `pdftohtml -xml` + // returns an unmapped glyph for SARA AA (U+FFFD) that `pdftotext` maps correctly + // — so the block's words are broken where that vowel belongs, "ล้�งผ้�ถูพื้น" + // against the printed "ล้างผ้าถูพื้น". 11 more are Latin pages where the two + // tools divide a hyphenated compound differently. + // + // 160 and not 153 because the right-to-left pages are no longer named as pages + // and are judged block by block like every other page, which is the point of the + // sharpening: 7 of their blocks hold more than [maxInventedShare] of words the + // reference does not have, and those 7 are the same Arabic shaping and + // combining-mark disagreements the Latin 11 are, not a reversal. The number did + // not move again when the region took over deciding direction, which is worth + // asserting: those 7 were never the reversal either. + if got := rep.Count(verify.KindInvented); got != 160 { + t.Errorf("invented text: %d block(s), was 160 (153 while every right-to-left "+ + "page was named instead of judged)", got) + } + + // Back to 72. It was 73 for one commit, when page 204's support URL arrived with + // its seventeen runs reversed and this check caught it sideways as + // `إىل faqs-and- manuals-us`. The URL is whole again — the line now reads + // `يُرجى االنتقال إىل https://global.dreametech.com/pages/user-manuals -and-faqs` + // — and that finding is gone with it. + if got := rep.Count(verify.KindJoinHyphen); got != 72 { + t.Errorf("hyphen joins: %d block(s), was 72 (73 while page 204's URL was "+ + "stored run-reversed)", got) + } + // 7. Three arrived with the bidi repair and are not new damage — Hebrew page 200 + // and Arabic 206, 207 print two columns that the conversion interleaves into one + // line, and in visual order the two halves met inside a word the comparison could + // not recognise, so reading the line logically is what makes `סוללות|מדריך` legible + // as a glued pair. + // + // It was 7 for two commits, when page 204's laser standard lost the space in + // `IEC 60825` and transposed the halves of `EN 60825- 1:2014/`. Both came from + // passing a run through visualToLogical after deciding to emit it in PRINTED order: + // such a run is already in logical order, and reversing it splits it at any space + // whose neighbours are not both strongly left-to-right. Only a run that reverses + // gets that repair now, and the standard reads `IEC 60825-1:2014/ EN 60825- 1:2014/`. + // + // 5 since the columns of a one-column page came apart, and the one that left is + // the one this comment names first: `סוללות|מדריך` on Hebrew page 200 was two + // columns' words meeting inside a block, and the two columns are now two blocks. + // The finding was correct and its cause is gone; the remaining 5 are the Arabic + // and Thai shaping pairs, which are a different thing. + if got := rep.Count(verify.KindJoinGlued); got != 5 { + t.Errorf("glued words: %d, was 5 (6 before Hebrew page 200's two columns came "+ + "apart, 7 while page 204's laser standard was repaired twice over, 3 before "+ + "right-to-left lines were read in order)", got) + } + + // 2 blank bands where there were 6 before the clip was read. Merging candidate + // boxes that overlap did not move this, which is worth stating, because a + // merged box is bigger than either of its parts and could easily have arrived + // with empty space in it: it does not, because the parts overlap. + if got := rep.Count(verify.KindFigureBand); got != 2 { + t.Errorf("blank bands: %d figure(s), was 6 before the clip and 2 after", got) + } + // 24, down from 70, and this is where merging overlapping candidates pays off + // twice. The residual findings of this document were never the trim and never + // the clip: they were the crowded diagram pages 521-531, where a drawing had + // clustered in pieces and each piece's box was crossed by the shapes of the + // piece beside it. Merging the pieces removes the crossing along with the + // duplicate picture. What is left is the leader-line case this package has to + // guess at, matching a shape to a figure by geometry because doc.Figure carries + // how many shapes it holds and not which. + // + // 25 until growing a box onto its labels took one more away, and that number is + // worth keeping here because the wrong reading of it was 27. A grown crop reaches + // over whatever sits in the corridor beside it, so asking BOTH questions of the + // rendered box makes a figure adopt the neighbouring drawing's leader and then + // report itself as cut by it. [verify.clipped] matches a shape by the drawn + // extent and tests it against the rendered one, and that is what makes this + // number fall as the crop grows rather than rise. + // + // AND BACK TO 25 NOW THAT THE CROP NO LONGER GROWS, which is the same arithmetic + // read backwards rather than a regression. The crop is the drawing exactly, so a + // leader running past the drawn extent is no longer inside it — page 521 figure 2's + // box is x=579-765 where the grown one was x=459-872. The labels those leaders point + // at are not lost with them: they are carried as text and drawn beside the picture, + // which is what took page 521 from 23 labels held whole to 34. A crop that also + // contained them would print every one twice. + // + // AND 14 ONCE THE CROP BECAME THE BAND THE PAGE PRINTS. The same arithmetic a third + // time: the crop is wider than the drawing again, so a leader running past the drawn + // extent is inside it again — and further than growth ever reached, because the band + // stops at nothing and growth stopped at the neighbour. Read the sequence + // 74 -> 71 -> 70 -> 25 -> 24 -> 25 -> 14 rather than the last value. This is the + // only count in this file that the band moved; blocks, labels, coverage and + // reading-order are identical before and after, measured over all 34 languages. + if got := rep.Count(verify.KindFigureClipped); got != 14 { + t.Errorf("clipped figures: %d of 134, was 74 of 163 before the clip, "+ + "71 while the trim cut labels off, 70 of 168 before overlapping "+ + "candidates were merged, 25 before a crop grew onto its labels and 24 "+ + "while it did", got) + } + // 20, and the 23 that doc/figures.go's header quotes is a different count at a + // different level: doc finds 195 figures over 23 pages, and conversion keeps the + // 134 that fall inside a language region, which land on 20 of those pages. Both + // are right and they are not the same number. The page count did not move when + // the figure count fell from 168 to 134, which is what says those 34 were pieces + // of pictures already found rather than pictures lost. + if got := figurePages(conv); got != 20 { + t.Errorf("figures land on %d page(s), was 20", got) + } + + // The one reading-order class either manual has: the routine-maintenance page + // of each language section lays its intervals out as an unruled grid, which + // conversion.md records as invisible to the table detector, and reading it in + // columns puts the intervals out of order. 36 findings over 34 sections — it was + // 37 until the furniture pass took a tab out of the block that carried it, which + // left that block under minOrderChars. + // + // 38 since the bidi repair, and the 2 are a second real class this document had + // been hiding rather than a regression. Arabic page 216, the battery-disposal + // page, prints two columns; the conversion reads them interleaved, block 3 in the + // right column at x=664-863, block 4 in the LEFT at x=351-587 and further down, + // then block 5 back in the right. That interleave was always there — its Hebrew + // twin on page 200 still has it, invisible, because those two columns are joined + // inside one block. What changed is that a list marker leads its line in logical + // order, so page 216's line came apart into the per-column blocks the check can + // see between. + // + // 24 since reading order got its own strips for the pages the column detector + // declines to call two-column, and this is the number that says the change did + // what it claims. The 14 that left are the second class above, whole: every + // finding on a two-column disposal or product-overview page, including the two on + // Arabic page 216 that this comment says "was always there" and its Hebrew twin on + // page 200 which it says still had it, invisible. Both are read column by column + // now, and the right-to-left ones right column first. + // + // What remains is the first class and nothing else: the routine-maintenance grid of + // one language section after another, still invisible to the table detector and + // still read in columns. Nothing here is new. + // + // 23 since a figure's callout labels stopped being judged for reading order, and + // the one that left was never a defect. doc.RegionBlocks appends the labels after a + // region's content, exactly as it appends the furniture, so they arrive in an order + // that is not a reading order at all — and page 522's two label columns are disjoint + // in x and descending in y, which is this check's violation shape exactly. It was + // reporting interleaving between two labels of one drawing. Page furniture escapes + // the same trap only by position: a tab or a folio sits at the top or the bottom of + // a page, so the block after it is further UP and the check skips it before it can + // fire. That is luck holding rather than a decision. + if got := rep.Count(verify.KindReadingOrder); got != 23 { + t.Errorf("reading order: %d finding(s), was 23 (24 while callout labels were "+ + "judged for reading order, 38 before the columns of a one-column page came "+ + "apart, 36 before right-to-left lines were read in order, 37 before the "+ + "furniture pass)", got) + } + if got := rep.PagesFlagged(verify.KindReadingOrder); got < 16 { + t.Errorf("reading-order findings cover %d pages, was 18 (26 while the "+ + "two-column disposal pages were in this class too) — a class this "+ + "concentrated on one page per section is what makes it explainable", got) + } +} diff --git a/internal/verify/verify_test.go b/internal/verify/verify_test.go new file mode 100644 index 0000000..56963c1 --- /dev/null +++ b/internal/verify/verify_test.go @@ -0,0 +1,535 @@ +package verify_test + +import ( + "bytes" + "image" + "image/color" + "image/png" + "strings" + "testing" + + "github.com/gordon2/manualbox/internal/doc" + "github.com/gordon2/manualbox/internal/verify" +) + +// Every check gets two tests: one proving it fires on a real fault and one +// proving it stays quiet on correct input. A check that cannot be made to fire is +// worse than no check, and one that fires on everything is the same thing with +// more output — so both halves are asserted for all five. +// +// All of it is hermetic. The input is hand-built, so these run in the default +// suite with no fixture, no poppler and no network, which is what lets them guard +// the checks in CI. The fixture-backed tests measure the same code against the two +// real manuals; see verify_fixture_test.go. + +// page builds one page of pdftotext's reading. +func page(no int, text string) doc.Page { + return doc.Page{No: no, Text: text, Chars: len([]rune(text))} +} + +// block builds one converted block. Chars is derived rather than passed, because +// two of the checks read it and a test that set it inconsistently would be +// asserting on a block that cannot exist. +func block(pg, idx int, x0, x1, y0 float64, text string) doc.Block { + return doc.Block{ + Page: pg, Index: idx, Kind: doc.BlockParagraph, Text: text, + X0: x0, X1: x1, Y0: y0, Y1: y0 + 12, + Chars: len([]rune(text)), Lines: 1, + } +} + +func count(t *testing.T, in verify.Input, k verify.Kind) int { + t.Helper() + return verify.Inspect(in).Count(k) +} + +// --- 1. coverage + +const prose = "Der Gehäusedeckel wird abgenommen und der Filter herausgezogen. " + + "Anschließend den Frischwassertank mit klarem Wasser ausspülen." + +func TestCoverageFiresWhenAPageLosesText(t *testing.T) { + // Half of the page's text never became a block, which is what dropping a + // column of a page looks like from outside. + half := prose[:len(prose)/2] + in := verify.Input{ + Blocks: []doc.Block{block(7, 0, 40, 300, 100, half)}, + Text: []doc.Page{page(7, prose)}, + } + rep := verify.Inspect(in) + if got := rep.Count(verify.KindCoverage); got != 1 { + t.Fatalf("want one coverage finding, got %d: %+v", got, rep.Findings) + } + f := rep.Findings[0] + if f.Page != 7 || f.Got >= f.Want || f.Total <= f.Count { + t.Errorf("finding does not carry the numbers behind it: %+v", f) + } + if len(rep.Coverage) != 1 || rep.Coverage[0].Ratio <= 0 { + t.Errorf("the measurement was not kept: %+v", rep.Coverage) + } +} + +func TestCoverageQuietWhenThePageIsWhole(t *testing.T) { + in := verify.Input{ + Blocks: []doc.Block{block(7, 0, 40, 300, 100, prose)}, + Text: []doc.Page{page(7, prose)}, + } + if got := count(t, in, verify.KindCoverage); got != 0 { + t.Fatalf("want no coverage finding, got %d", got) + } +} + +func TestCoverageIgnoresAPageWithAlmostNoText(t *testing.T) { + // A folio and a language badge are a page's whole text on 34 pages of the + // sequential manual, and their ratio means nothing. + in := verify.Input{ + Blocks: []doc.Block{block(7, 0, 40, 60, 800, "18")}, + Text: []doc.Page{page(7, "DE 18")}, + } + if got := count(t, in, verify.KindCoverage); got != 0 { + t.Fatalf("a page of furniture was judged for coverage: %d", got) + } +} + +// TestCoverageDoesNotCountPageFurniture is what makes this check able to refute +// doc's furniture rule, and it is the whole reason the exclusion is deliberate +// rather than an oversight. +// +// The furniture the rule claims really is printed, so `pdftotext` reports it and +// counting it would leave every ratio exactly where it was. Counting it would also +// make a rule that wrongly claims a paragraph invisible here — the paragraph would +// still be in the sum. So a page whose whole text is flagged reads as a page that +// dropped its whole text, which is what a coverage finding is for. +func TestCoverageDoesNotCountPageFurniture(t *testing.T) { + claimed := block(7, 0, 40, 300, 100, prose) + claimed.Furniture = true + in := verify.Input{ + Blocks: []doc.Block{claimed}, + Text: []doc.Page{page(7, prose)}, + } + if got := count(t, in, verify.KindCoverage); got != 1 { + t.Fatalf("a page whose only block is claimed as furniture reported %d coverage "+ + "finding(s); counting furniture would hide a rule that eats a paragraph", got) + } + + // And the same block unflagged is the page being whole, which is the control: + // the finding above is the flag and not the text. + in.Blocks[0].Furniture = false + if got := count(t, in, verify.KindCoverage); got != 0 { + t.Fatalf("the same block unflagged reported %d coverage finding(s)", got) + } +} + +// --- 2. invented text + +func TestInventedTextFiresOnWordsThePageNeverPrinted(t *testing.T) { + in := verify.Input{ + Blocks: []doc.Block{block(62, 3, 43, 443, 400, + "Verpackung schützt Beanspruchung gleichzusetzender")}, + Text: []doc.Page{page(62, "Die Verpackung schützt das Gerät.")}, + } + rep := verify.Inspect(in) + if got := rep.Count(verify.KindInvented); got != 1 { + t.Fatalf("want one invented-text finding, got %d: %+v", got, rep.Findings) + } + f := rep.Findings[0] + if f.Count != 2 || f.Total != 4 { + t.Errorf("want 2 of 4 words absent, got %d of %d", f.Count, f.Total) + } + if !strings.Contains(f.Sample, "beanspruchung") { + t.Errorf("the sample does not name the absent words: %q", f.Sample) + } +} + +func TestInventedTextQuietWhenEveryWordIsPrinted(t *testing.T) { + in := verify.Input{ + Blocks: []doc.Block{block(62, 3, 43, 443, 400, "Die Verpackung schützt das Gerät")}, + Text: []doc.Page{page(62, "Die Verpackung schützt das Gerät.")}, + } + if got := count(t, in, verify.KindInvented); got != 0 { + t.Fatalf("want no invented-text finding, got %d", got) + } +} + +func TestInventedTextToleratesOneOddWordInALongBlock(t *testing.T) { + // The two extractions disagree about a ligature or a soft hyphen from time to + // time, measured at 0.45% of the sequential manual's words, so one absence in + // a long block is not a finding. + in := verify.Input{ + Blocks: []doc.Block{block(62, 3, 43, 443, 400, + "Die Verpackung schützt das Gerät gegen Transportschäden xyzzy")}, + Text: []doc.Page{page(62, "Die Verpackung schützt das Gerät gegen Transportschäden.")}, + } + if got := count(t, in, verify.KindInvented); got != 0 { + t.Fatalf("one absent word in eight was reported: %d", got) + } +} + +// --- 2b. right to left, which is a known defect and must stay one finding + +// rtlEmbed and popDirectional are the bidi controls pdftotext wraps a +// right-to-left line in, written as escapes because they are invisible: a test +// whose input cannot be seen in the source is a test nobody can check. +const ( + rtlEmbed = "\u202b" + popDirectional = "\u202c" +) + +func TestRightToLeftIsOneNamedFindingPerPage(t *testing.T) { + // pdftohtml returns the line in visual order; pdftotext returns it logically, + // wrapped in bidi controls. So every word of the block is absent, and every one + // of them is present reversed — which is the finding's evidence. + printed := rtlEmbed + "הגבלות שימוש על המכשיר" + popDirectional + visual := "שומיש תולבגה רישכמה לע" + in := verify.Input{ + Blocks: []doc.Block{block(185, 0, 55, 800, 95, visual)}, + Text: []doc.Page{page(185, printed)}, + } + rep := verify.Inspect(in) + if got := rep.Count(verify.KindRightToLeft); got != 1 { + t.Fatalf("want one right-to-left finding, got %d: %+v", got, rep.Findings) + } + if got := rep.Count(verify.KindInvented); got != 0 { + t.Errorf("a right-to-left page also reported %d generic invented-text findings, "+ + "which is what naming the defect is meant to prevent", got) + } + f := rep.Findings[0] + if f.Count != 4 || f.Got != 4 { + t.Errorf("want 4 absent words all 4 reversible, got %d absent and %.0f reversible", + f.Count, f.Got) + } + // The sample is the evidence, so it has to be legible as evidence: each word as + // stored, with the spelling the page prints beside it. + if !strings.Contains(f.Sample, "שומיש for שימוש") { + t.Errorf("the sample does not show the reversal it is reporting: %q", f.Sample) + } +} + +// TestRightToLeftNeedsAReversalAndNotJustHebrew is the sharpening the bidi repair +// forced, and it is the half of the check that measured worst: for as long as every +// Hebrew page arrived backwards, "is this page right to left" and "is this page +// reversed" were the same question, and once doc/bidi.go split them the check kept +// answering the first while claiming the second. On the sequential manual that was +// 25 pages reported over 220 absent words in 6,834, three of them on one page of +// 510 — see [verify.minReversibleWords]. +// +// Here the page is Hebrew and correctly ordered, and one word of it disagrees with +// the reference the way the two extractions ordinarily do. Nothing about it is +// backwards, so it is not a right-to-left finding; it is judged block by block like +// any other page. +func TestRightToLeftNeedsAReversalAndNotJustHebrew(t *testing.T) { + printed := rtlEmbed + "הגבלות שימוש על המכשיר בטמפרטורה" + popDirectional + in := verify.Input{ + // "בטמפרטורה" against the printed "בטמפרטורה" — one word the reference + // spells differently, which is what a combining mark or a shaping difference + // looks like. Its reverse is nowhere on the page. + Blocks: []doc.Block{block(185, 0, 55, 800, 95, "הגבלות שימוש על המכשיר בטמפרטורת")}, + Text: []doc.Page{page(185, printed)}, + } + rep := verify.Inspect(in) + if got := rep.Count(verify.KindRightToLeft); got != 0 { + t.Fatalf("a Hebrew page with no reversed word on it was named "+ + "right-to-left-reversed: %+v", rep.Findings) + } + // One absent word in five is under maxInventedShare, so the block check is quiet + // too — which is the point: the page is fine and the report says nothing. + if got := rep.Count(verify.KindInvented); got != 0 { + t.Errorf("invented text reported %d block(s) on one ordinary disagreement", got) + } + + // The same page with a block that really is assembled wrong falls through to the + // block check rather than disappearing, so nothing is hidden by the sharpening. + in.Blocks = []doc.Block{block(185, 0, 55, 800, 95, "אבגד הוזח חטיכ למנס")} + if got := verify.Inspect(in).Count(verify.KindInvented); got != 1 { + t.Errorf("a right-to-left block full of words the page never printed reported "+ + "%d invented-text finding(s), want 1", got) + } +} + +func TestRightToLeftQuietWhenTheOrderIsRight(t *testing.T) { + printed := rtlEmbed + "הגבלות שימוש על המכשיר" + popDirectional + in := verify.Input{ + Blocks: []doc.Block{block(185, 0, 55, 800, 95, "הגבלות שימוש על המכשיר")}, + Text: []doc.Page{page(185, printed)}, + } + rep := verify.Inspect(in) + if got := rep.Count(verify.KindRightToLeft); got != 0 { + t.Fatalf("a correctly ordered Hebrew page reported %d findings: %+v", + got, rep.Findings) + } + if got := rep.Count(verify.KindInvented); got != 0 { + t.Fatalf("a correctly ordered Hebrew page reported %d invented-text findings", got) + } +} + +// --- 3. suspicious joins + +func TestJoinsFireOnEachShape(t *testing.T) { + for _, tc := range []struct { + name string + text string + want verify.Kind + }{ + {"a hyphen followed by a space mid-word", "Der Gehäusede- ckel wird abgenommen", + verify.KindJoinHyphen}, + {"two words glued together", "Der Filter derDüse wird gereinigt", + verify.KindJoinGlued}, + {"a doubled space", "Der Filter wird gereinigt", verify.KindJoinSpace}, + } { + t.Run(tc.name, func(t *testing.T) { + in := verify.Input{ + Blocks: []doc.Block{block(4, 0, 43, 443, 200, tc.text)}, + Text: []doc.Page{page(4, "Der Gehäusedeckel wird abgenommen. "+ + "Der Filter der Düse wird gereinigt.")}, + } + rep := verify.Inspect(in) + if got := rep.Count(tc.want); got != 1 { + t.Fatalf("want one %s finding, got %d: %+v", tc.want, got, rep.Findings) + } + }) + } +} + +func TestJoinsQuietOnCleanText(t *testing.T) { + // A dash used as punctuation and no doubled space. Two of these hang directly + // off a letter — "230V- 50" and "Typ M- Amfibia" — which is what makes them the + // case the shape gets wrong: only the digit and the capital after the space say + // they are not a broken word. + const clean = "Spannungsversorgung: 230V- 50 Hz, Typ M- Amfibia, Modell 788/M - 2024" + in := verify.Input{ + Blocks: []doc.Block{block(4, 0, 43, 443, 200, clean)}, + Text: []doc.Page{page(4, clean)}, + } + rep := verify.Inspect(in) + for _, k := range []verify.Kind{verify.KindJoinHyphen, verify.KindJoinGlued, + verify.KindJoinSpace} { + if got := rep.Count(k); got != 0 { + t.Errorf("%s fired on clean text: %+v", k, rep.Findings) + } + } +} + +// --- 4. figure geometry, which is two faults + +// figurePNG renders a white image with a black rectangle in it, which is enough to +// stand in for a line drawing: the check reads where the paint is, not what it +// draws. +func figurePNG(t *testing.T, w, h int, painted image.Rectangle) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.Set(x, y, color.White) + } + } + for y := painted.Min.Y; y < painted.Max.Y; y++ { + for x := painted.Min.X; x < painted.Max.X; x++ { + img.Set(x, y, color.Black) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatalf("encode: %v", err) + } + return buf.Bytes() +} + +func figure(t *testing.T, rect doc.CellRect, painted image.Rectangle) doc.ConvertedFigure { + t.Helper() + pw := int(rect.Width() * 2) + ph := int(rect.Height() * 2) + return doc.ConvertedFigure{Figure: doc.Figure{ + Page: 14, Index: 0, Rect: rect, DPI: 216, + PixelWidth: pw, PixelHeight: ph, Ink: 40, + PNG: figurePNG(t, pw, ph, painted), + }} +} + +func TestFigureBandFiresOnARenderThatIsMostlyMargin(t *testing.T) { + // A 100x100 box rendered at 200x200 pixels, painted only below y=100 — which is + // 50 units of blank band at the top, the fault the user reports. + fig := figure(t, doc.CellRect{X0: 43, Y0: 241, X1: 143, Y1: 341}, + image.Rect(0, 100, 200, 200)) + rep := verify.Inspect(verify.Input{Figures: []doc.ConvertedFigure{fig}}) + if got := rep.Count(verify.KindFigureBand); got != 1 { + t.Fatalf("want one blank-band finding, got %d: %+v", got, rep.Findings) + } + if f := rep.Findings[0]; f.Got < 49 || f.Got > 51 { + t.Errorf("want a band of about 50 units, got %.1f", f.Got) + } +} + +func TestFigureBandQuietWhenThePictureFillsItsBox(t *testing.T) { + fig := figure(t, doc.CellRect{X0: 43, Y0: 241, X1: 143, Y1: 341}, + image.Rect(0, 0, 200, 200)) + if got := count(t, verify.Input{Figures: []doc.ConvertedFigure{fig}}, + verify.KindFigureBand); got != 0 { + t.Fatalf("want no blank-band finding, got %d", got) + } +} + +func TestFigureClippedFiresWhenAShapeCrossesTheBox(t *testing.T) { + box := doc.CellRect{X0: 0, Y0: 0, X1: 100, Y1: 100} + fig := figure(t, box, image.Rect(0, 0, 200, 200)) + in := verify.Input{ + Figures: []doc.ConvertedFigure{fig}, + Ink: map[int][]doc.Ink{14: { + {Rect: doc.CellRect{X0: 10, Y0: 10, X1: 90, Y1: 90}}, + {Rect: doc.CellRect{X0: 50, Y0: 40, X1: 150, Y1: 60}, Stroked: true}, + }}, + } + rep := verify.Inspect(in) + if got := rep.Count(verify.KindFigureClipped); got != 1 { + t.Fatalf("want one clipped finding, got %d: %+v", got, rep.Findings) + } + if f := rep.Findings[0]; f.Count != 1 || f.Total != 2 || f.Got < 49 { + t.Errorf("want 1 of 2 shapes crossing by about 50 units, got %d of %d by %.0f", + f.Count, f.Total, f.Got) + } +} + +func TestFigureClippedQuietWhenEveryShapeIsInside(t *testing.T) { + box := doc.CellRect{X0: 0, Y0: 0, X1: 100, Y1: 100} + fig := figure(t, box, image.Rect(0, 0, 200, 200)) + in := verify.Input{ + Figures: []doc.ConvertedFigure{fig}, + Ink: map[int][]doc.Ink{14: { + {Rect: doc.CellRect{X0: 10, Y0: 10, X1: 90, Y1: 90}}, + // A horizontal rule: zero height, and inside. It must not read as + // crossing, which an area comparison would make it. + {Rect: doc.CellRect{X0: 10, Y0: 50, X1: 90, Y1: 50}}, + // A page-sized background path, mostly outside: not this figure's. + {Rect: doc.CellRect{X0: -500, Y0: -500, X1: 900, Y1: 900}}, + }}, + } + if got := count(t, in, verify.KindFigureClipped); got != 0 { + t.Fatalf("want no clipped finding, got %d", got) + } +} + +// --- 5. reading order + +func TestReadingOrderFiresOnInterleavedColumns(t *testing.T) { + // The page-62 failure conversion.md describes: two columns read line by line, + // so the second block is in the other column and no higher up the page. + left := "Die Verpackung schützt das Gerät gegen Transportschäden" + right := "Gerät Garantie gemäß nachstehenden Bedingungen" + in := verify.Input{Blocks: []doc.Block{ + block(62, 0, 43, 443, 100, left), + block(62, 1, 463, 863, 118, right), + }} + rep := verify.Inspect(in) + if got := rep.Count(verify.KindReadingOrder); got != 1 { + t.Fatalf("want one reading-order finding, got %d: %+v", got, rep.Findings) + } + if f := rep.Findings[0]; f.Index != 1 || f.Got < f.Want { + t.Errorf("finding does not carry the two positions: %+v", f) + } +} + +func TestReadingOrderQuietWhenColumnsAreReadInTurn(t *testing.T) { + // Correct output: down the left column, then back up to the top of the right + // one. Going back up is right, and must not be reported. + in := verify.Input{Blocks: []doc.Block{ + block(62, 0, 43, 443, 100, "Die Verpackung schützt das Gerät gegen Transport"), + block(62, 1, 43, 443, 300, "Bitte entsorgen Sie das Material umweltgerecht"), + block(62, 2, 463, 863, 100, "Gerät Garantie gemäß nachstehenden Bedingungen"), + block(62, 3, 463, 863, 300, "Bei gewerblicher Benutzung oder gleichzusetzender"), + }} + rep := verify.Inspect(in) + if got := rep.Count(verify.KindReadingOrder); got != 0 { + t.Fatalf("correct column order reported %d findings: %+v", got, rep.Findings) + } +} + +func TestReadingOrderIgnoresTableCellsAndFurniture(t *testing.T) { + // A table is read row-major on purpose, which is this check's violation shape, + // and a two-letter language badge below a heading is the same shape again. + cells := []doc.Block{ + {Page: 57, Index: 0, Kind: doc.BlockTable, Text: "Gerät saugt nicht", Chars: 17, + X0: 30, X1: 173, Y0: 100, Y1: 130}, + {Page: 57, Index: 1, Kind: doc.BlockTable, Text: "Filter reinigen und wieder einsetzen", + Chars: 36, X0: 200, X1: 428, Y0: 100, Y1: 130}, + } + badge := []doc.Block{ + block(24, 0, 55, 243, 52, "Routine Maintenance"), + block(24, 1, 27, 41, 58, "DE"), + block(24, 2, 55, 813, 95, "Die Wartung erfolgt in den beschriebenen Abständen"), + } + in := verify.Input{Blocks: append(cells, badge...)} + rep := verify.Inspect(in) + if got := rep.Count(verify.KindReadingOrder); got != 0 { + t.Fatalf("table cells or page furniture were reported: %+v", rep.Findings) + } +} + +// TestReadingOrderDoesNotReadAcrossATable is the column manual's page 57. Its two +// side-by-side troubleshooting tables print a header row above a top border the +// document does not draw, so those headers are the page's only prose — and the left +// table's header is followed, across every cell of that table, by the right table's +// header at the same y. Dropping the cells from the comparison must not make those +// two consecutive: they are not, and the page is read correctly. +func TestReadingOrderDoesNotReadAcrossATable(t *testing.T) { + blocks := []doc.Block{ + block(57, 0, 36, 238, 67, "Aufgetretene Störungen und Fehlfunktionen"), + } + for i := 0; i < 12; i++ { + y := 107 + float64(i)*40 + blocks = append(blocks, doc.Block{Page: 57, Index: len(blocks), Kind: doc.BlockTable, + Text: "Zelle", Chars: 5, X0: 36, X1: 164, Y0: y, Y1: y + 30}) + } + blocks = append(blocks, block(57, len(blocks), 457, 659, 67, + "Aufgetretene Störungen und Fehlfunktionen")) + + rep := verify.Inspect(verify.Input{Blocks: blocks}) + if got := rep.Count(verify.KindReadingOrder); got != 0 { + t.Fatalf("the check read across a table it cannot see: %+v", rep.Findings) + } + + // Without the table between them the same two blocks ARE consecutive, and the + // check must still fire — otherwise this is silence and not a repair. + bare := []doc.Block{blocks[0], blocks[len(blocks)-1]} + bare[1].Index = 1 + if got := verify.Inspect(verify.Input{Blocks: bare}).Count(verify.KindReadingOrder); got != 1 { + t.Errorf("two level blocks in different columns with nothing between them "+ + "reported %d findings, want 1", got) + } +} + +// --- the report itself + +func TestReportSaysWhatItCouldNotCheck(t *testing.T) { + // No pdftotext reading and no ink: the checks that need them are skipped and + // said to be skipped, rather than passing silently. + in := verify.Input{ + Blocks: []doc.Block{block(1, 0, 40, 300, 100, prose)}, + Figures: []doc.ConvertedFigure{{Figure: doc.Figure{Page: 1}}}, + } + rep := verify.Inspect(in) + if len(rep.Notes) != 3 { + t.Fatalf("want three notes, got %v", rep.Notes) + } + joined := strings.Join(rep.Notes, " | ") + for _, want := range []string{"pdftotext", "ink", "rendered bytes"} { + if !strings.Contains(joined, want) { + t.Errorf("no note about %s: %v", want, rep.Notes) + } + } + if rep.Count(verify.KindCoverage) != 0 { + t.Error("coverage was judged with nothing to judge it against") + } +} + +func TestSummaryCountsEveryKind(t *testing.T) { + in := verify.Input{ + Blocks: []doc.Block{block(7, 0, 40, 300, 100, prose[:len(prose)/2])}, + Text: []doc.Page{page(7, prose)}, + } + rep := verify.Inspect(in) + s := rep.Summary() + for _, want := range []string{"1 finding(s)", string(verify.KindCoverage), "median coverage"} { + if !strings.Contains(s, want) { + t.Errorf("Summary() = %q, want it to mention %q", s, want) + } + } +} diff --git a/testdata/fixtures/dreame-l40-ultra.json b/testdata/fixtures/dreame-l40-ultra.json index cded26b..c4a62c0 100644 --- a/testdata/fixtures/dreame-l40-ultra.json +++ b/testdata/fixtures/dreame-l40-ultra.json @@ -3,7 +3,14 @@ "Test fixture manifest for a real 34-language appliance manual.", "The PDF is NOT committed: 15 MB, and it is a third-party copyrighted", "manual — committing it would break manualbox's own rule against", - "redistributing manuals. Tests fetch it on demand and skip if absent." + "redistributing manuals. Tests fetch it on demand and skip if absent.", + "", + "printed_page is what the PRINTED INDEX claims for each section, which is", + "not always the folio actually printed on that page — that unreliability is", + "part of what this fixture exists to test. pdf_start/pdf_end are measured", + "truth, established from the per-page language tag (see page_language_tag).", + "printed_to_pdf_offset is constant at +6 for every section: pdf page =", + "printed folio + 6, because six pages of front matter precede the content." ], "name": "dreame-l40-ultra", "url": "https://cdn.shopify.com/s/files/1/0302/5276/1220/files/User_Manual-L40_Ultra_AE-EN_DE_FR_IT_ES_PL_NL_NO_SV_EL_PT_HE_AR_MS_FI_DA_KK_UZ_UA_CZ_HU_SL_SR_LT_LV_SK_RO_TR_VI_TH_ID_ZH-HK_RU_JA.pdf", @@ -11,18 +18,35 @@ "bytes": 15285327, "pages": 560, "has_text_layer": true, - "median_chars_per_page": 2241, + "median_chars_per_page": 1693, "content_starts_on_pdf_page": 7, + "page_box": { + "width": 918, + "height": 620 + }, + "text_runs": 34413, + "layout": "sequential-sections", + "layout_note": "One language per page — but NOT one column per page, which was assumed once and is wrong. Measured column counts across all 560 pages: 0 on 6, one on 148, two on 136, three on 199, four on 71. The multi-column pages are overwhelmingly side-by-side troubleshooting tables; pages 20 and 100 were checked against their renders and each is two tables of two cells, whose cells are what the detector returns. So column count is not language count on this manual either, for the opposite reason to thomas-drybox-amfibia: there, one page holds several languages; here, one language is laid out in several table cells.", "index_pages": [ 2, - 3 + 3, + 4 ], "why_this_fixture": [ "34 language sections in one document — the normal appliance-manual shape", - "the printed index contains a typo (CZ listed at 207; content is at 305)", - "printed-to-pdf offset drifts (+6 early, +8 later) as sections vary 16 vs 17 pages", + "every content page prints its own language code, which is exact and free", + " (see page_language_tag) — but contents pages 2-4 mimic it and must be excluded", + "the printed index contains a typo (CZ listed at 207; content is at printed 307)", + "the index's claimed printed pages drift 0-2 from the real folio, because IT", + " and PL run 17 pages rather than 16 — the printed-to-pdf offset itself is a", + " constant +6 and never drifts", "right-to-left (HE, AR) and CJK (ZH-HK, JA) scripts alongside Latin", - "sibling languages the detector alone cannot separate (ID/MS, DA/NO, SK/CZ)" + "HE and AR pages print no numeric folio at all", + "sibling languages a detector alone cannot separate: ID/MS, DA/NO(nb), SK/CZ,", + " and SR/HR/BS — Latin-script Serbian is misread on all 16 of its pages", + "UZ is not supported by lingua-go at all, so no detector can ever label it —", + " the printed tag and the printed index are the only sources for that section", + "page 560 is a back cover: English colophon, no tag, not part of the JA section" ], "sections": [ { @@ -32,7 +56,7 @@ "pdf_start": 7, "pdf_end": 22, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "DE", @@ -41,7 +65,7 @@ "pdf_start": 23, "pdf_end": 38, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "FR", @@ -50,7 +74,7 @@ "pdf_start": 39, "pdf_end": 54, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "IT", @@ -59,7 +83,7 @@ "pdf_start": 55, "pdf_end": 71, "pages": 17, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "ES", @@ -68,7 +92,7 @@ "pdf_start": 72, "pdf_end": 87, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "PL", @@ -77,7 +101,7 @@ "pdf_start": 88, "pdf_end": 104, "pages": 17, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "NL", @@ -86,7 +110,7 @@ "pdf_start": 105, "pdf_end": 120, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "NO", @@ -95,7 +119,7 @@ "pdf_start": 121, "pdf_end": 136, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "SV", @@ -104,7 +128,7 @@ "pdf_start": 137, "pdf_end": 152, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "EL", @@ -113,7 +137,7 @@ "pdf_start": 153, "pdf_end": 168, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "PT", @@ -122,7 +146,7 @@ "pdf_start": 169, "pdf_end": 184, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "HE", @@ -131,7 +155,7 @@ "pdf_start": 185, "pdf_end": 200, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "AR", @@ -140,7 +164,7 @@ "pdf_start": 201, "pdf_end": 216, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "MS", @@ -149,25 +173,27 @@ "pdf_start": 217, "pdf_end": 232, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "FI", "title": "Käyttöohjeet", "printed_page": 227, "pdf_start": 233, - "pdf_end": 256, - "pages": 24, - "boundary_source": "detected" + "pdf_end": 248, + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously ended this section at 256 (24 pages). The printed page tag reads FI through p248 and DA from p249; orthography agrees (pp233-248 carry 1252 ae/oe-umlauts and zero aesc/oslash/aring, pp249-264 the reverse)." }, { "code": "DA", "title": "Brugermanual", "printed_page": 243, - "pdf_start": 257, + "pdf_start": 249, "pdf_end": 264, - "pages": 8, - "boundary_source": "detected" + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously started this section at 257 (8 pages), contradicting its own printed_page of 243 - printed folio 243 is on pdf p249. Corrected to 16 pages, this manual's standard section length." }, { "code": "KK", @@ -176,7 +202,7 @@ "pdf_start": 265, "pdf_end": 280, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "UZ", @@ -185,7 +211,7 @@ "pdf_start": 281, "pdf_end": 296, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "UA", @@ -194,7 +220,7 @@ "pdf_start": 297, "pdf_end": 312, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "CZ", @@ -205,7 +231,7 @@ "pdf_start": 313, "pdf_end": 328, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "HU", @@ -214,7 +240,7 @@ "pdf_start": 329, "pdf_end": 344, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "SL", @@ -223,43 +249,47 @@ "pdf_start": 345, "pdf_end": 360, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "SR", "title": "Uputstvo za upotrebu", "printed_page": 355, "pdf_start": 361, - "pdf_end": 380, - "pages": 20, - "boundary_source": "detected" + "pdf_end": 376, + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously ended this section at 380 (20 pages). Serbian here is Latin script, so a statistical detector reads it as Croatian or Bosnian on all 16 pages; the printed page tag is the only reliable label." }, { "code": "LT", "title": "Naudotojo vadovas", "printed_page": 371, - "pdf_start": 381, + "pdf_start": 377, "pdf_end": 392, - "pages": 12, - "boundary_source": "detected" + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously started this section at 381 (12 pages), contradicting its own printed_page of 371 - printed folio 371 is on pdf p377." }, { "code": "LV", "title": "Lietotāja rokasgrāmata", "printed_page": 387, "pdf_start": 393, - "pdf_end": 412, - "pages": 20, - "boundary_source": "detected" + "pdf_end": 408, + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously ended this section at 412 (20 pages). Latvian macrons (aa/ee/ii/uu, g/k/l/n cedilla) appear on pp393-408 and never after; Slovak ae/o-circumflex/d/l-caron appear from p409." }, { "code": "SK", "title": "Príručka používateľa", "printed_page": 403, - "pdf_start": 413, + "pdf_start": 409, "pdf_end": 424, - "pages": 12, - "boundary_source": "detected" + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously started this section at 413 (12 pages), contradicting its own printed_page of 403 - printed folio 403 is on pdf p409." }, { "code": "RO", @@ -268,25 +298,27 @@ "pdf_start": 425, "pdf_end": 440, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "TR", "title": "Kullanıcı Kılavuzu", "printed_page": 435, "pdf_start": 441, - "pdf_end": 461, - "pages": 21, - "boundary_source": "detected" + "pdf_end": 456, + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously ended this section at 461 (21 pages). Turkish dotless-i and g-breve appear on pp441-456 and never after." }, { "code": "VI", "title": "Hướng dẫn sử dụng", "printed_page": 451, - "pdf_start": 462, + "pdf_start": 457, "pdf_end": 472, - "pages": 11, - "boundary_source": "detected" + "pages": 16, + "boundary_source": "page-tag", + "note": "Fixture previously started this section at 462 (11 pages), contradicting its own printed_page of 451 - printed folio 451 is on pdf p457." }, { "code": "TH", @@ -295,7 +327,7 @@ "pdf_start": 473, "pdf_end": 488, "pages": 16, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "ID", @@ -304,7 +336,7 @@ "pdf_start": 489, "pdf_end": 504, "pages": 16, - "boundary_source": "inferred (detector cannot distinguish from a sibling language)" + "boundary_source": "page-tag" }, { "code": "ZH-HK", @@ -313,7 +345,7 @@ "pdf_start": 505, "pdf_end": 516, "pages": 12, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "RU", @@ -322,16 +354,65 @@ "pdf_start": 517, "pdf_end": 538, "pages": 22, - "boundary_source": "detected" + "boundary_source": "page-tag" }, { "code": "JA", "title": "取扱説明書", "printed_page": 533, "pdf_start": 539, - "pdf_end": 560, - "pages": 22, - "boundary_source": "detected" + "pdf_end": 559, + "pages": 21, + "boundary_source": "page-tag", + "note": "Fixture previously ended this section at 560 (22 pages). Page 560 is the back cover - an English colophon and manufacturing address, carrying no page tag - so content ends at p559." } - ] + ], + "content_ends_on_pdf_page": 559, + "median_bytes_per_page": 2240, + "printed_to_pdf_offset": 6, + "page_language_tag": { + "$comment": [ + "Every content page in this manual prints its own ISO language code in a", + "small tab in the top-left corner. This is the cheapest and most accurate", + "language signal available, and it is already present in the plain", + "pdftotext output that the text probe runs anyway — it is the FIRST", + "non-blank line of each page. It costs nothing extra to read.", + "", + "Measured on this document: 553 of 553 content pages carry a tag and all", + "553 agree with the boundaries recorded here. It labels correctly the two", + "sections a statistical detector cannot: UZ (unsupported by lingua-go at", + "all) and Latin-script SR (read as Croatian or Bosnian on every page).", + "", + "It is not universal across manuals, so it is a high-confidence signal", + "when present rather than a replacement for detection. Two guards are", + "required, both measured here:", + " 1. Contents pages produce FALSE POSITIVES. Pages 2-4 list language", + " codes in the same corner, yielding spurious single-page runs for", + " EN, MS and RO. Requiring a run of >= 2 consecutive pages removes", + " all three and leaves exactly the 34 real sections.", + " 2. A bare two-letter uppercase token is not necessarily a language", + " code (ON, OK, NO, TV all match). Cross-check the tag against the", + " page's dominant Unicode script before trusting it." + ], + "present": true, + "position": "top-left", + "appears_as": "first non-blank line of plain pdftotext output", + "color": "#ffffff", + "pages_with_tag": 553, + "pages_without_tag": [ + 1, + 2, + 3, + 4, + 5, + 6, + 560 + ], + "min_run_pages_to_trust": 2, + "false_positive_pages": [ + 2, + 3, + 4 + ] + } } diff --git a/testdata/fixtures/thomas-drybox-amfibia.json b/testdata/fixtures/thomas-drybox-amfibia.json new file mode 100644 index 0000000..b9aef64 --- /dev/null +++ b/testdata/fixtures/thomas-drybox-amfibia.json @@ -0,0 +1,1649 @@ +{ + "$comment": [ + "Ground truth for a multi-language manual whose languages sit in PARALLEL", + "COLUMNS rather than sequential sections. The counter-example to", + "dreame-l40-ultra.json. The PDF is NOT committed: 9 MB, third-party copyright.", + "", + "PROVENANCE IS PER PAGE, and it matters. verified=\"image\" means a human", + "compared the page against its render and confirmed the column count; those", + "eight pages are the acceptance test. verified=\"detector\" means the entry was", + "produced by internal/doc DetectColumns and has NOT been checked by eye — it", + "records the current reading, not established truth, and a detector cannot be", + "held to it without the reasoning going in a circle.", + "", + "An earlier version of this file was generated by an ad-hoc script that split", + "columns on gaps wider than 90px. It was wrong on 4 of the 8 verified pages,", + "because the real gutters here are 9 to 17px. That is why provenance is", + "recorded now instead of being assumed.", + "", + "Languages come from the character-repertoire signal, which is independent of", + "the column geometry. An empty lang means it declined rather than guessed." + ], + "name": "thomas-drybox-amfibia", + "url": "https://thomas.ua/manual/788596.pdf", + "sha256": "f8b3b2c5cc72330f4352682201020b0313039bd04a6b61469fe7a344391264da", + "bytes": 9424525, + "pages": 68, + "has_text_layer": true, + "median_chars_per_page": 3556, + "tagged": false, + "layout": "parallel-columns", + "layout_note": "Not uniform. One document, several arrangements: three columns of three languages, two of two, two of ONE language, a single column beside a full-height image, and five pages of side-by-side tables. Column count alone identifies none of them.", + "languages": [ + "de", + "kk", + "pl", + "ru", + "uk" + ], + "columns_established": 165, + "columns_total": 169, + "coordinate_space": "pdftohtml -xml, page width 892. A raster from pdftoppm -r 108 matches it 1:1, which is what lets a detected box be checked against the render.", + "page_box": { + "width": 892, + "height": 850 + }, + "text_runs": 7493, + "known_limitations": [ + "TABLE PAGES ARE 52-61, NOT 57-61. Measured from the document's own ruled lines:", + " pages 52-56 carry genuine small tables (Anwendungsfall | Duese/Zubehoer) that", + " this manifest recorded as ordinary column pages. And pages 62-66 print", + " Technische Daten as label/value pairs with NO ruled lines at all - tables that", + " nothing detects and that were recorded nowhere. See docs/design/conversion.md.", + "Pages 57-61 are troubleshooting tables: two side-by-side tables of two cells each.", + " Geometry cannot tell a table cell from a text column; that call belongs above", + " this layer, and is made there by dividing a page on language rather than on", + " cells. Page 57's narrow left cell WAS the document's one language misread", + " (German read as Finnish). It is fixed, and the per-page entry below still", + " records the reading it had - deliberately, because that entry is what a human", + " verified against the render. The cause turned out to be that the page has no", + " language columns at all: its four 'columns' are two tables' cell dividers, so", + " the detector was asked to name the language of a column of table labels. Now", + " that a table's dividers are excluded, page 57 is one whole-page German region", + " and the alphabet has u-umlaut x17 and eszett to read. See", + " docs/design/conversion.md.", + "THE CONTENTS PAGES OF THIS MANUAL CANNOT BE PARSED. They are pages 2 and 3, laid", + " out as one column per language with title-and-dot-leader entries rather than", + " the code/title/page triples the parser expects, so no language vocabulary is", + " recovered from them at all. Two measured consequences: printed-tag naming", + " reaches 53 of 169 columns instead of the 79 the real vocabulary allows, and", + " page 57 is misread. Until this is fixed the index signal contributes NOTHING", + " to this document.", + "Page 68, the back page of service addresses in six languages, is genuinely", + " unnameable and its three columns are recorded as establishing nothing. It was", + " previously suppressed by accident, the address page being misread as a contents", + " table; now it is refused deliberately, because only one of its three columns", + " names anything.", + "Page 1 (cover) is the weakest reading: both 0 and 1 columns are defensible." + ], + "why_this_fixture": [ + "languages share a page, so a language is a column and not a span of pages", + "the arrangement changes within the document, so one layout per file is wrong", + "two columns of one language proves column count is not language count", + "only 3 of 5 languages print a page tag, so a present signal need not cover a document", + "Russian, Ukrainian and Kazakh share a script, which Unicode analysis cannot separate", + "gutters are 9-17px, far narrower than a naive threshold assumes", + "the text layer carries invisible production artifacts that bridge gutters", + "218 runs on one page are parked off-page at negative coordinates" + ], + "page_facts": [ + { + "page": 1, + "columns": 1, + "spanning": 4, + "verified": "detector", + "cols": [ + { + "x0": 499, + "x1": 850, + "runs": 8, + "lang": "kk" + } + ] + }, + { + "page": 2, + "columns": 3, + "spanning": 0, + "verified": "image", + "cols": [ + { + "x0": 43, + "x1": 305, + "runs": 50, + "lang": "de" + }, + { + "x0": 323, + "x1": 585, + "runs": 47, + "lang": "pl" + }, + { + "x0": 604, + "x1": 866, + "runs": 46, + "lang": "ru" + } + ] + }, + { + "page": 3, + "columns": 2, + "spanning": 0, + "verified": "image", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 46, + "lang": "uk" + }, + { + "x0": 310, + "x1": 573, + "runs": 46, + "lang": "kk" + } + ] + }, + { + "page": 4, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 305, + "runs": 40, + "lang": "de" + }, + { + "x0": 323, + "x1": 585, + "runs": 43, + "lang": "pl" + }, + { + "x0": 604, + "x1": 866, + "runs": 39, + "lang": "ru" + } + ] + }, + { + "page": 5, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 43, + "lang": "uk" + }, + { + "x0": 310, + "x1": 573, + "runs": 42, + "lang": "kk" + } + ] + }, + { + "page": 6, + "columns": 2, + "spanning": 0, + "verified": "image", + "cols": [ + { + "x0": 43, + "x1": 446, + "runs": 40, + "lang": "de" + }, + { + "x0": 463, + "x1": 866, + "runs": 19, + "lang": "de" + } + ] + }, + { + "page": 7, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 434, + "runs": 38, + "lang": "pl" + }, + { + "x0": 451, + "x1": 852, + "runs": 21, + "lang": "pl" + } + ] + }, + { + "page": 8, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 447, + "runs": 39, + "lang": "ru" + }, + { + "x0": 463, + "x1": 867, + "runs": 18, + "lang": "ru" + } + ] + }, + { + "page": 9, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 434, + "runs": 39, + "lang": "uk" + }, + { + "x0": 451, + "x1": 854, + "runs": 16, + "lang": "uk" + } + ] + }, + { + "page": 10, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 447, + "runs": 39, + "lang": "kk" + }, + { + "x0": 463, + "x1": 866, + "runs": 16, + "lang": "kk" + } + ] + }, + { + "page": 11, + "columns": 1, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 591, + "x1": 849, + "runs": 88, + "lang": "de" + } + ] + }, + { + "page": 12, + "columns": 1, + "spanning": 0, + "verified": "image", + "cols": [ + { + "x0": 604, + "x1": 865, + "runs": 94, + "lang": "pl" + } + ] + }, + { + "page": 13, + "columns": 3, + "spanning": 0, + "verified": "image", + "cols": [ + { + "x0": 30, + "x1": 291, + "runs": 97, + "lang": "ru" + }, + { + "x0": 310, + "x1": 570, + "runs": 96, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 95, + "lang": "kk" + } + ] + }, + { + "page": 14, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 584, + "runs": 25, + "lang": "de" + }, + { + "x0": 604, + "x1": 862, + "runs": 25, + "lang": "pl" + } + ] + }, + { + "page": 15, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 285, + "runs": 23, + "lang": "ru" + }, + { + "x0": 310, + "x1": 570, + "runs": 25, + "lang": "uk" + }, + { + "x0": 591, + "x1": 845, + "runs": 23, + "lang": "kk" + } + ] + }, + { + "page": 16, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 583, + "runs": 31, + "lang": "de" + }, + { + "x0": 604, + "x1": 866, + "runs": 31, + "lang": "pl" + } + ] + }, + { + "page": 17, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 291, + "runs": 32, + "lang": "ru" + }, + { + "x0": 310, + "x1": 571, + "runs": 34, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 32, + "lang": "kk" + } + ] + }, + { + "page": 18, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 581, + "runs": 33, + "lang": "de" + }, + { + "x0": 604, + "x1": 864, + "runs": 36, + "lang": "pl" + } + ] + }, + { + "page": 19, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 287, + "runs": 32, + "lang": "ru" + }, + { + "x0": 310, + "x1": 566, + "runs": 35, + "lang": "uk" + }, + { + "x0": 591, + "x1": 853, + "runs": 33, + "lang": "kk" + } + ] + }, + { + "page": 20, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 585, + "runs": 36, + "lang": "de" + }, + { + "x0": 604, + "x1": 866, + "runs": 33, + "lang": "pl" + } + ] + }, + { + "page": 21, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 40, + "lang": "ru" + }, + { + "x0": 310, + "x1": 571, + "runs": 38, + "lang": "uk" + }, + { + "x0": 591, + "x1": 852, + "runs": 35, + "lang": "kk" + } + ] + }, + { + "page": 22, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 584, + "runs": 18, + "lang": "de" + }, + { + "x0": 604, + "x1": 867, + "runs": 20, + "lang": "pl" + } + ] + }, + { + "page": 23, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 288, + "runs": 19, + "lang": "ru" + }, + { + "x0": 310, + "x1": 568, + "runs": 19, + "lang": "uk" + }, + { + "x0": 591, + "x1": 853, + "runs": 17, + "lang": "kk" + } + ] + }, + { + "page": 24, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 585, + "runs": 38, + "lang": "de" + }, + { + "x0": 604, + "x1": 867, + "runs": 39, + "lang": "pl" + } + ] + }, + { + "page": 25, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 293, + "runs": 38, + "lang": "ru" + }, + { + "x0": 310, + "x1": 573, + "runs": 40, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 38, + "lang": "kk" + } + ] + }, + { + "page": 26, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 583, + "runs": 20, + "lang": "de" + }, + { + "x0": 604, + "x1": 859, + "runs": 22, + "lang": "pl" + } + ] + }, + { + "page": 27, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 289, + "runs": 23, + "lang": "ru" + }, + { + "x0": 310, + "x1": 572, + "runs": 22, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 23, + "lang": "kk" + } + ] + }, + { + "page": 28, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 584, + "runs": 25, + "lang": "de" + }, + { + "x0": 604, + "x1": 866, + "runs": 25, + "lang": "pl" + } + ] + }, + { + "page": 29, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 30, + "lang": "ru" + }, + { + "x0": 310, + "x1": 570, + "runs": 26, + "lang": "uk" + }, + { + "x0": 591, + "x1": 853, + "runs": 25, + "lang": "kk" + } + ] + }, + { + "page": 30, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 585, + "runs": 32, + "lang": "de" + }, + { + "x0": 604, + "x1": 865, + "runs": 28, + "lang": "pl" + } + ] + }, + { + "page": 31, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 291, + "runs": 31, + "lang": "ru" + }, + { + "x0": 310, + "x1": 565, + "runs": 33, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 31, + "lang": "kk" + } + ] + }, + { + "page": 32, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 583, + "runs": 19, + "lang": "de" + }, + { + "x0": 604, + "x1": 864, + "runs": 18, + "lang": "pl" + } + ] + }, + { + "page": 33, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 18, + "lang": "ru" + }, + { + "x0": 310, + "x1": 567, + "runs": 20, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 20, + "lang": "kk" + } + ] + }, + { + "page": 34, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 585, + "runs": 22, + "lang": "de" + }, + { + "x0": 604, + "x1": 865, + "runs": 29, + "lang": "pl" + } + ] + }, + { + "page": 35, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 291, + "runs": 24, + "lang": "ru" + }, + { + "x0": 310, + "x1": 567, + "runs": 24, + "lang": "uk" + }, + { + "x0": 591, + "x1": 852, + "runs": 25, + "lang": "kk" + } + ] + }, + { + "page": 36, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 586, + "runs": 28, + "lang": "de" + }, + { + "x0": 604, + "x1": 863, + "runs": 29, + "lang": "pl" + } + ] + }, + { + "page": 37, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 293, + "runs": 27, + "lang": "ru" + }, + { + "x0": 310, + "x1": 573, + "runs": 28, + "lang": "uk" + }, + { + "x0": 591, + "x1": 845, + "runs": 26, + "lang": "kk" + } + ] + }, + { + "page": 38, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 578, + "runs": 47, + "lang": "de" + }, + { + "x0": 604, + "x1": 866, + "runs": 47, + "lang": "pl" + } + ] + }, + { + "page": 39, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 33, + "lang": "ru" + }, + { + "x0": 310, + "x1": 572, + "runs": 35, + "lang": "uk" + }, + { + "x0": 591, + "x1": 853, + "runs": 42, + "lang": "kk" + } + ] + }, + { + "page": 40, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 585, + "runs": 35, + "lang": "de" + }, + { + "x0": 604, + "x1": 865, + "runs": 36, + "lang": "pl" + } + ] + }, + { + "page": 41, + "columns": 3, + "spanning": 0, + "verified": "image", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 35, + "lang": "ru" + }, + { + "x0": 310, + "x1": 572, + "runs": 33, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 36, + "lang": "kk" + } + ] + }, + { + "page": 42, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 584, + "runs": 16, + "lang": "de" + }, + { + "x0": 604, + "x1": 864, + "runs": 21, + "lang": "pl" + } + ] + }, + { + "page": 43, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 285, + "runs": 13, + "lang": "ru" + }, + { + "x0": 310, + "x1": 572, + "runs": 15, + "lang": "uk" + }, + { + "x0": 591, + "x1": 852, + "runs": 14, + "lang": "kk" + } + ] + }, + { + "page": 44, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 584, + "runs": 25, + "lang": "de" + }, + { + "x0": 604, + "x1": 862, + "runs": 18, + "lang": "pl" + } + ] + }, + { + "page": 45, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 290, + "runs": 24, + "lang": "ru" + }, + { + "x0": 310, + "x1": 570, + "runs": 24, + "lang": "uk" + }, + { + "x0": 591, + "x1": 852, + "runs": 27, + "lang": "kk" + } + ] + }, + { + "page": 46, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 585, + "runs": 26, + "lang": "de" + }, + { + "x0": 604, + "x1": 866, + "runs": 28, + "lang": "pl" + } + ] + }, + { + "page": 47, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 289, + "runs": 33, + "lang": "ru" + }, + { + "x0": 310, + "x1": 571, + "runs": 29, + "lang": "uk" + }, + { + "x0": 591, + "x1": 851, + "runs": 30, + "lang": "kk" + } + ] + }, + { + "page": 48, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 584, + "runs": 35, + "lang": "de" + }, + { + "x0": 604, + "x1": 866, + "runs": 35, + "lang": "pl" + } + ] + }, + { + "page": 49, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 289, + "runs": 39, + "lang": "ru" + }, + { + "x0": 310, + "x1": 572, + "runs": 38, + "lang": "uk" + }, + { + "x0": 591, + "x1": 853, + "runs": 37, + "lang": "kk" + } + ] + }, + { + "page": 50, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 323, + "x1": 584, + "runs": 38, + "lang": "de" + }, + { + "x0": 604, + "x1": 863, + "runs": 30, + "lang": "pl" + } + ] + }, + { + "page": 51, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 289, + "runs": 35, + "lang": "ru" + }, + { + "x0": 310, + "x1": 572, + "runs": 36, + "lang": "uk" + }, + { + "x0": 591, + "x1": 852, + "runs": 39, + "lang": "kk" + } + ] + }, + { + "page": 52, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 305, + "runs": 55, + "lang": "de" + }, + { + "x0": 323, + "x1": 585, + "runs": 41, + "lang": "de" + }, + { + "x0": 604, + "x1": 866, + "runs": 47, + "lang": "de" + } + ] + }, + { + "page": 53, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 49, + "lang": "pl" + }, + { + "x0": 310, + "x1": 572, + "runs": 47, + "lang": "pl" + }, + { + "x0": 591, + "x1": 853, + "runs": 49, + "lang": "pl" + } + ] + }, + { + "page": 54, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 305, + "runs": 59, + "lang": "ru" + }, + { + "x0": 323, + "x1": 585, + "runs": 50, + "lang": "ru" + }, + { + "x0": 604, + "x1": 866, + "runs": 54, + "lang": "ru" + } + ] + }, + { + "page": 55, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 292, + "runs": 59, + "lang": "uk" + }, + { + "x0": 310, + "x1": 572, + "runs": 40, + "lang": "uk" + }, + { + "x0": 591, + "x1": 853, + "runs": 48, + "lang": "uk" + } + ] + }, + { + "page": 56, + "columns": 3, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 305, + "runs": 64, + "lang": "kk" + }, + { + "x0": 323, + "x1": 585, + "runs": 38, + "lang": "kk" + }, + { + "x0": 604, + "x1": 866, + "runs": 49, + "lang": "kk" + } + ] + }, + { + "page": 57, + "columns": 4, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 36, + "x1": 178, + "runs": 13, + "lang": "", + "note": "read as Finnish; it is German. Recorded as unestablished rather than wrong. The cause is NOT that the cell is too short, which is what this note used to say: the page prints its language as D in the oval at its top left, that run falls inside this column's box, and columnTag finds it and then rejects it because a single letter must be corroborated by the document's own index vocabulary. This manual's contents pages are laid out in parallel columns and the index parser cannot read them, so there is no vocabulary. Supply the real one and the column reads code=D lang=de src=page-tag and the alphabet is never consulted. The alphabet's own reading is a distant second cause: the cell holds a-umlaut x5 and o-umlaut x1 and no u-umlaut or sharp-s at all, so Finnish uses 2 of its 2 distinctive letters where German uses 2 of 4, and Finnish scores higher." + }, + { + "x0": 179, + "x1": 424, + "runs": 57, + "lang": "de" + }, + { + "x0": 457, + "x1": 589, + "runs": 19, + "lang": "de" + }, + { + "x0": 601, + "x1": 846, + "runs": 47, + "lang": "de" + } + ] + }, + { + "page": 58, + "columns": 4, + "spanning": 3, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 165, + "runs": 10, + "lang": "pl" + }, + { + "x0": 192, + "x1": 438, + "runs": 49, + "lang": "pl" + }, + { + "x0": 469, + "x1": 591, + "runs": 15, + "lang": "pl" + }, + { + "x0": 613, + "x1": 857, + "runs": 46, + "lang": "pl" + } + ] + }, + { + "page": 59, + "columns": 4, + "spanning": 1, + "verified": "detector", + "cols": [ + { + "x0": 36, + "x1": 159, + "runs": 11, + "lang": "ru" + }, + { + "x0": 180, + "x1": 425, + "runs": 58, + "lang": "ru" + }, + { + "x0": 457, + "x1": 591, + "runs": 17, + "lang": "ru" + }, + { + "x0": 601, + "x1": 845, + "runs": 51, + "lang": "ru" + } + ] + }, + { + "page": 60, + "columns": 4, + "spanning": 5, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 180, + "runs": 9, + "lang": "uk" + }, + { + "x0": 192, + "x1": 435, + "runs": 53, + "lang": "uk" + }, + { + "x0": 470, + "x1": 602, + "runs": 13, + "lang": "uk" + }, + { + "x0": 613, + "x1": 858, + "runs": 48, + "lang": "uk" + } + ] + }, + { + "page": 61, + "columns": 4, + "spanning": 1, + "verified": "detector", + "cols": [ + { + "x0": 36, + "x1": 170, + "runs": 9, + "lang": "kk" + }, + { + "x0": 180, + "x1": 425, + "runs": 55, + "lang": "kk" + }, + { + "x0": 457, + "x1": 584, + "runs": 14, + "lang": "kk" + }, + { + "x0": 601, + "x1": 847, + "runs": 50, + "lang": "kk" + } + ] + }, + { + "page": 62, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 443, + "runs": 43, + "lang": "de" + }, + { + "x0": 463, + "x1": 863, + "runs": 34, + "lang": "de" + } + ] + }, + { + "page": 63, + "columns": 2, + "spanning": 1, + "verified": "image", + "cols": [ + { + "x0": 30, + "x1": 425, + "runs": 41, + "lang": "pl" + }, + { + "x0": 451, + "x1": 851, + "runs": 39, + "lang": "pl" + } + ] + }, + { + "page": 64, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 441, + "runs": 50, + "lang": "ru" + }, + { + "x0": 463, + "x1": 865, + "runs": 41, + "lang": "ru" + } + ] + }, + { + "page": 65, + "columns": 2, + "spanning": 0, + "verified": "detector", + "cols": [ + { + "x0": 30, + "x1": 428, + "runs": 49, + "lang": "uk" + }, + { + "x0": 451, + "x1": 852, + "runs": 35, + "lang": "uk" + } + ] + }, + { + "page": 66, + "columns": 2, + "spanning": 1, + "verified": "detector", + "cols": [ + { + "x0": 43, + "x1": 445, + "runs": 48, + "lang": "kk" + }, + { + "x0": 463, + "x1": 865, + "runs": 32, + "lang": "kk" + } + ] + }, + { + "page": 67, + "columns": 0, + "spanning": 0, + "verified": "detector", + "cols": [] + }, + { + "page": 68, + "columns": 3, + "spanning": 2, + "verified": "image", + "cols": [ + { + "x0": 61, + "x1": 296, + "runs": 75, + "lang": "" + }, + { + "x0": 332, + "x1": 559, + "runs": 197, + "lang": "" + }, + { + "x0": 564, + "x1": 840, + "runs": 207, + "lang": "" + } + ] + } + ] +} \ No newline at end of file diff --git a/web/folio-browser-check.html b/web/folio-browser-check.html new file mode 100644 index 0000000..c048a18 --- /dev/null +++ b/web/folio-browser-check.html @@ -0,0 +1,12 @@ + + + + + + manualbox — folio check + + +
+ + + diff --git a/web/folio-browser-check.tsx b/web/folio-browser-check.tsx new file mode 100644 index 0000000..1f7933b --- /dev/null +++ b/web/folio-browser-check.tsx @@ -0,0 +1,83 @@ +/** + * Look at a contents entry in a real browser, and click it. + * + * reader-check.tsx renders the reader to static markup, which is enough to read what + * is on the page but cannot answer the question a link raises: does clicking it move + * the reader. So this mounts the REAL [Reader] -- not a copy of its wiring -- in a + * real browser, with a real document's conversion JSON standing in for the network, + * and lets Chrome click. + * + * `fetch` is replaced before mounting rather than the component being given props it + * does not have, so what runs is the same code path the app runs: Reader asks the api + * client, the client asks fetch, and the conversion arrives with whatever + * `folioOffset` the server actually sent -- including none, which is a case worth + * looking at. + * + * Usage, from web/: + * npx vite build --config folio-browser-check.vite.ts + * open .folio-check/index.html + * + * The conversion JSON is inlined at build time from FOLIO_CHECK_JSON. It is a real + * manual's text and is never committed; .folio-check is build output. + */ +import { StrictMode } from "react"; +import { createRoot } from "react-dom/client"; + +import type { Conversion, Doc } from "./src/api/types"; +import { Reader } from "./src/screens/Reader"; +import "./src/index.css"; + +// Inlined by the vite config's `define`. +declare const __CONVERSION__: Conversion; +declare const __DROP_OFFSET__: boolean; +declare const __FORCE_OFFSET__: number | null; + +const conversion: Conversion = JSON.parse(JSON.stringify(__CONVERSION__)); +if (__DROP_OFFSET__) { + // The document whose folios agreed on nothing. Every entry must fall back to + // plain text, and the number must still be readable. + delete conversion.folioOffset; +} else if (__FORCE_OFFSET__ !== null) { + // An offset that lands most entries on pages this language does not hold. The + // columns manual does not do this to itself -- each language's contents page + // prints its own folios, which are its own pages -- so it is forced here to see + // what a reader meets when a target is not servable. + conversion.folioOffset = __FORCE_OFFSET__; +} + +window.fetch = (async () => + new Response(JSON.stringify(conversion), { + status: 200, + headers: { "content-type": "application/json" }, + })) as typeof fetch; + +const doc: Doc = { + id: "doc_example", + deviceId: "dev_example", + blobSha256: "0".repeat(64), + filename: "wet-and-dry-vacuum.pdf", + kind: "manual", + state: "ready", + pageCount: 68, + createdAt: "", + updatedAt: "", +}; + +const root = document.getElementById("root"); +if (!root) throw new Error("no #root"); +createRoot(root).render( + +
+ undefined} + /> +
+ , +); diff --git a/web/folio-browser-check.vite.ts b/web/folio-browser-check.vite.ts new file mode 100644 index 0000000..e403556 --- /dev/null +++ b/web/folio-browser-check.vite.ts @@ -0,0 +1,28 @@ +/** + * Build for folio-browser-check.tsx: the real reader, a real conversion, one page a + * browser can open. Not part of `npm run build`; see the header of the .tsx. + */ +import { readFileSync } from "node:fs"; + +import tailwindcss from "@tailwindcss/vite"; +import react from "@vitejs/plugin-react"; +import { defineConfig } from "vite"; + +const json = process.env.FOLIO_CHECK_JSON; +if (!json) throw new Error("set FOLIO_CHECK_JSON to a conversion response"); + +export default defineConfig({ + plugins: [react(), tailwindcss()], + // Relative, so the built page opens over file:// without a server. + base: "./", + define: { + __CONVERSION__: readFileSync(json, "utf8"), + __DROP_OFFSET__: process.env.FOLIO_CHECK_DROP_OFFSET === "1", + __FORCE_OFFSET__: process.env.FOLIO_CHECK_OFFSET ?? "null", + }, + build: { + outDir: process.env.FOLIO_CHECK_OUT ?? ".folio-check", + emptyOutDir: true, + rollupOptions: { input: "folio-browser-check.html" }, + }, +}); diff --git a/web/gate-pages-test.ts b/web/gate-pages-test.ts new file mode 100644 index 0000000..2c2b467 --- /dev/null +++ b/web/gate-pages-test.ts @@ -0,0 +1,47 @@ +/** + * The page-range wording the gate's second scope uses. + * + * Sits beside reader-flow-test.ts and for its reasons: `node --test` strips the + * types and runs this directly, and the rule under test is in src/ where it is + * typechecked with the rest of the app. + */ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { pageRanges } from "./src/screens/gate-pages.ts"; + +test("the sequential manual's neutral set reads as a range and a stray", () => { + // Measured: pages 1-6 are the cover, three contents pages and the two diagram + // plates; 560 is the colophon. Seven numbers in a row would be unreadable, and + // "1-560" would be a lie about a 560-page document. + assert.equal(pageRanges([1, 2, 3, 4, 5, 6, 560]), "1–6 and 560"); +}); + +test("the columns manual's two adjacent pages are one range", () => { + assert.equal(pageRanges([67, 68]), "67–68"); +}); + +test("a single page is just the number", () => { + assert.equal(pageRanges([5]), "5"); +}); + +test("separate pages are listed with an 'and' before the last", () => { + assert.equal(pageRanges([1, 3, 5]), "1, 3 and 5"); +}); + +test("a long tail of runs is capped rather than printed in full", () => { + // Eight separate pages is eight runs; six are named and the rest counted, because + // a card is not the place for forty numbers. + assert.equal( + pageRanges([1, 3, 5, 7, 9, 11, 13, 15]), + "1, 3, 5, 7, 9, 11 and 2 more", + ); +}); + +test("the thousands separator is applied, so page 1000 is not '1000'", () => { + assert.equal(pageRanges([1000]), (1000).toLocaleString()); +}); + +test("no pages is an empty string, not the word undefined", () => { + assert.equal(pageRanges([]), ""); +}); diff --git a/web/package.json b/web/package.json index 81b6b87..0eeb441 100644 --- a/web/package.json +++ b/web/package.json @@ -6,6 +6,7 @@ "dev": "vite", "build": "tsc -b && vite build", "typecheck": "tsc --noEmit", + "test": "node --test --disable-warning=ExperimentalWarning \"*-test.ts\"", "preview": "vite preview", "format": "prettier --write \"src/**/*.{ts,tsx,css}\"", "generate:api": "echo \"types are hand-written in src/api/types.ts; see docs/api/openapi.yaml\"" diff --git a/web/reader-check.tsx b/web/reader-check.tsx new file mode 100644 index 0000000..6f31f78 --- /dev/null +++ b/web/reader-check.tsx @@ -0,0 +1,172 @@ +/** + * Render the reader to HTML and read what comes out. + * + * The screen is behind a session, so this is how it is looked at: hand the real screen + * a real document's conversion JSON, render it with react-dom/server, and print the + * text and structure a person should see — or feed the `--html` output to a headless + * browser with the compiled stylesheet, which is how the search screen was checked. + * It caught three things a green typecheck did not — a list marker printed twice, a + * table's columns mirrored the wrong way, and a figure landing after the paragraph + * that introduces it. + * + * Usage, from web/: + * npx vite build --ssr reader-check.tsx --outDir .reader-check --logLevel error + * node .reader-check/reader-check.js [--html] [--pages 2,14,57] + * + * The JSON is whatever `GET /api/v1/documents/{id}/conversion?lang=de` returned. It + * is not committed: it is a real manual's text. + */ +import { renderToStaticMarkup } from "react-dom/server"; + +import type { Conversion, Doc } from "./src/api/types"; +import { readingOrder } from "./src/screens/reader-flow"; +import { Reader, ReaderPages } from "./src/screens/Reader"; + +const args = process.argv.slice(2); +const path = args.find((a) => !a.startsWith("--")); +if (!path) { + console.error("usage: node reader-check.js [--html] [--pages 2,14]"); + process.exit(2); +} +const only = (() => { + const i = args.indexOf("--pages"); + if (i < 0) return null; + const list = args[i + 1]; + if (!list) return null; + return new Set(list.split(",").map((n) => Number(n))); +})(); + +const fs = await import("node:fs"); +const conversion = JSON.parse(fs.readFileSync(path, "utf8")) as Conversion; + +if (args.includes("--shell")) { + // The screen around the document: the way back, the title, the language chips and + // the state a document that is not ready shows instead of content. Effects do not + // run in a server render, so this is the first paint, before the fetch returns. + const doc = { + id: "doc_example", + deviceId: "dev_example", + blobSha256: "0".repeat(64), + filename: "wet-and-dry-vacuum.pdf", + kind: "manual", + state: "ready", + pageCount: 68, + createdAt: "", + updatedAt: "", + } satisfies Doc; + console.log( + renderToStaticMarkup( + undefined} + />, + ) + .replace(/>\n<") + .replace(/ class="[^"]*"/g, ""), + ); + process.exit(0); +} + +const started = performance.now(); +const pages = readingOrder(conversion.blocks, conversion.figures); +const ordered = performance.now(); +const shown = only ? pages.filter((p) => only.has(p.page)) : pages; +const html = renderToStaticMarkup(); +const rendered = performance.now(); + +console.error( + [ + `${conversion.blocks.length} blocks and ${conversion.figures.length} figures`, + `lang=${JSON.stringify(conversion.lang)} state=${conversion.state}`, + `${pages.length} pages, ${shown.length} rendered`, + `readingOrder ${(ordered - started).toFixed(1)} ms`, + `renderToStaticMarkup ${(rendered - ordered).toFixed(1)} ms`, + `${html.length.toLocaleString()} bytes of HTML`, + `${(html.match(/<[a-z]/g) ?? []).length.toLocaleString()} elements`, + `${(html.match(/]*)>/g; + let cursor = 0; + let depth = 0; + const stack: string[] = []; + let pending: string[] = []; + + const flush = () => { + const text = pending.join("").replace(/\s+/g, " ").trim(); + pending = []; + if (!text) return; + const owner = stack[stack.length - 1] ?? ""; + lines.push(`${" ".repeat(depth)}${label(owner)}${decode(text)}`); + }; + + for (let m = tag.exec(markup); m; m = tag.exec(markup)) { + pending.push(markup.slice(cursor, m.index)); + cursor = m.index + m[0].length; + const [, closing, name, attrs = ""] = m; + if (name === "img") { + flush(); + const alt = /alt="([^"]*)"/.exec(attrs)?.[1] ?? ""; + const width = /width="(\d+)"/.exec(attrs)?.[1] ?? "?"; + const height = /height="(\d+)"/.exec(attrs)?.[1] ?? "?"; + lines.push(`${" ".repeat(depth)}[image ${width}x${height}] ${decode(alt)}`); + continue; + } + if (closing) { + flush(); + stack.pop(); + depth = Math.max(0, depth - 1); + continue; + } + flush(); + const dir = /dir="(rtl|ltr)"/.exec(attrs)?.[1]; + const span = /colspan="(\d+)"/i.exec(attrs)?.[1]; + // A container that holds no text of its own would otherwise be invisible, and + // two tables printed side by side arriving as one is exactly the mistake this + // check exists to catch. + if (name === "table" || name === "tr" || name === "ul" || name === "figure") { + lines.push(`${" ".repeat(depth)}<${name}${dir === "rtl" ? " dir=rtl" : ""}>`); + } + stack.push(name + (dir === "rtl" ? " rtl" : "") + (span ? ` span=${span}` : "")); + depth++; + } + pending.push(markup.slice(cursor)); + flush(); + return lines.join("\n"); +} + +function label(owner: string): string { + if (!owner) return ""; + const [name, ...rest] = owner.split(" "); + const extra = rest.length > 0 ? ` ${rest.join(" ")}` : ""; + return `${(name ?? "").padEnd(5)}${extra ? extra.padEnd(8) : " "} `; +} + +function decode(s: string): string { + return s + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/’/g, "’"); +} diff --git a/web/reader-flow-test.ts b/web/reader-flow-test.ts new file mode 100644 index 0000000..8240dda --- /dev/null +++ b/web/reader-flow-test.ts @@ -0,0 +1,341 @@ +/** + * The rules in reader-flow that are worth asserting rather than only looking at. + * + * Run from web/ with `npm test`, which is `node --test`. Node runs TypeScript directly + * by stripping the types, so this needs no test framework, no transform and no new + * dependency -- which is why it exists at all: this project had no JavaScript test + * runner, and adding one to assert a rule this size would have cost more than the + * rule. + * + * It sits beside reader-check.tsx rather than under src/, which is where this repo + * already keeps the code that runs in Node: tsconfig.json includes only `src` and + * `vite.config.ts`, so a file here is outside the browser app typecheck and can + * import `node:test` and a `.ts` path without pulling `@types/node` into the + * application scope and without relaxing `allowImportingTsExtensions` for every + * file. The rule under test is in src and is typechecked; a wrong call from here + * fails as an assertion. + */ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { contentsTarget, placeOnPage, type Slot } from "./src/screens/reader-flow.ts"; + +// The columns manual's German conversion: 68 pages exist, and this language holds +// some of them. The gaps are real -- the pages between are other languages'. +const german = new Set([1, 2, 3, 12, 14, 15, 20, 23, 52, 57, 68]); + +test("a contents entry whose target this language holds becomes a link", () => { + // The columns manual: offset 0, and page 14 is one of its German pages. + assert.equal(contentsTarget("14", 0, german), 14); +}); + +test("the offset is added, and offset zero is a real answer", () => { + // The sequential manual's offset is 6: printed 9 is PDF page 15. + assert.equal(contentsTarget("9", 6, new Set([15])), 15); + // The same printed number with the columns manual's offset is a different page, + // which is what makes 0 an answer rather than the absence of one. + assert.equal(contentsTarget("9", 0, new Set([9, 15])), 9); +}); + +test("a range entry links to its first page", () => { + // "Сухая уборка ... 15 – 23" goes to 15, at offset 0. The en dash is the one the + // splitEntry pattern admits, so both dashes are checked. + assert.equal(contentsTarget("15 – 23", 0, german), 15); + assert.equal(contentsTarget("15 - 23", 0, german), 15); + // And the end of the range is not the destination even when it would be servable. + assert.equal(contentsTarget("15 – 23", 0, new Set([15, 23])), 15); +}); + +test("no offset was served, so nothing is a link", () => { + // The folios did not agree on one. Every entry stays plain text -- and this must + // not be read as offset 0, which is why the parameter is optional rather than + // defaulted. + assert.equal(contentsTarget("14", undefined, german), null); +}); + +test("a target outside the document is not a link", () => { + // Printed 9999 with offset 0 is no page of anything. + assert.equal(contentsTarget("9999", 0, german), null); + // And a printed number the offset drags below page 1. + assert.equal(contentsTarget("3", -10, german), null); +}); + +test("a target this language's conversion does not hold is not a link", () => { + // The one that fires on a real document: the columns manual prints five + // languages' contents pages, so a German entry can name a page that is entirely + // Russian. Page 13 exists in the PDF and is not in the German conversion. + assert.ok(!german.has(13)); + assert.equal(contentsTarget("13", 0, german), null); +}); + +test("an entry with no page number is not a link", () => { + // splitEntry returns "" where the line had no leader or nothing after it. + assert.equal(contentsTarget("", 0, german), null); +}); + +// --------------------------------------------------------------------------- +// Where a picture goes: placeOnPage. +// +// Every box below is copied from a real conversion response, page and all, because +// the whole rule is a reading of real geometry and made-up coordinates would only +// test the arithmetic. The text is cut to a few words; nothing else is changed. +// --------------------------------------------------------------------------- + +let seq = 0; +/** A block at a measured box. `kind` and `note` matter only where a test uses them. */ +function block( + page: number, + x0: number, + x1: number, + y0: number, + y1: number, + text: string, + extra: { kind?: string; note?: string; lang?: string } = {}, +) { + return { + page, + regionX0: 0, + index: seq++, + kind: (extra.kind ?? "paragraph") as never, + text, + lang: extra.lang ?? "de", + x0, + x1, + y0, + y1, + lines: 1, + chars: text.length, + ...(extra.note === undefined ? {} : { note: extra.note }), + }; +} + +/** A figure at a measured box. */ +function figure(page: number, index: number, x0: number, x1: number, y0: number, y1: number) { + return { + page, + index, + x0, + y0, + x1, + y1, + ink: 100, + textFraction: 0, + dpi: 216, + pixelWidth: Math.round((x1 - x0) * 2), + pixelHeight: Math.round((y1 - y0) * 2), + sha256: `${index}`.padStart(64, "0"), + }; +} + +/** Every flow under a slot, in the order a reader meets them. */ +function flowsUnder(slots: Slot[]): string[] { + return slots.flatMap((slot) => + slot.kind === "flows" + ? slot.flows.map((flow) => + flow.kind === "figure" + ? `figure#${flow.figure.index}` + : flow.kind === "paragraph" || flow.kind === "heading" + ? flow.block.text + : flow.kind === "table" + ? `table(${flow.rows.length} rows)` + : flow.kind === "list" + ? `list(${flow.items.length})` + : `contents(${flow.entries.length})`, + ) + : slot.columns.flatMap((c) => flowsUnder(c.slots)), + ); +} + +test("a drawing printed beside its text is set beside it, and beside the whole run", () => { + // Page 42 of the columns manual, its first printed row: one drawing in the left rail + // and TWO paragraphs to the right of it. The unit matters — the paper is not putting + // the picture next to a paragraph, it is putting it next to a step, and 44 of the + // 49 such rows across that manual hold 2 or more blocks. + const slots = placeOnPage( + [ + block(42, 323, 584, 61, 111, "Sofern sich noch Schmutz"), + block(42, 323, 581, 114, 164, "Bei starker Verschmutzung"), + ], + [figure(42, 0, 43, 288, 65, 241)], + ); + assert.equal(slots.length, 1); + const slot = slots[0]; + assert.equal(slot?.kind, "beside"); + if (slot?.kind !== "beside") return; + assert.equal(slot.strip, false); + assert.deepEqual(flowsUnder(slot.columns[0]?.slots ?? []), ["figure#0"]); + assert.deepEqual(flowsUnder(slot.columns[1]?.slots ?? []), [ + "Sofern sich noch Schmutz", + "Bei starker Verschmutzung", + ]); +}); + +test("drawings printed in a row become a row, not a pile", () => { + // Page 533 of the sequential manual: two drawings of one operation, printed side by + // side under one sentence. Ordering by `y0` alone put them one above the other, which + // is the complaint this rule exists for -- and it also got them backwards, because + // the right-hand drawing starts higher up the page than the left-hand one. + const slots = placeOnPage( + [], + [figure(533, 0, 306, 395, 151, 318), figure(533, 2, 84, 206, 194, 295)], + ); + assert.equal(slots.length, 1); + const slot = slots[0]; + assert.equal(slot?.kind, "beside"); + if (slot?.kind !== "beside") return; + // A strip, so it keeps its row at any width rather than stacking. + assert.equal(slot.strip, true); + // Printed order, left to right: the one at x84 is read first. + assert.deepEqual(flowsUnder(slots), ["figure#2", "figure#0"]); +}); + +test("two printed columns are not interleaved by height", () => { + // Page 529 of the sequential manual. Left column x67-448, right x479-854, gutter 31 + // units wide. By `y0` the right column's step 3 came out ABOVE the left column's + // heading, which is the "merged text" half of the report. + const slots = placeOnPage( + [ + block(529, 67, 247, 98, 112, "Основание промывочной панели", { lang: "ru" }), + block(529, 67, 435, 117, 155, "Базовая станция будет", { lang: "ru" }), + block(529, 480, 839, 93, 107, "3. Переверните промывочную", { lang: "ru" }), + block(529, 480, 816, 105, 119, "ролика и сам ролик", { lang: "ru" }), + ], + [], + ); + const slot = slots[0]; + assert.equal(slots.length, 1); + assert.equal(slot?.kind, "beside"); + if (slot?.kind !== "beside") return; + assert.deepEqual(flowsUnder(slot.columns[0]?.slots ?? []), [ + "Основание промывочной панели", + "Базовая станция будет", + ]); + assert.deepEqual(flowsUnder(slot.columns[1]?.slots ?? []), [ + "3. Переверните промывочную", + "ролика и сам ролик", + ]); +}); + +test("a ruled table is never cut, however its cells are placed", () => { + // Page 57 of the columns manual. Its cells sit in two vertical groups with a clear + // 15-to-38 unit gap between them, so cutting on geometry shattered the table into + // its 25 cells and `tables` could no longer assemble any of them. conversion.md + // already measured why the boxes cannot be trusted for this: a cell's box is its + // TEXT's extent, not the ruled cell's. + const cell = (x0: number, x1: number, y0: number, y1: number, row: number, col: number) => + block(57, x0, x1, y0, y1, `r${row}c${col}`, { + kind: "table", + note: `row ${row} of 7, column ${col} of 2 of a ruled table`, + }); + const slots = placeOnPage( + [ + cell(36, 164, 130, 163, 2, 1), + cell(180, 407, 130, 196, 2, 2), + cell(36, 137, 241, 258, 4, 1), + cell(179, 424, 241, 388, 4, 2), + ], + [], + ); + // One run, no cut, and the four cells reach the reader as one two-row grid. + assert.deepEqual( + slots.map((s) => s.kind), + ["flows"], + ); + assert.deepEqual(flowsUnder(slots), ["table(2 rows)"]); +}); + +test("below the top level, prose is not set beside prose", () => { + // Page 52 of the columns manual. Its own gutter is at x585-604, and INSIDE the left + // column two runs of German sit 22 units apart at x305/x327 -- wide enough to look + // like a gutter and not one: "Parkettreinigungsdüse" is a caption within the same + // measure, not a column beside "Die nachfolgenden Unterkapitel". Without the guard + // the page came apart into stacks of fragments; with it the page is cut once. + const slots = placeOnPage( + [ + block(52, 43, 305, 62, 127, "Die nachfolgenden Unterkapitel"), + block(52, 327, 577, 66, 137, "Parkettreinigungsdüse + Micro"), + block(52, 43, 98, 137, 154, "Trockensaugen"), + block(52, 604, 866, 62, 484, "Sollten auf den gereinigten"), + ], + [], + ); + assert.equal(slots.length, 1); + const slot = slots[0]; + assert.equal(slot?.kind, "beside"); + if (slot?.kind !== "beside") return; + // The left column is ONE run: the 22-unit gap inside it was not cut. + assert.deepEqual( + slot.columns.map((c) => c.slots.map((s) => s.kind)), + [["flows"], ["flows"]], + ); + assert.deepEqual(flowsUnder(slot.columns[0]?.slots ?? []), [ + "Die nachfolgenden Unterkapitel", + "Parkettreinigungsdüse + Micro", + "Trockensaugen", + ]); +}); + +test("a page the paper did not divide is left as one run", () => { + // Page 521 of the sequential manual: a full-bleed callout diagram. Its labels reach + // across the drawings they annotate -- "ИК-камера ... Вентиляционное отверстие" runs + // x266-539, straight over the 469-484 gap that would otherwise read as a gutter -- + // so no empty vertical band survives and the page must come out exactly as it did + // before this rule existed. It is the page the report named first, and it is the one + // the rule has to leave alone. + const slots = placeOnPage( + [ + block(521, 66, 250, 43, 76, "Обзор изделия", { lang: "ru" }), + block(521, 266, 469, 158, 172, "Вспомогательная светодиодная подсветка", { lang: "ru" }), + block(521, 266, 539, 178, 228, "ИК-камера на основе ИИ Вентиляционное", { lang: "ru" }), + block(521, 754, 876, 116, 161, "кнопку в течение 3 секунд", { lang: "ru" }), + ], + [figure(521, 0, 484, 748, 96, 278), figure(521, 1, 66, 397, 117, 364)], + ); + assert.deepEqual( + slots.map((s) => s.kind), + ["flows"], + ); +}); + +test("a right-to-left page reads its first column on the right", () => { + // The DOM order is the reading order, and `dir` on the row does the laying out, so + // the rightmost column has to come FIRST in the markup. Hebrew boxes are in the same + // left-origin space as everything else -- the language decides the order, not the + // coordinates. + const slots = placeOnPage( + [ + block(1, 60, 300, 100, 140, "left on the page", { lang: "he" }), + block(1, 400, 640, 100, 140, "right on the page", { lang: "he" }), + ], + [], + ); + assert.deepEqual(flowsUnder(slots), ["right on the page", "left on the page"]); + // And the same geometry in a left-to-right language reads the other way round. + const ltr = placeOnPage( + [ + block(1, 60, 300, 100, 140, "left on the page", { lang: "de" }), + block(1, 400, 640, 100, 140, "right on the page", { lang: "de" }), + ], + [], + ); + assert.deepEqual(flowsUnder(ltr), ["left on the page", "right on the page"]); +}); + +test("a strip of drawings still knows which way the page reads", () => { + // The case that has no text to read a direction off. A strip holds only pictures, so + // asking its content which way it goes returns nothing -- and the first version of + // this did exactly that, defaulting to left-to-right and laying the columns out + // against the logical order it had just put them in. The direction is a fact about + // the page, so it travels on the slot. + const rtl = placeOnPage( + [block(1, 60, 640, 40, 60, "כותרת", { lang: "he" })], + [figure(1, 0, 60, 300, 100, 260), figure(1, 1, 400, 640, 100, 260)], + ); + const strip = rtl.find((s) => s.kind === "beside"); + assert.equal(strip?.kind, "beside"); + if (strip?.kind !== "beside") return; + assert.equal(strip.strip, true); + assert.equal(strip.rtl, true); + // Logical order: the picture at x400 is the right-hand one, so it is read first. + assert.deepEqual(flowsUnder([strip]), ["figure#1", "figure#0"]); +}); diff --git a/web/search-check.tsx b/web/search-check.tsx new file mode 100644 index 0000000..1f350f4 --- /dev/null +++ b/web/search-check.tsx @@ -0,0 +1,41 @@ +/** + * Render the search screen to HTML and look at it. + * + * The screen is behind a session, so the way to see it is to render it the way it + * first paints: hand SearchResultsView a real `GET /search` response and print the + * markup. Screenshot the result with a headless browser and the compiled stylesheet + * to see what a person sees. + * + * Usage, from web/: + * npx vite build --ssr search-check.tsx --outDir .search-check --logLevel error + * node .search-check/search-check.js [ …] + * + * The JSON is whatever `GET /api/v1/search?q=…` returned. It is not committed: it is + * a real manual's text. + */ +import { renderToStaticMarkup } from "react-dom/server"; + +import type { SearchResults } from "./src/api/types"; +import { SearchBox, SearchResultsView } from "./src/screens/Search"; + +const paths = process.argv.slice(2).filter((a) => !a.startsWith("--")); +if (paths.length === 0) { + console.error("usage: node search-check.js [ …]"); + process.exit(2); +} + +const fs = await import("node:fs"); + +const sections = paths.map((path) => { + const results = JSON.parse(fs.readFileSync(path, "utf8")) as SearchResults; + return renderToStaticMarkup( +
+
+ undefined} /> + undefined} opening={null} /> +
+
, + ); +}); + +console.log(sections.join('\n
\n')); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index ec47186..e083c8b 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -1,9 +1,18 @@ import type { + Conversion, + Device, + Doc, + DocumentKind, + Gate, Health, Instance, Job, JobEvent, JobState, + LanguageRun, + LanguageSource, + Location, + SearchResults, Session, SetupStatus, User, @@ -39,7 +48,11 @@ async function request(path: string, init?: RequestInit): Promise { response = await fetch(BASE + path, { ...init, headers: { - ...(init?.body ? { "Content-Type": "application/json" } : {}), + // FormData sets its own Content-Type with a multipart boundary; adding + // application/json here produces a body the server cannot parse. + ...(init?.body && !(init.body instanceof FormData) + ? { "Content-Type": "application/json" } + : {}), ...init?.headers, }, // Session auth is cookie-based, so the cookie must ride along. @@ -108,6 +121,101 @@ export const api = { }, cancelJob: (id: string) => request(`/jobs/${encodeURIComponent(id)}/cancel`, { method: "POST" }), + + locations: () => request<{ locations: Location[] }>("/locations"), + + createLocation: (name: string) => + request("/locations", { method: "POST", body: JSON.stringify({ name }) }), + + devices: () => request<{ devices: Device[] }>("/devices"), + + device: (id: string) => request(`/devices/${encodeURIComponent(id)}`), + + createDevice: (input: Partial) => + request("/devices", { method: "POST", body: JSON.stringify(input) }), + + deleteDevice: (id: string) => + request(`/devices/${encodeURIComponent(id)}`, { method: "DELETE" }), + + documents: (deviceId: string) => + request<{ documents: Doc[] }>(`/devices/${encodeURIComponent(deviceId)}/documents`), + + /** + * Uploads a file and returns the document plus the id of the queued probe. + * + * No Content-Type is set: the browser must add its own multipart boundary, and + * overriding it produces a body the server cannot parse. + */ + uploadDocument: (deviceId: string, file: File, kind: DocumentKind = "manual") => { + const form = new FormData(); + form.append("file", file); + form.append("kind", kind); + return request<{ document: Doc; duplicate: boolean; jobId?: string }>( + `/devices/${encodeURIComponent(deviceId)}/documents`, + { method: "POST", body: form }, + ); + }, + + documentGate: (id: string) => request(`/documents/${encodeURIComponent(id)}/gate`), + + documentLanguages: (id: string, source?: LanguageSource) => { + const query = source ? `?source=${encodeURIComponent(source)}` : ""; + return request<{ source: LanguageSource; runs: LanguageRun[] }>( + `/documents/${encodeURIComponent(id)}/languages${query}`, + ); + }, + + declineDocument: (id: string) => + request(`/documents/${encodeURIComponent(id)}/decline`, { method: "POST" }), + + /** + * Authorise the work the gate reported. + * + * There is no language argument and there must not be one: the languages are the + * household's configuration, which is what the gate rendered. The one thing a + * caller may say is whether to include the pages the gate offered as `neutral`, + * and that is a yes or no to a set the server computed — never a set of its own. + * See ingest.Service.Approve for the whole argument. + */ + approveDocument: (id: string, opts?: { includeNeutralPages?: boolean }) => + request<{ document: Doc; jobId?: string }>(`/documents/${encodeURIComponent(id)}/approve`, { + method: "POST", + body: JSON.stringify({ includeNeutralPages: opts?.includeNeutralPages === true }), + }), + + documentContentURL: (id: string) => `${BASE}/documents/${encodeURIComponent(id)}/content`, + + /** + * What the conversion produced. + * + * `lang` is passed through when it is a string, including the empty string: + * `?lang=` is a real question — the content nothing could name — and not the + * absence of a filter. Omitting the argument asks for everything stored. + */ + documentConversion: (id: string, lang?: string) => { + const query = lang === undefined ? "" : `?lang=${encodeURIComponent(lang)}`; + return request(`/documents/${encodeURIComponent(id)}/conversion${query}`); + }, + + /** + * Which manual says this, and where. + * + * The query is sent exactly as typed: the endpoint has no pattern language, so no + * character here needs escaping beyond the URL encoding URLSearchParams does. An + * empty or whitespace-only query is a 400 from the server rather than an empty + * result set, so callers must not send one — see SearchResults for the difference + * between "no manual says that" and "nothing has been converted yet". + */ + search: (q: string, options: { documentId?: string; limit?: number } = {}) => { + const params = new URLSearchParams({ q }); + if (options.documentId) params.set("documentId", options.documentId); + if (options.limit !== undefined) params.set("limit", String(options.limit)); + return request(`/search?${params}`); + }, + + /** The PNG a figure was rendered to. The digest is the name and the content. */ + documentFigureURL: (id: string, sha256: string) => + `${BASE}/documents/${encodeURIComponent(id)}/figures/${encodeURIComponent(sha256)}`, }; /** diff --git a/web/src/api/types.ts b/web/src/api/types.ts index ee5d033..deb0a75 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -99,3 +99,393 @@ export interface ApiErrorBody { message: string; }; } + +// --- M1: the registry and the document pipeline --- + +export interface Location { + id: string; + name: string; + parentId?: string; + notes?: string; + createdAt: string; + updatedAt: string; +} + +export interface Device { + id: string; + name: string; + brand?: string; + model?: string; + category?: string; + locationId?: string; + notes?: string; + /** Date only, as YYYY-MM-DD. */ + purchasedAt?: string; + createdAt: string; + updatedAt: string; +} + +export type DocumentKind = "manual" | "receipt" | "warranty" | "photo" | "other"; + +/** + * Where a document is in the pipeline. `awaiting_scope` is the gate: the document + * has been read for free and nothing further happens until the user decides. + */ +export type DocumentState = + | "uploaded" + | "probing" + | "awaiting_scope" + | "declined" + | "converting" + | "ready" + | "failed"; + +export interface Doc { + id: string; + deviceId: string; + blobSha256: string; + filename?: string; + mediaType?: string; + kind: DocumentKind; + state: DocumentState; + lastError?: string; + pageCount?: number; + encrypted?: boolean; + tagged?: boolean; + hasTextLayer?: boolean; + medianCharsPerPage?: number; + contentStartPage?: number; + contentEndPage?: number; + createdAt: string; + updatedAt: string; + probedAt?: string; +} + +/** + * Which signal established a language run. + * + * `repertoire` is which alphabet the text uses — the letters only some languages + * sharing a script can write, which is what separates Russian, Ukrainian and + * Kazakh in one document. The empty string is a real, reportable state and not a + * defect: a page of service addresses in six languages is genuinely unnameable, + * and saying so beats guessing. + */ +export type LanguageSource = + | "" + | "page-tag" + | "index" + | "script" + | "repertoire" + | "detector" + | "reconciled"; + +export interface LanguageRun { + source: LanguageSource; + /** The label the document itself prints, which may not be a valid tag: UA, CZ. */ + code: string; + /** The BCP-47 tag. */ + lang: string; + /** The English display name. */ + name: string; + title?: string; + start: number; + end: number; + pages: number; + printedPage?: number; + confidence: number; + /** The signals disagreed about this run. Shown, never silently resolved. */ + conflict: boolean; + note?: string; +} + +/** + * One of a document's languages as the gate reports it: everything a stored run + * carries, plus what only the region map can say about size. + * + * Characters lead and pages are context. A language occupying one of three + * parallel columns on 26 of 68 pages is not 26 pages of reading, and + * `sharesPages` is what says so. A language the per-page signals never named has + * no run behind it — a parallel-columns manual has none at all — and then `title` + * and `printedPage` are absent and `confidence` is 0, because regions store + * neither and inventing them would be an estimate. + */ +export interface GateLanguage extends LanguageRun { + /** Runes, not bytes: the same writing in Cyrillic or CJK runs about a third more bytes. */ + chars: number; + /** `chars` as a fraction of the document's named text, 0 to 1. */ + share: number; + /** + * This language does not have its pages to itself: somewhere it occupies a box + * on a page another language also occupies. + */ + sharesPages: boolean; +} + +/** + * The pre-flight question, answered before anything is spent. `cost.available` + * is false when there is no honest number to show rather than a guessed one. + */ +export interface Gate { + documentId: string; + deviceId: string; + filename?: string; + kind: DocumentKind; + state: DocumentState; + probed: boolean; + pages: number; + encrypted: boolean; + hasTextLayer: boolean; + medianChars: number; + /** + * The document's named text, and the denominator of every `share`. Text nothing + * could name is excluded, so a signal that failed cannot silently shrink a + * language's share. + */ + chars: number; + household: string[]; + inScope: GateLanguage[]; + other: GateLanguage[]; + /** + * Distinct pages carrying an in-scope language, not a sum over languages: on a + * parallel-columns manual a sum reports 133 pages of a 68-page document. + */ + scopePages: number; + scopeFraction: number; + scopeChars: number; + /** + * `scopeChars` over `chars`. The honest measure of how much of a document a + * household reads: on the measured columns manual German is 38% by pages and + * 20% by characters. + */ + scopeCharFraction: number; + /** + * How many runs the signals disagreed about — the document's own contents table + * against its pages. A region's disagreement, a column's alphabet against the + * page's printed tab, is a different thing and arrives as `conflict` on the + * language itself. + */ + conflicts: number; + /** + * Content pages carrying text that no signal could name. Front matter and a + * back cover are excluded: they legitimately belong to no section. + */ + unlabelledPages: number; + /** + * The pages that carry content and that no language owns — the second, opt-in + * scope. Absent when there are none, so the question the UI asks is "is there + * another scope?" rather than "is its page list empty?". + */ + neutral?: GateNeutral; + requiresApproval: boolean; + maxPagesAuto: number; + cost: { available: boolean; chars: number; reason?: string }; + summary: string; +} + +/** + * The pages belonging to no language, offered as an extra scope. + * + * They exist because the funnel converts the pages a household's languages occupy, + * so anything outside every language is never converted — usually a cover, and on + * the measured sequential manual an exploded parts diagram that 31 places in the + * document tell you to look at. + * + * `figures` leads over `chars` in the UI, and the reason is measured: the sequential + * manual's 7 pages hold 1,656 characters and 61 pictures, the columns manual's 2 + * hold 11,256 characters and none. Characters alone rank the two sets in exactly the + * wrong order. + */ +export interface GateNeutral { + /** The PDF pages on offer, ascending. */ + pages: number[]; + chars: number; + /** + * How many pictures they hold, and **absent when nobody counted** rather than 0 — + * 0 is the columns manual's real answer and must stay distinguishable from it. + */ + figures?: number; + /** Why the pictures were not counted, when they were not. */ + note?: string; + /** Already approved with these pages in scope. */ + included: boolean; +} + +// --- M1: what the conversion produced --- + +/** + * What a block is. `figure` is declared and never produced, deliberately: a + * figure's natural key would have to invent a region left edge or collide with a + * real block's, so pictures come back as their own list and a reader merges the two + * by page and vertical position. + */ +export type BlockKind = "heading" | "paragraph" | "list-item" | "table" | "figure"; + +/** + * One piece of readable content, in document order. + * + * `regionX0` is an integer because that is what is stored and what a block joins to + * a region on; the block's own box is unrounded, because nothing keys on it and a + * caller drawing it over a 108 dpi render wants what was measured. + */ +export interface Block { + page: number; + regionX0: number; + index: number; + kind: BlockKind; + /** The heading level, 1 for the most prominent. 0, and so absent, for anything else. */ + level?: number; + text: string; + /** The region's language, absent where none was established. */ + lang?: string; + /** `lang` for a person to read: "Ukrainian", not "uk". */ + name?: string; + x0: number; + x1: number; + y0: number; + y1: number; + lines: number; + chars: number; + note?: string; +} + +/** + * One illustration. + * + * There is no language field, and that is the contract rather than an omission: a + * picture belonging to no language belongs to every language. Asking for a language + * on the conversion endpoint returns that language's own pictures plus every + * neutral one. + * + * The pictures in a manual are not the images in the file — every illustration in + * both measured fixtures is vector — so `sha256` names a PNG rendered from the crop, + * fetchable at `/documents/{id}/figures/{sha256}`. + */ +export interface Figure { + page: number; + index: number; + x0: number; + y0: number; + x1: number; + y1: number; + /** How many drawn shapes the figure holds: the shape guard's evidence, kept. */ + ink: number; + /** How much of its area is covered by text: the text guard's evidence. */ + textFraction: number; + dpi: number; + pixelWidth: number; + pixelHeight: number; + /** The blob store's name for the PNG, which is also the PNG's digest. */ + sha256: string; + /** + * The figure's callout labels as printed text, in the printed reading order the + * server chose — absent for a picture nothing points at, which is most of them. + * + * THE PNG CONTAINS THESE. The crop is wide enough to hold the drawing and its + * labels exactly as the paper arranges them, so a client that renders only the + * image is already showing every leader ending in its word, and drawing these + * strings anywhere near the picture would print each label twice. + * + * They are carried because nothing can read pixels: this is the accessible copy of + * what the picture prints, for a screen reader now and for search and translation + * later. Each string is in logical order, so it is safe to put in the DOM with a + * `dir` when it is ever shown. + */ + labels?: string[]; +} + +/** + * A document's converted content. + * + * `state` is here because no count can distinguish "converted and empty" from "not + * converted": a document that has not been through the gate has no blocks, and that + * is not the claim that it has no content. + */ +export interface Conversion { + documentId: string; + state: DocumentState; + /** Present only when the request filtered to one language. `""` asks for the unnamed content. */ + lang?: string; + blocks: Block[]; + figures: Figure[]; + /** + * How far this document's PDF pages run ahead of the numbers printed on its + * paper: the PDF page for a printed page number is `printed + folioOffset`. It is + * one constant for the whole document. + * + * **Absent** where the stored folios do not agree on one, which is a different + * answer from zero and must not be collapsed into it: a manual whose page 1 is + * its cover really does have offset 0, and reading a missing field as 0 would + * turn every contents entry of a document with no mapping into a link to the + * wrong page. + */ + folioOffset?: number; + lastError?: string; +} + +/** + * Which path answered a search. + * + * `index` is the FTS5 trigram index, with bm25 ranking. `substring` is the scan that + * covers a query the index cannot represent: a trigram index holds no token shorter + * than three characters, so a query with any word shorter than that would otherwise + * match nothing at all — which matters in Chinese and Japanese, where two characters + * is an ordinary word. The scan does not rank. + */ +export type SearchMode = "index" | "substring"; + +/** + * One search hit: which manual, which page, which language, and enough text to + * recognise. + * + * `page`, `regionX0` and `index` are the block's natural key, the same citation + * `Block` carries, so a hit deep-links to the exact paragraph and still points there + * after a re-conversion. `filename` and `deviceName` are what a household recognises + * — "page 47 of something" answers nothing. + */ +export interface SearchHit { + documentId: string; + filename?: string; + deviceId: string; + deviceName: string; + /** The document's state, so a hit from a manual that is mid-re-conversion shows as such. */ + state: DocumentState; + page: number; + regionX0: number; + index: number; + kind: BlockKind; + level?: number; + lang?: string; + /** `lang` for a person to read: "Japanese", not "ja". */ + name?: string; + /** About 64 characters of the block around the match. */ + snippet: string; + /** The whole block's length in runes, so a snippet is distinguishable from a complete block. */ + chars: number; + /** FTS5's relevance, negative and lower-is-better. 0 in `substring` mode. */ + bm25: number; + /** + * What the results are ordered by: `bm25` minus 1.0 for a heading. + * + * The heading bonus is a judgement — a heading names a section, so it answers + * "where does it say this" better than a passing mention — and both numbers are + * reported so it can be argued with rather than merely trusted. + */ + score: number; +} + +/** + * The hits, plus what was actually asked and how it was answered. + * + * `indexed` appears **only** when nothing matched, and it is the difference between + * "no manual says that" and "nothing has been converted yet", which are the same + * empty list otherwise. + */ +export interface SearchResults { + query: string; + mode: SearchMode; + limit: number; + /** The limit cut the list off: these are the first hits rather than the hits. */ + truncated: boolean; + hits: SearchHit[]; + indexed?: number; +} diff --git a/web/src/screens/DeviceDetail.tsx b/web/src/screens/DeviceDetail.tsx new file mode 100644 index 0000000..c4d1ce3 --- /dev/null +++ b/web/src/screens/DeviceDetail.tsx @@ -0,0 +1,596 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import { api, ApiError, subscribeToJobs } from "../api/client"; +import type { Device, Doc, Gate, GateLanguage } from "../api/types"; +import { Alert, Button, Card } from "../ui"; +import { pageRanges } from "./gate-pages"; +import { readerLanguages, type ReaderLanguage } from "./Reader"; + +/** One device: what it is, and the manuals belonging to it. */ +export function DeviceDetail({ + device, + onBack, + onRead, +}: { + device: Device; + onBack: () => void; + /** Open the reader. The languages come from the gate, which is already loaded here. */ + onRead: (doc: Doc, languages: ReaderLanguage[]) => void; +}) { + const [documents, setDocuments] = useState(null); + const [error, setError] = useState(null); + + const reload = useCallback(async () => { + try { + const { documents } = await api.documents(device.id); + setDocuments(documents); + setError(null); + } catch (cause) { + setError(cause instanceof ApiError ? cause.message : "Could not load the documents."); + } + }, [device.id]); + + useEffect(() => { + void reload(); + // Probing runs in the background, so the page follows the job stream rather + // than polling: a finished probe should appear without a refresh. + return subscribeToJobs(() => void reload()); + }, [reload]); + + return ( +
+
+ +

{device.name}

+ {device.brand || device.model ? ( +

+ {[device.brand, device.model].filter(Boolean).join(" ")} +

+ ) : null} +
+ + + +
+

Documents

+ {error ? ( +
+ {error} +
+ ) : null} + + {documents === null ? ( + Loading… + ) : documents.length === 0 ? ( + + No documents yet. Upload the manual above. + + ) : ( +
    + {documents.map((document) => ( + + ))} +
+ )} +
+
+ ); +} + +function Upload({ deviceId, onUploaded }: { deviceId: string; onUploaded: () => void }) { + const input = useRef(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [note, setNote] = useState(null); + + async function send(file: File) { + setBusy(true); + setError(null); + setNote(null); + try { + const { duplicate } = await api.uploadDocument(deviceId, file); + setNote( + duplicate + ? "You already had this exact file, so nothing was duplicated." + : "Uploaded. Reading it now — this costs nothing.", + ); + onUploaded(); + } catch (cause) { + setError(cause instanceof ApiError ? cause.message : "The upload failed."); + } finally { + setBusy(false); + if (input.current) input.current.value = ""; + } + } + + return ( +
+

Add a manual

+

+ The file is stored unchanged and read locally to work out what it contains. Nothing is sent + anywhere and nothing is spent until you say so. +

+
+ { + const file = event.target.files?.[0]; + if (file) void send(file); + }} + className="block w-full cursor-pointer rounded-md border border-rule bg-paper-raised px-3 py-2 text-sm text-ink-soft file:mr-3 file:rounded file:border-0 file:bg-rule/60 file:px-3 file:py-1.5 file:text-sm file:text-ink" + /> +
+ {busy ?

Uploading…

: null} + {note ?

{note}

: null} + {error ? ( +
+ {error} +
+ ) : null} +
+ ); +} + +const stateLabels: Record = { + uploaded: "queued to read", + probing: "reading…", + awaiting_scope: "waiting for you", + declined: "not processed", + converting: "converting…", + ready: "ready", + failed: "failed", +}; + +function DocumentCard({ + document, + onChanged, + onRead, +}: { + document: Doc; + onChanged: () => void; + onRead: (doc: Doc, languages: ReaderLanguage[]) => void; +}) { + const [gate, setGate] = useState(null); + + // Which languages the reader may ask for — readerLanguages says why the gate's + // in-scope list is the right one. It costs no extra request here because the gate is + // already loaded for the card itself. + const languages: ReaderLanguage[] = gate ? readerLanguages(gate) : []; + + useEffect(() => { + if (!document.probedAt) { + setGate(null); + return; + } + api + .documentGate(document.id) + .then(setGate) + .catch(() => undefined); + }, [document.id, document.probedAt, document.state]); + + return ( +
  • + +
    + + {document.filename || "Untitled document"} + + + {stateLabels[document.state]} + +
    + {/* Only a converted document has a reader. `converting` gets one too, + because the reader is where the progress belongs once you have asked + for it — and it fills in by itself when the job finishes. */} + {document.state === "ready" || document.state === "converting" ? ( + + ) : null} + + Original + +
    +
    + + {document.lastError ? ( +

    {document.lastError}

    + ) : null} + + {gate ? : null} +     +
    • + ); +} + +/** + * The pre-flight gate. It states what the document is, what would be processed, + * and what that costs, before anything is spent — and it lists the languages that + * are *not* being processed, because the original is kept whole and importing one + * later must be a button rather than a re-upload. + */ +function GatePanel({ gate, onChanged }: { gate: Gate; onChanged: () => void }) { + const [busy, setBusy] = useState(false); + const [showOther, setShowOther] = useState(false); + const [error, setError] = useState(null); + // The extra scope starts off, and it starts off even when it holds the pictures + // this exists for. Defaulting it on would convert pages the user never chose, + // which is the opposite of a gate. + // + // A document already approved with them shows them ticked, so reloading the screen + // reports what was chosen rather than an unticked box over converted pages. + const [withNeutral, setWithNeutral] = useState(gate.neutral?.included === true); + + const inScope = bySize(gate.inScope); + const other = bySize(gate.other); + + async function decline() { + setBusy(true); + setError(null); + try { + await api.declineDocument(gate.documentId); + onChanged(); + } catch (cause) { + setError(cause instanceof ApiError ? cause.message : "That could not be recorded."); + } finally { + setBusy(false); + } + } + + async function approve() { + setBusy(true); + setError(null); + try { + await api.approveDocument(gate.documentId, { includeNeutralPages: withNeutral }); + onChanged(); + } catch (cause) { + setError(cause instanceof ApiError ? cause.message : "The import could not be started."); + } finally { + setBusy(false); + } + } + + return ( +
    +

    {gate.summary}

    + +
    + + {/* Characters lead and pages are context: this row used to read "52 (76%)" + by pages for a document of which the household reads 40% of the text. */} + + + +
    + + {inScope.length > 0 ? ( +
    +

    Would be processed

    +
      + {inScope.map((run) => ( + + ))} +
    +
    + ) : null} + + {other.length > 0 ? ( +
    +

    Also present: {nameList(other, 6)}

    + + {showOther ? ( +
      + {other.map((run) => ( + + ))} +
    + ) : null} +

    + These are kept in the original and can be imported later without re-uploading. +

    +
    + ) : null} + + {gate.neutral ? ( + + ) : null} + + {gate.unlabelledPages > 0 ? ( +

    + {count(gate.unlabelledPages)}{" "} + {gate.unlabelledPages === 1 ? "page carries" : "pages carry"} text that no signal could + name, so {gate.unlabelledPages === 1 ? "it counts" : "they count"} towards none of the + languages above. +

    + ) : null} + + {gate.conflicts > 0 ? ( +

    + {count(gate.conflicts)} {gate.conflicts === 1 ? "section" : "sections"} where the + document’s own contents table disagrees with the pages themselves. Shown rather than + guessed at. +

    + ) : null} + +
    + {/* It counts characters because "Import 52 pages" is the misleading unit the + rest of this panel stopped using: cost.chars is the same measured quantity + as scopeChars, carried on the struct a caller asks about spending. + + The label names the extra pages when they are ticked, because the button is + the last thing read before the work starts and it must describe the scope + that is about to run rather than the smaller one. */} + + {gate.state !== "declined" ? ( + + ) : null} + + {gate.cost.available ? null : gate.cost.reason} + +
    + + {/* Next to the action, not at the top of the card. */} + {error ? ( +
    + {error} +
    + ) : null} +
    + ); +} + +/** + * The second scope: the pages that carry content and that no language owns. + * + * WHY THIS IS WORDED AROUND PICTURES. The gate's other rows lead with characters, + * and this one must not, because for this particular set the character count points + * the wrong way — measured, the sequential manual's 7 unowned pages hold 1,656 + * characters and 61 pictures while the columns manual's 2 hold 11,256 characters and + * none. Leading with characters here would invite the user to decline the diagram + * plates and accept a page of service addresses. + * + * So the pictures lead when they were counted, the characters sit underneath as + * context, and when nobody counted them the panel says so rather than showing a zero + * that would read as "no pictures". + * + * The pages are listed rather than merely counted, and the original is one click + * away, because the decision this asks for is one only a person looking at the paper + * can make. + */ +function NeutralOffer({ + neutral, + documentId, + checked, + disabled, + onChange, +}: { + neutral: NonNullable; + documentId: string; + checked: boolean; + disabled: boolean; + onChange: (v: boolean) => void; +}) { + const pages = neutral.pages.length; + const hasFigures = neutral.figures !== undefined && neutral.figures > 0; + + return ( +
    + +
    + ); +} + +function Stat({ label, value, note }: { label: string; value: string; note?: string }) { + return ( +
    +
    {label}
    +
    {value}
    + {note ?
    {note}
    : null} +
    + ); +} + +/** + * One of the document's languages, measured in characters first. + * + * The old row read "26 pp · 2–62", which on a manual laid out in parallel columns + * says "26 pages of German" where German is a fifth of each of those 26 pages. So + * the characters and the share lead, the pages are a locator underneath, and + * sharesPages decides which sentence describes them. + */ +function LanguageRow({ run }: { run: GateLanguage }) { + // A language below 1% of the text is not a peer of one at 20%, and the manuals + // measured here have one: 289 characters of Finnish read out of a single table + // cell. It is shown rather than filtered — that would need the threshold + // regions.md refused — but shown at a precision, and in an emphasis, that says + // what it is. + const negligible = run.share > 0 && run.share < 0.01; + return ( +
  • +
    + {run.name} + {run.code} + {run.conflict ? ( + + disputed + + ) : null} + + {count(run.chars)} chars · {percentOfText(run.share)} of the text + +
    +

    + {/* The title is the section name the manual prints in its own contents + table. Only the printed index can supply it, and only some documents + have one. */} + {run.title ? “{run.title}” · : null} + {placement(run)} +

    +
    • + ); +} + +/** + * Where a language sits, worded by whether it owns its pages. + * + * The two layouts must not read the same. A sequential manual's section has its + * pages to itself and a span is the whole truth about it; a column on a parallel + * manual's page shares that page with four other languages, and "26 pages" without + * that said is the misreading this screen existed to cause. + */ +function placement(run: GateLanguage): string { + if (run.pages === 0 || run.start === 0) return "no pages could be placed"; + if (run.sharesPages) { + // One shared page can be named, and "1 page, sharing each" is not English. + if (run.pages === 1) return `appears on page ${count(run.start)}, shared with other languages`; + return `appears on ${count(run.pages)} pages, sharing each with other languages`; + } + if (run.start === run.end) return `page ${count(run.start)}, all its own`; + return `pages ${count(run.start)}–${count(run.end)}, all its own`; +} + +/** Biggest first: a list headlined by size but ordered by page number invites the + * 289-character language to be read as one of the real ones. */ +function bySize(langs: GateLanguage[]): GateLanguage[] { + return [...langs].sort((a, b) => b.chars - a.chars || a.name.localeCompare(b.name)); +} + +/** Names up to limit languages; a 34-language manual gets a count for the rest. */ +function nameList(langs: GateLanguage[], limit: number): string { + const names = langs.slice(0, limit).map((l) => l.name); + const rest = langs.length - names.length; + return rest > 0 ? `${names.join(", ")}, and ${count(rest)} more` : names.join(", "); +} + +/** 47641 → "47,641", in the reader's own locale. */ +function count(n: number): string { + return n.toLocaleString(); +} + +/** + * A share of the document's text as a percentage. Below 1% it keeps a decimal, + * because 289 characters of 240,622 rounding to "0%" reads as a bug, and rounding + * up to "1%" would overstate it by an order of magnitude. + */ +function percentOfText(share: number): string { + const pct = 100 * share; + if (pct > 0 && pct < 0.05) return "under 0.1%"; + const digits = pct > 0 && pct < 1 ? 1 : 0; + return `${pct.toLocaleString(undefined, { + minimumFractionDigits: digits, + maximumFractionDigits: digits, + })}%`; +} diff --git a/web/src/screens/Devices.tsx b/web/src/screens/Devices.tsx new file mode 100644 index 0000000..3bca5cb --- /dev/null +++ b/web/src/screens/Devices.tsx @@ -0,0 +1,140 @@ +import { useCallback, useEffect, useState } from "react"; + +import { api, ApiError } from "../api/client"; +import type { Device } from "../api/types"; +import { Alert, Button, Card, Field } from "../ui"; + +/** The inventory: everything the household owns, and the way in to each one. */ +export function Devices({ onOpen }: { onOpen: (device: Device) => void }) { + const [devices, setDevices] = useState(null); + const [error, setError] = useState(null); + const [adding, setAdding] = useState(false); + + const reload = useCallback(async () => { + try { + const { devices } = await api.devices(); + setDevices(devices); + setError(null); + } catch (cause) { + setError(cause instanceof ApiError ? cause.message : "Could not load your devices."); + } + }, []); + + useEffect(() => { + void reload(); + }, [reload]); + + return ( +
    +
    +

    Devices

    + {!adding ? ( + + ) : null} +
    + + {error ?
    + {error} +
    : null} + + {adding ? ( + setAdding(false)} + onAdded={(device) => { + setAdding(false); + void reload(); + onOpen(device); + }} + /> + ) : null} + + {devices === null ? ( + Loading… + ) : devices.length === 0 ? ( + + Nothing yet. Add a device, then upload its manual. + + ) : ( +
      + {devices.map((device) => ( +
    • + + + +
      • + ))} +
    + )} +
    + ); +} + +function AddDevice({ + onAdded, + onCancel, +}: { + onAdded: (device: Device) => void; + onCancel: () => void; +}) { + const [name, setName] = useState(""); + const [brand, setBrand] = useState(""); + const [model, setModel] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + async function submit(event: React.FormEvent) { + event.preventDefault(); + setBusy(true); + setError(null); + try { + onAdded(await api.createDevice({ name: name.trim(), brand: brand.trim(), model: model.trim() })); + } catch (cause) { + setError(cause instanceof ApiError ? cause.message : "Could not add the device."); + } finally { + setBusy(false); + } + } + + return ( + +
    + setName(e.target.value)} + placeholder="Dishwasher" + autoFocus + required + /> +
    + setBrand(e.target.value)} placeholder="Bosch" /> + setModel(e.target.value)} placeholder="SMS4HVW33E" /> +
    + {/* Serial number and price are deliberately absent: they are encrypted + fields and the keyring is not wired into the schema yet. */} + {error ? {error} : null} +
    + + +
    + +
    + ); +} diff --git a/web/src/screens/Home.tsx b/web/src/screens/Home.tsx index 6cce8fe..ad4f069 100644 --- a/web/src/screens/Home.tsx +++ b/web/src/screens/Home.tsx @@ -1,13 +1,50 @@ import { useCallback, useEffect, useState } from "react"; import { api, ApiError, subscribeToJobs } from "../api/client"; -import type { Instance, Job, JobState, User } from "../api/types"; -import { Button, Card, Wordmark } from "../ui"; +import type { Device, Doc, Instance, Job, JobState, SearchHit, User } from "../api/types"; +import { Alert, Button, Card, Wordmark } from "../ui"; +import { DeviceDetail } from "./DeviceDetail"; +import { Devices } from "./Devices"; +import { Reader, readerLanguages, type ReaderLanguage } from "./Reader"; +import { SearchBox, SearchHits } from "./Search"; + +/** + * What the reader is showing, and what it goes back to. + * + * `backTo` is carried rather than derived, because the reader is now reached two ways: + * from a device, and from a search hit that may belong to a device that is not open. + * A back link reading "← Wet and dry vacuum" that returned to a list of search results + * would be a lie about where it goes. + */ +interface Reading { + doc: Doc; + languages: ReaderLanguage[]; + backTo: string; + startLang?: string | undefined; + startPage?: number | undefined; +} export function Home({ user, onSignedOut }: { user: User; onSignedOut: () => void }) { const [instance, setInstance] = useState(null); const [jobs, setJobs] = useState([]); const [streamLive, setStreamLive] = useState(false); + // Navigation is a single piece of state rather than a router: there are two + // screens, and a dependency to move between them would not earn its place yet. + const [openDevice, setOpenDevice] = useState(null); + // The reader is the third, and it is deliberately held here rather than inside + // DeviceDetail: it takes the whole page, including the space the activity list + // occupies, and a screen cannot hide its own parent's sections. Closing it falls + // back to the device that is still open underneath. + const [reading, setReading] = useState(null); + // A submitted query, which is the fourth screen. It sits above the device list + // rather than replacing it in the same slot, because search spans the household: + // "which manual says X" is asked by someone who does not know which device to open, + // so it cannot live inside one. + const [query, setQuery] = useState(""); + // Following a hit needs the document and its languages, neither of which is in the + // hit, so it is two requests and can fail while the user waits. + const [opening, setOpening] = useState(null); + const [openError, setOpenError] = useState(null); const reloadJobs = useCallback(async () => { try { @@ -41,6 +78,42 @@ export function Home({ user, onSignedOut }: { user: User; onSignedOut: () => voi }; }, [reloadJobs]); + /** + * Take a hit to the page it names. + * + * A hit carries the ids but not the document itself, and the reader needs the + * document for its title and page count and the gate for the languages it may ask + * for — the same list DeviceDetail hands it. Both are fetched here rather than + * joined into the search response, which docs/design/search.md keeps deliberately + * flat. + */ + async function openHit(hit: SearchHit) { + setOpening(hit.documentId); + setOpenError(null); + try { + const [{ documents }, gate] = await Promise.all([ + api.documents(hit.deviceId), + api.documentGate(hit.documentId), + ]); + const doc = documents.find((candidate) => candidate.id === hit.documentId); + if (!doc) { + setOpenError("That manual is no longer there. Search again to see what is."); + return; + } + setReading({ + doc, + languages: readerLanguages(gate), + backTo: `Results for “${query}”`, + startLang: hit.lang, + startPage: hit.page, + }); + } catch (cause) { + setOpenError(cause instanceof ApiError ? cause.message : "Could not open that manual."); + } finally { + setOpening(null); + } + } + async function signOut() { try { await api.logout(); @@ -66,41 +139,65 @@ export function Home({ user, onSignedOut }: { user: User; onSignedOut: () => voi
    -
    -

    Milestone 0

    -

    - Authentication, the database, the blob store, and the background job queue are working. - Adding devices and manuals arrives with the next milestone — this page exists to prove - the stack end to end, including live job progress over a server-sent event stream. -

    -
    - - {instance ? : null} - -
    -
    -

    Activity

    - - setReading(null)} + /> + ) : ( + <> +
    + + {openError ? {openError} : null} + {query ? : null} +
    + + {/* A query takes over the page. The library is still one click away — the + box empties — and leaving the device list under 25 hits would bury it + and the activity list both. */} + {query ? null : openDevice ? ( + setOpenDevice(null)} + onRead={(doc, languages) => setReading({ doc, languages, backTo: openDevice.name })} /> - {streamLive ? "live" : "reconnecting"} -     -
    + ) : ( + <> + + {instance ? : null} + + )} + +
    +
    +

    Activity

    + + + {streamLive ? "live" : "reconnecting"} + +
    - {jobs.length === 0 ? ( - - No background jobs yet. - - ) : ( -
      - {jobs.map((job) => ( - - ))} -
    - )} -
    + {jobs.length === 0 ? ( + + No background jobs yet. + + ) : ( +
      + {jobs.map((job) => ( + + ))} +
    + )} +
    + + )}
    ); diff --git a/web/src/screens/Reader.tsx b/web/src/screens/Reader.tsx new file mode 100644 index 0000000..7de8767 --- /dev/null +++ b/web/src/screens/Reader.tsx @@ -0,0 +1,721 @@ +import { useCallback, useEffect, useState } from "react"; + +import { api, ApiError, subscribeToJobs } from "../api/client"; +import type { Block, Conversion, Doc, Figure, Gate } from "../api/types"; +import { Alert, Card } from "../ui"; +import { + contentsTarget, + dirOf, + readingOrder, + type Flow, + type ReaderPage, + type Slot, +} from "./reader-flow"; + +/** One of the languages this document was converted for. */ +export interface ReaderLanguage { + lang: string; + name: string; +} + +/** + * The languages a reader may ask for, biggest first. + * + * The gate's in-scope list is exactly what approving converted — approve takes no + * language argument for that reason — so it is the right list wherever the reader is + * opened from. Biggest first so the reader opens on the language most of the document + * is in rather than on whichever sorts first. + */ +export function readerLanguages(gate: Gate): ReaderLanguage[] { + return [...gate.inScope] + .sort((a, b) => b.chars - a.chars || a.name.localeCompare(b.name)) + .map((run) => ({ lang: run.lang, name: run.name })); +} + +/** + * Reading a converted manual. + * + * # Direction + * + * This is the one screen in the app that handles right to left, and + * docs/design/conversion.md records why it is the exception: for a new screen the + * cost is nil, every block already carries its own language, and rewriting it later + * would not be free. So every inline offset here is logical — `ms`, `me`, `ps`, + * `pe`, `text-start` — and `dir` comes from the block rather than from the app. There + * is no exception: nothing on this screen is placed at a physical offset. A figure's + * callout labels used to be, and they are not placed at all any more — the crop the + * server renders already contains them, arranged exactly as the paper arranges them, + * so the picture carries its own direction inside it. + * + * The defect this screen used to have to apologise for is fixed underneath it: the + * stored text of a right-to-left language was in *visual* order and rendered + * mirrored, and internal/doc now puts it back into the order it is written in. What + * is left is Arabic letter shaping, which no part of this pipeline can do anything + * about, and it is stated where a reader will meet it — see [ShapingWarning]. + */ +export function Reader({ + doc, + backTo, + languages, + startLang, + startPage, + onBack, +}: { + doc: Doc; + /** What going back returns to, named: a device, or the results that led here. */ + backTo: string; + /** Empty asks for everything stored, which is already only what was charged for. */ + languages: ReaderLanguage[]; + /** + * Which language to open in, when something already knows. A search hit does: the + * matching text is in one language, and opening on the biggest one instead would + * show a page that does not contain what was searched for. + */ + startLang?: string | undefined; + /** Which page to open on. A page of the original, not an index into the pages shown. */ + startPage?: number | undefined; + onBack: () => void; +}) { + const first = languages[0]; + const [lang, setLang] = useState( + startLang ?? (first ? first.lang : undefined), + ); + const [conversion, setConversion] = useState(null); + const [error, setError] = useState(null); + // Which page the reader is currently opened at. Seeded from startPage and then + // owned here, because following a contents entry is the same act as following a + // search hit and has to move the same marker; the prop only says where to begin. + // + // The effect keeps the two in step if the prop ever changes under a mounted + // reader. Today it cannot -- Home unmounts this screen on the way back to the + // results -- so without it nothing would break yet; it is here because a state + // seeded from a prop and never resynchronised is a bug waiting for the first + // caller that keeps the screen mounted, and that caller would see the marker + // silently ignore where it was told to go. + const [openedPage, setOpenedPage] = useState(startPage); + useEffect(() => { + setOpenedPage(startPage); + }, [startPage]); + + const load = useCallback(async () => { + try { + setConversion(await api.documentConversion(doc.id, lang)); + setError(null); + } catch (cause) { + setError(cause instanceof ApiError ? cause.message : "Could not load the converted manual."); + } + }, [doc.id, lang]); + + useEffect(() => { + void load(); + }, [load]); + + // A conversion in flight finishes in the background, so the reader follows the + // same job stream the rest of the app does instead of polling. Refetching on any + // event is what Home does, and for the same reason: the server's answer is the + // truth, an event only says to go and ask again. + const settled = conversion !== null && conversion.state !== "converting"; + useEffect(() => { + if (settled) return; + return subscribeToJobs(() => void load()); + }, [settled, load]); + + const pages = conversion ? readingOrder(conversion.blocks, conversion.figures) : []; + const shown = languages.find((l) => l.lang === lang); + + // What a contents entry needs to become a link. `pages` is what THIS language's + // conversion actually holds, which is the check that matters: the columns manual + // prints five languages' contents, so a German entry can name a page that is + // entirely Russian. + const jump: ContentsJump = { + folioOffset: conversion?.folioOffset, + pages: new Set(pages.map((page) => page.page)), + onJump: setOpenedPage, + }; + + return ( +
    +
    + +

    + {doc.filename || "Untitled document"} +

    +

    {summary(doc, conversion, pages, shown)}

    +
    + + {languages.length > 1 ? ( +
    + {languages.map((option) => { + const active = option.lang === lang; + return ( + + ); + })} +
    + ) : null} + + {error ? {error} : null} + + {conversion === null ? ( + error ? null : ( + Loading… + ) + ) : conversion.state === "ready" ? ( + pages.length === 0 ? ( + + Nothing was converted for {shown ? shown.name : "this language"}. The original is kept + whole, so another language can be imported without re-uploading. + + ) : ( + <> + {languages.some((l) => l.lang === lang && isUnshaped(l.lang)) ? ( + + ) : null} + {openedPage !== undefined && !pages.some((page) => page.page === openedPage) ? ( + // Following a hit lands on a page in the hit's own language. Switching + // language afterwards can leave that page behind entirely, and a reader + // who scrolled nowhere deserves to know why rather than assume a bug. +

    + Page {openedPage} has nothing in {shown ? shown.name : "this language"}. +

    + ) : null} + + + ) + ) : ( + + )} +
    + ); +} + +/** What is being read, in one line under the title. */ +function summary( + doc: Doc, + conversion: Conversion | null, + pages: ReaderPage[], + shown: ReaderLanguage | undefined, +): string { + const parts: string[] = []; + if (shown) parts.push(shown.name); + if (conversion && conversion.state === "ready" && pages.length > 0) { + const first = pages[0] as ReaderPage; + const last = pages[pages.length - 1] as ReaderPage; + const span = + first.page === last.page ? `page ${first.page}` : `pages ${first.page}–${last.page}`; + parts.push( + `${pages.length.toLocaleString()} of ${(doc.pageCount ?? 0).toLocaleString()} pages, ${span} of the original`, + ); + } + return parts.join(" · "); +} + +/** + * The one thing this screen cannot render correctly, said out loud. + * + * `pdftohtml -xml` returns a right-to-left line in visual order — the Hebrew heading + * of the sequential manual comes back as the exact reverse of its logical string, + * confirmed codepoint by codepoint — while `pdftotext` on the same page returns it + * logically, wrapped in bidi controls. So the letters of every stored Hebrew and + * Arabic block are in the wrong order before this screen ever sees them, and no + * `dir` can undo that: the bidi algorithm reorders a strong run whichever base + * direction it is given. + * + * Reversing the string here was rejected. It would silently mangle the Latin words + * and digits real manuals mix in, and it would double-reverse the day the pipeline is + * fixed. The fix belongs where the text is read. + */ +/** + * What is still wrong with an Arabic-script language, now that the order is right. + * + * This used to say the text below reads backwards, which was true and is not any + * more: internal/doc puts a right-to-left line back into the order it is written in, + * and the verifier that measured the defect went from 8,120 reversed words to none. + * Saying so anyway would be worse than saying nothing. + * + * What is left is real but narrower and belongs to Arabic alone. The letters arrive + * in their isolated forms rather than joined — `السالمة` where the page prints + * `السلامة` — because the document's font maps its glyphs that way, and pdftotext, + * which is checked against for everything else here, reads them identically. It is + * not an ordering fault and nothing in the pipeline can join them. + * + * Hebrew gets no warning now, because there is nothing left to warn a reader about. + */ +function ShapingWarning({ name }: { name: string }) { + return ( +
    + {name} is written right to left, and it reads in the right order here. Its letters, though, + arrive one by one instead of joined up, because that is how this document’s font maps + them — a second, independent reader of the same file sees exactly the same thing. The words + are right; the letter shapes are not. +
    + ); +} + +/** + * Whether this language's letters are known to arrive unshaped. + * + * The Arabic script, not every right-to-left one: Hebrew does not join its letters, + * so it has nothing to lose this way. + */ +function isUnshaped(lang: string): boolean { + const base = (lang || "").split("-")[0]?.toLowerCase() ?? ""; + return ["ar", "fa", "ur", "ps", "sd", "ug", "ku"].includes(base); +} + +/** What is happening to a document that has no reader yet. */ +function Progress({ conversion }: { conversion: Conversion }) { + if (conversion.state === "failed") { + return ( + {conversion.lastError || "The conversion failed and no reason was recorded."} + ); + } + const messages: Record = { + uploaded: "This document is queued to be read. Nothing has been converted yet.", + probing: "Reading the document to find out what is in it. This costs nothing.", + awaiting_scope: + "Nothing has been converted yet. The gate on the device page is waiting for you to say what to import.", + declined: "This document was not processed, so there is nothing to read.", + converting: "Converting the languages you asked for. This page will fill in when it finishes.", + }; + return ( + + {messages[conversion.state] ?? "There is nothing to read yet."} + + ); +} + +/** + * The converted document itself, separated from the screen around it. + * + * Separate because this is the part worth rendering without a fetch: hand it a real + * document's blocks and read the HTML that comes out, or point a headless browser at + * that HTML and look at the page. It takes only data, which is what makes both + * possible — the screen around it is behind a session. + * + * (An earlier version of this comment said there is no browser automation on this + * machine. There is: Chrome is installed, screenshots headlessly with `--screenshot`, + * and can be driven over the DevTools protocol with no dependency at all.) + */ +export function ReaderPages({ + pages, + documentId, + startPage, + jump, +}: { + pages: ReaderPage[]; + documentId: string; + /** The page to open on, marked and scrolled to. */ + startPage?: number | undefined; + /** Absent: contents entries print their page number as plain text. */ + jump?: ContentsJump | undefined; +}) { + return ( +
    + {pages.map((page) => ( + + ))} +
    + ); +} + +/** + * What a printed contents entry needs before its page number can be a link. + * + * Carried as one object through three levels rather than three props, and passed + * rather than put in a context, so that rendering [ReaderPages] on its own -- which + * is how this screen is looked at, see the note above it -- can turn linking on and + * off explicitly instead of inheriting whatever a provider happened to hold. + */ +export interface ContentsJump { + /** The document's one offset. Absent where the folios agreed on none. */ + folioOffset?: number | undefined; + /** The pages this language's conversion holds, which is what a target must be in. */ + pages: ReadonlySet; + onJump: (page: number) => void; +} + +/** + * One page of the original: a marker, then everything printed on it. + * + * `opened` is the page a search hit sent the reader to. It scrolls itself into view + * through a callback ref rather than an effect looking the element up by id, so the + * scroll happens exactly when that page's element exists — the conversion arrives + * asynchronously, and an effect keyed on anything else would run before it. Figures + * carry their stored width and height, so nothing below reflows afterwards and the + * page does not drift back out of view. + */ +function PageView({ + page, + documentId, + opened, + jump, +}: { + page: ReaderPage; + documentId: string; + opened: boolean; + jump?: ContentsJump | undefined; +}) { + const scrollHere = useCallback((node: HTMLElement | null) => { + node?.scrollIntoView({ block: "start" }); + }, []); + + return ( +
    +
    + + page {page.page} + {opened ? " · opened here" : ""} + + +
    +
    + +
    +
    + ); +} + +/** + * A page's content, stacked or set beside itself as the paper set it. + * + * # What "beside" costs, and what happens when it cannot be paid + * + * Two printed columns need roughly twice the measure of one, and below about 40 + * characters a column of prose stops being readable. So a printed column stacks below + * `md` — in reading order, which is the DOM order [placeOnPage] already put it in, so + * the fallback is the same content read down the page and never a scramble. That is + * the deliberate answer to a narrow viewport: the picture goes back to sitting above + * or below its text, which is where the sequential manual prints it anyway. + * + * A strip of drawings has no breakpoint, because a drawing has no measure to lose: it + * keeps its row for as long as the row fits and wraps when it does not. Measured on + * page 533 in Chrome — at 1265 px the three strips are each one row, at 390 px each has + * wrapped to one drawing per line, and the page's own scrollWidth equals the viewport at + * both, so nothing is ever cut off sideways. + * + * # Direction + * + * `dir` is on the flex row rather than on each child, and the children are already in + * logical order, so a right-to-left document reads its first column on the right with + * no second rule and no physical property anywhere. Same reason every offset on this + * screen is `ms`/`me` rather than `ml`/`mr`. + * + * It comes off the slot rather than off the content under it. Reading it back from the + * first block found was the first version and it is wrong for the case that has no + * blocks at all: a strip of drawings would have defaulted to left-to-right and undone + * the logical ordering it was given. + */ +function SlotsView({ + slots, + documentId, + jump, +}: { + slots: Slot[]; + documentId: string; + jump?: ContentsJump | undefined; +}) { + return ( + <> + {slots.map((slot, i) => + slot.kind === "flows" ? ( + slot.flows.map((flow, j) => ( + + )) + ) : ( +
    + {slot.columns.map((column, j) => ( +
    + +
    + ))} +
    + ), + )} + + ); +} + +function FlowView({ + flow, + documentId, + jump, +}: { + flow: Flow; + documentId: string; + jump?: ContentsJump | undefined; +}) { + switch (flow.kind) { + case "heading": + return ; + + case "paragraph": + return ( +

    + {flow.block.text} +

    + ); + + case "list": + return ( +
      + {flow.items.map((item, i) => ( +
    • + {/* The document's own marker, kept rather than replaced: its numbers + restart and skip, and a CSS counter would renumber them silently. + An item whose marker could not be separated shows no marker at all + instead of an invented bullet — see splitMarker. */} + {item.marker ? ( + {item.marker} + ) : null} + {item.text} +
      • + ))} +
    + ); + + case "contents": + return ; + + case "table": + return ; + + case "figure": + return ; + } +} + +/** + * A printed table of contents, as the list of entries it is. + * + * It arrived as one run-together paragraph of dot leaders until internal/doc learned + * to give each printed line its own block — 17 entries on the columns manual's + * contents page, glued into one because consecutive entries sit at exactly the line + * pitch and the paragraph rule has nothing else to separate them by. + * + * The leader is drawn with a rule rather than with the document's own periods: a row + * of literal dots is noise to a screen reader, and the dots are still in the block's + * text where search and the coverage check can see them. + * + * The number the paper prints is what is shown, always, and it is what is read out: + * a reader who is holding the manual is looking for that number, and replacing it + * with the PDF's own would help nobody. Where the mapping exists the same number + * becomes a button that opens the PDF page it means -- see [contentsTarget] for the + * three cases that stay plain text, of which "this language's conversion does not + * hold that page" is the one that actually fires on a real document. + * + * A button rather than an anchor: there is no router and no URL for a page, so an + * `href` would either be a lie or a hash this app does not read back. What happens + * is a state change, which is what a button means. + */ +function ContentsView({ + flow, + jump, +}: { + flow: Extract; + jump?: ContentsJump | undefined; +}) { + return ( +
      + {flow.entries.map((entry, i) => { + const target = jump ? contentsTarget(entry.page, jump.folioOffset, jump.pages) : null; + return ( +
    • + {entry.title} + + {entry.page ? ( + target !== null && jump ? ( + + ) : ( + {entry.page} + ) + ) : null} +
      • + ); + })} +
    + ); +} + +/** + * A heading, at one of the two levels there are. + * + * conversion.md: the level is 1 or 2 and never more. Level 1 takes the display serif + * the rest of the app uses for headings; level 2 stays in the body face and is + * separated by weight, because a manual's subheading is usually a whole instruction + * ("Öffnen Sie den Gehäusedeckel.") and setting a sentence in a serif display size + * reads as prose that happens to be large. + */ +function Heading({ block, level }: { block: Block; level: number }) { + const dir = dirOf(block.lang); + if (level === 1) { + return ( +

    + {block.text} +

    + ); + } + return ( +

    + {block.text} +

    + ); +} + +/** + * A table, as the ruled grid it is printed as. + * + * A row whose single cell spans every column is the section label the page prints + * across the top of a group of rows — "Allgemein (alle Funktionen)" on page 57 — so it + * is marked up as a header for that group rather than as another data cell. + */ +function TableView({ flow }: { flow: Extract }) { + return ( +
    + + + {flow.rows.map((cells, r) => ( + + {cells.map((cell, c) => + cell.colSpan === flow.columns ? ( + + ) : ( + + ), + )} + + ))} + +
    + {cell.block ? cell.block.text : null} + + {cell.block ? cell.block.text : null} +
    +
    + ); +} + +/** + * One illustration, printed as the page prints it. + * + * The stored size is given as `width` and `height` so the page does not reflow as + * pictures arrive, and the image is capped at the measure rather than shown at its + * rendered 216 dpi size. `loading="lazy"` is what keeps a heavily illustrated section + * cheap: the sequential manual's Russian is 81 figures, and none of them is fetched + * until it is near the viewport. + * + * # Why the labels are not drawn here + * + * They are already in the picture. The crop internal/doc renders is wide enough to + * hold the drawing AND its callout labels in the arrangement the paper prints, so + * every leader ends in its word without this screen doing anything. The reader used + * to rebuild that arrangement from text and coordinates, and every defect it had came + * from rebuilding it: labels colliding with each other, a wrapped label's tail + * orphaned, one figure placed blind to its neighbour. Rebuilding an arrangement the + * source already has is the mistake, not the placement algorithm. + * + * So the labels are carried as text only, and they go in `alt` rather than into a + * visible `
    `: a caption would print every label a second time, right + * beside the copy the picture itself shows. `alt` reaches exactly the readers pixels + * do not. + */ +function FigureView({ figure, documentId }: { figure: Figure; documentId: string }) { + const labels = figure.labels ?? []; + const base = `Illustration printed on page ${figure.page} of the original`; + return ( +
    + {labels.length 0 ? `${base}. Labels: ${labels.join("; ")}` : base} + className="h-auto max-w-full rounded border border-rule" + /> +
    + ); +} diff --git a/web/src/screens/Search.tsx b/web/src/screens/Search.tsx new file mode 100644 index 0000000..a5ee2bf --- /dev/null +++ b/web/src/screens/Search.tsx @@ -0,0 +1,276 @@ +import { useCallback, useEffect, useState } from "react"; + +import { api, ApiError } from "../api/client"; +import type { SearchHit, SearchResults as Results } from "../api/types"; +import { Alert, Button, Card } from "../ui"; +import { dirOf } from "./reader-flow"; + +/** + * The search box, and the hits it produced. + * + * # Why a submit rather than a keystroke + * + * docs/design/search.md measures a query at 0.2 ms through the index and 1.9 ms + * through the scan, so searching on every keystroke would be affordable — but the + * short-query fallback makes it dishonest. Typing `Sau` towards `Saugkraft` passes + * through `S` and `Sa`, each of which the index cannot hold and each of which is + * answered by a different path with a different notice. A box that changed its own + * explanation twice per word would read as a bug. So the query is submitted, and the + * notice describes one query the user actually asked. + * + * # What has to be visible + * + * `mode` is not decoration: `substring` means the trigram index could not represent + * the query and a scan answered it instead, unranked and case-folding only ASCII. + * `truncated` means these are the first hits and not the hits. `indexed` separates + * "no manual says that" from "nothing has been converted yet". All three are stated + * rather than left to be inferred from a short list. + */ +export function SearchBox({ query, onSearch }: { query: string; onSearch: (q: string) => void }) { + const [draft, setDraft] = useState(query); + + return ( +
    { + event.preventDefault(); + onSearch(draft.trim()); + }} + className="flex items-end gap-2" + > + + +
    + ); +} + +/** The hits for one submitted query, or the reason there are none. */ +export function SearchHits({ + query, + onOpen, + opening, +}: { + query: string; + /** Open the reader on the page this hit is printed on, in this hit's language. */ + onOpen: (hit: SearchHit) => void; + /** The document being opened, so the hit that was clicked can say it is working. */ + opening: string | null; +}) { + const [results, setResults] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(false); + + const run = useCallback(async () => { + setLoading(true); + try { + setResults(await api.search(query)); + setError(null); + } catch (cause) { + setResults(null); + setError(cause instanceof ApiError ? cause.message : "The search could not be run."); + } finally { + setLoading(false); + } + }, [query]); + + useEffect(() => { + void run(); + }, [run]); + + if (loading && results === null && error === null) { + return

    Searching…

    ; + } + if (error) return {error}; + if (!results) return null; + + return ; +} + +/** + * One response, rendered — separated from the fetch for the same reason + * [ReaderPages] is: it takes only data, so it can be handed a real response and + * rendered without a browser or a server. The screenshots that checked this screen + * were taken that way. + */ +export function SearchResultsView({ + results, + onOpen, + opening, +}: { + results: Results; + onOpen: (hit: SearchHit) => void; + opening: string | null; +}) { + if (results.hits.length === 0) { + return ; + } + + return ( +
    + +
      + {results.hits.map((hit) => ( + + ))} +
    +
    + ); +} + +/** + * One hit: which manual, and where. + * + * The device's name leads, because that is what a household calls the thing — the + * filename is often a model number or a download's digest, and it is kept underneath + * for the case where one device has several manuals. Both come from the response; + * search.md specifies that every hit joins `documents` and `devices` for exactly this + * reason. + * + * `dir` is per hit and taken from the hit's own language, and every inline offset here + * is logical, so a Hebrew snippet needs no rework when extraction stops storing + * right-to-left text in visual order. That defect is upstream and this screen cannot + * repair it either; the reader states it where a reader meets it. + */ +function Hit({ + hit, + onOpen, + busy, +}: { + hit: SearchHit; + onOpen: (hit: SearchHit) => void; + busy: boolean; +}) { + return ( +
  • + + + +
    • + ); +} + +/** What kind of thing matched, in the words the reader will see it in. */ +function label(hit: SearchHit): string { + switch (hit.kind) { + case "list-item": + return "list"; + case "table": + return "table cell"; + default: + return hit.kind; + } +} + +/** + * How the query was answered, and whether the list is complete. + * + * Both notices are unconditional facts about this response rather than warnings, so + * they are set as quiet prose. `substring` gets the warn colour because it changes + * what the results mean: they are unranked, and case folding on that path is ASCII + * only, so a two-letter Cyrillic query is case-sensitive. + */ +function Notices({ results }: { results: Results }) { + return ( +
    +

    + {results.truncated ? "The first " : ""} + {results.hits.length.toLocaleString()} {results.hits.length === 1 ? "result" : "results"} + {results.mode === "index" ? ", best match first" : ""} +

    + + {results.mode === "substring" ? ( +

    + Part of “{results.query}” is shorter than three characters, which the index + cannot hold, so every stored block was scanned instead. These hits are in no particular + order, and outside the Latin alphabet the match is case-sensitive. +

    + ) : null} + + {results.truncated ? ( +

    + Cut off at {results.limit.toLocaleString()}: these are the first hits, not all of them. + Add a word to narrow the search. +

    + ) : null} +
    + ); +} + +/** + * Nothing matched — which is two different situations. + * + * `indexed` is in the response only when the hits are empty, and it is the number of + * blocks there were to search. Zero means nothing has been converted yet, which is + * not a search failure and has a different next step. + */ +function NothingFound({ results }: { results: Results }) { + if (results.indexed === 0) { + return ( + + No manual has been converted yet, so there is nothing to search. Upload one to a device and + approve what to import. + + ); + } + return ( + + No manual contains “{results.query}” + {results.indexed === undefined + ? "." + : `, across the ${results.indexed.toLocaleString()} passages that were searched.`}{" "} + Try a shorter word: a match is on any part of a word, so filter also finds{" "} + Luftfilter. + + ); +} diff --git a/web/src/screens/gate-pages.ts b/web/src/screens/gate-pages.ts new file mode 100644 index 0000000..056d597 --- /dev/null +++ b/web/src/screens/gate-pages.ts @@ -0,0 +1,38 @@ +/** + * Wording a set of page numbers the way a person would say it. + * + * Its own module rather than a helper inside DeviceDetail.tsx for reader-flow.ts's + * reason: it is a rule worth asserting, and a rule in a .tsx file cannot be imported + * by `node --test` without dragging React and JSX in with it. + */ + +/** 47641 → "47,641", in the reader's own locale. */ +function count(n: number): string { + return n.toLocaleString(); +} + +/** + * Page numbers as the ranges a person would say: `1–6 and 560`, not seven numbers. + * + * The neutral set is front matter plus a back cover on both measured manuals, which + * is one long run and one stray — exactly the shape that reads badly as a list and + * well as a range. Capped at six runs, because the set is bounded but not tiny and a + * card is not the place for forty numbers; the original is one click away. + * + * Input must be ascending, which is what the gate returns. + */ +export function pageRanges(pages: number[]): string { + const runs: Array<[number, number]> = []; + for (const page of pages) { + const last = runs[runs.length - 1]; + if (last && page === last[1] + 1) last[1] = page; + else runs.push([page, page]); + } + + const shown = runs.slice(0, 6).map(([a, b]) => (a === b ? count(a) : `${count(a)}–${count(b)}`)); + const rest = runs.length - shown.length; + if (rest > 0) shown.push(`${count(rest)} more`); + + const last = shown.pop() ?? ""; + return shown.length === 0 ? last : `${shown.join(", ")} and ${last}`; +} diff --git a/web/src/screens/reader-flow.ts b/web/src/screens/reader-flow.ts new file mode 100644 index 0000000..a1edc35 --- /dev/null +++ b/web/src/screens/reader-flow.ts @@ -0,0 +1,706 @@ +// Turning what the conversion stored into what a person reads. +// +// The API hands back two flat lists — blocks in reading order and figures — and +// neither is shaped like a page. This module does the shaping, separately from the +// JSX so it can be exercised on a real document's JSON without a browser. +// +// Three things are being reconstructed here, and each is reconstructed because the +// API deliberately does not carry it: +// +// - Where a picture belongs. docs/design/conversion.md is explicit that a figure +// is not a block and never will be, because a language-neutral figure has no +// region to key on. So a reader merges the two lists by page and by vertical +// position, which is what mergePage does. +// - Which cells make a table. Every table cell is its own block and its grid +// position travels in the prose `note` — blocks.go says so in as many words: +// "The grid position travels in the note rather than in a field". Reading it +// back is therefore intended, and it is the only source: `Block` has no row or +// column field. Measured against the alternative on five real conversions, +// grouping the same cells geometrically instead agrees with the note on only +// 23 of 38 cells of the columns manual and 0 of 112 of the Hebrew one, because +// a cell's stored box is its *text's* extent and not the ruled cell's. +// - Which list items make a list. Adjacency plus the marker the pipeline records. + +import type { Block, Figure } from "../api/types"; + +/** A cell of a reconstructed table. `null` is a cell the pipeline did not recover. */ +export interface TableCell { + block: Block | null; + colSpan: number; +} + +/** One flow of content, in reading order. A list and a table span several blocks. */ +export type Flow = + | { kind: "heading"; block: Block; level: number } + | { kind: "paragraph"; block: Block } + | { kind: "list"; items: Array<{ block: Block; marker: string; text: string }> } + | { kind: "contents"; entries: Array<{ block: Block; title: string; page: string }> } + | { kind: "table"; rows: TableCell[][]; columns: number; lang: string } + | { kind: "figure"; figure: Figure }; + +/** + * A piece of a page, placed the way the page places it. + * + * `flows` is a run of content read top to bottom, as before. `beside` is the part + * this type exists for: things the paper set next to each other, in logical reading + * order, so the first child is the one read first in this document's direction. + * + * `strip` distinguishes the two things "beside" turns out to mean on real paper, and + * it is a measured distinction rather than a tidy one — see [placeOnPage]. A strip is + * several drawings of one operation printed in a row; it stays in a row at any width, + * because a drawing has no measure to lose. Anything else is a printed column, and a + * column of prose has to stack when the viewport cannot hold two. + */ +export type Slot = + | { kind: "flows"; flows: Flow[] } + | { + kind: "beside"; + strip: boolean; + /** + * The page's direction, carried rather than inferred. + * + * The columns below are already in logical order, so whatever lays them out has + * to be told which way that is. Reading it back off the content does not work and + * the case is real: a strip of drawings holds no text at all, so a right-to-left + * page's pictures would have come out left to right — ordered logically in the + * markup and then laid out against that order. + */ + rtl: boolean; + columns: Column[]; + }; + +/** One of the things set beside another, with the share of the width the paper gave it. */ +export interface Column { + slots: Slot[]; + /** The printed width of this column, for dividing the measure as the paper did. */ + width: number; +} + +/** One page of the original, and everything printed on it. */ +export interface ReaderPage { + page: number; + slots: Slot[]; +} + +/** + * The `note` a table cell carries. Coupled to one `fmt.Sprintf` in + * internal/doc/blocks.go, which is the coupling the file comment explains; a cell + * whose note does not match is rendered as prose rather than silently dropped. + */ +const CELL_NOTE = + /^row (\d+) of (\d+), column (\d+) of (\d+) of a ruled table(?:, spanning (\d+) of them)?$/; + +/** The marker a list item opens with, as the pipeline recorded it. */ +const MARKER_NOTE = /^opens with the list marker "(.+)"$/; + +/** + * Scripts written right to left, by primary subtag. + * + * `iw` and `in` are the retired codes for Hebrew and Indonesian; only the first is + * relevant here, but a document tagged with it must still read correctly. + */ +const RTL_LANGS = new Set(["ar", "he", "iw", "fa", "ur", "yi", "ji", "ps", "sd", "ug", "dv", "ku"]); + +/** True when this language is written right to left. */ +export function isRTL(lang: string | undefined): boolean { + if (!lang) return false; + const primary = lang.toLowerCase().split(/[-_]/)[0] ?? ""; + return RTL_LANGS.has(primary); +} + +/** "rtl" or "ltr", for a `dir` attribute. */ +export function dirOf(lang: string | undefined): "rtl" | "ltr" { + return isRTL(lang) ? "rtl" : "ltr"; +} + +/** + * Everything the conversion returned, as pages of flows. + * + * Blocks are already in reading order within a page and pages already ascend, so + * nothing is re-sorted except the figures being spliced in. + */ +export function readingOrder(blocks: Block[], figures: Figure[]): ReaderPage[] { + const pages: number[] = []; + const blocksByPage = new Map(); + const figuresByPage = new Map(); + + for (const block of blocks) { + const list = blocksByPage.get(block.page); + if (list) list.push(block); + else { + blocksByPage.set(block.page, [block]); + pages.push(block.page); + } + } + for (const figure of figures) { + const list = figuresByPage.get(figure.page); + if (list) list.push(figure); + else { + figuresByPage.set(figure.page, [figure]); + // A page can hold pictures and no text of this language: the columns manual + // sets page 11 as a single full-page diagram. + if (!blocksByPage.has(figure.page)) pages.push(figure.page); + } + } + + pages.sort((a, b) => a - b); + return pages.map((page) => ({ + page, + slots: placeOnPage(blocksByPage.get(page) ?? [], figuresByPage.get(page) ?? []), + })); +} + +/** + * One page's content, arranged the way the page arranges it. + * + * # Why this exists + * + * The reader used to merge a page's blocks and figures into one column by vertical + * position alone, and on a two-column page that is a scramble. Measured on the + * sequential manual's Russian: 16 of its 22 pages are printed in two columns, and + * ordering their 65 figures and 431 blocks by `y0` produced 16 places where two or + * more pictures came out consecutively with the surrounding sentences pushed away + * from them. Page 533 is the clearest: three drawings in a row and then text, where + * the paper prints two drawings under one sentence in the left column and one under a + * different sentence in the right. + * + * # What the paper actually does, measured + * + * The two fixtures do two different things, and the same rule reproduces both: + * + * - The **columns manual** prints a rail of drawings down one side with the prose + * beside it. 49 of its 113 printed rows are one picture beside a run of text, and + * that run is a *group* rather than a paragraph: 1 block 5 times, 2 to 4 blocks 31 + * times, 5 or 6 blocks 13 times, opening with a paragraph 32 times, a heading 14 + * and a list item 3. So "beside a paragraph" would have been the wrong unit — what + * the drawing is beside is a step or a whole section. + * - The **sequential manual** never puts a drawing beside prose. It puts the drawing + * *under* the sentence, inside a column, and where an operation takes several + * drawings it prints them in a row: 13 pairs of figures overlap vertically, with + * no text between them in any of the 13. That row is what the old reader unrolled + * into a vertical pile, and it is the complaint. + * + * So neither "always beside" nor "always below" is right, and neither needed to be + * chosen: both are recoverable from boxes the payload already carries. + * + * # The rule + * + * A recursive cut of the page. Split into horizontal bands separated by a gap no item + * crosses; inside a band, split at a vertical gap no item crosses; recurse. A band + * with no vertical gap is a run of content and is merged with the run before it, so + * that a list or a table still reaches [group] as one uninterrupted sequence. + * + * Two guards, both of them things that went wrong first: + * + * - **A run of ruled-table cells is one atom.** Cut geometrically instead and page + * 57's troubleshooting table shatters into 25 separate cells, none of which + * [tables] can then assemble — its cells' boxes do not line up with the ruled grid, + * which conversion.md already measured at 23 of 38 cells agreeing. + * - **Below the top level, a cut needs a picture on one side of it.** The page's own + * column gutter is worth honouring; a narrower gap between two runs of prose is + * not, and letting it through turned page 52 into six stacks of fragments. With the + * guard the same page is two columns and nothing else. + */ +export function placeOnPage(blocks: Block[], figures: Figure[]): Slot[] { + return cut(atoms(blocks, figures), 0, isRTL(blocks.find((b) => b.lang)?.lang)); +} + +/** + * One placeable thing: a single block, a single figure, or a run of table cells that + * must stay together. + * + * `blocks` keeps the document order internal/doc read them in — the boxes place an + * atom, and are never allowed to reorder the blocks inside one, because [mergePage] + * and [group] both take that order as given. + * + * The box is the union, so a table is placed by where the whole grid sits rather than + * by any one of its cells. + */ +interface Atom { + blocks: Block[]; + figures: Figure[]; + /** Where this atom sat in document order, so a leaf can restore it. */ + seq: number; + x0: number; + x1: number; + y0: number; + y1: number; + /** True where the atom is only pictures: what a cut below the top level needs. */ + picture: boolean; +} + +/** A page's blocks and figures as atoms, with each run of ruled-table cells fused. */ +function atoms(blocks: Block[], figures: Figure[]): Atom[] { + const out: Atom[] = []; + let run: Block[] = []; + const flush = () => { + if (run.length > 0) out.push(atom(run, [], out.length)); + run = []; + }; + for (const block of blocks) { + if (block.kind === "table") { + run.push(block); + continue; + } + flush(); + out.push(atom([block], [], out.length)); + } + flush(); + for (const figure of figures) out.push(atom([], [figure], out.length)); + return out; +} + +function atom(blocks: Block[], figures: Figure[], seq: number): Atom { + const boxes = [...blocks, ...figures]; + return { + blocks, + figures, + seq, + x0: Math.min(...boxes.map((b) => b.x0)), + x1: Math.max(...boxes.map((b) => b.x1)), + y0: Math.min(...boxes.map((b) => b.y0)), + y1: Math.max(...boxes.map((b) => b.y1)), + picture: blocks.length === 0, + }; +} + +/** + * The narrowest gap that counts as a printed column gutter. + * + * Measured over both manuals: every empty vertical band with text on both sides is + * either 17 units or wider — a real gutter, on the columns manual's two-column safety + * list, its table page and its warranty page — or narrower than 4. Nothing lands in + * between, so this number is not carrying the decision; it is the middle of an empty + * range, recorded so that a document that does land there fails visibly rather than + * silently. + */ +const MIN_GUTTER = 12; + +/** How deep the cut goes. Three levels is columns, then a rail, then a strip. */ +const MAX_DEPTH = 3; + +/** Maximal runs of atoms with no horizontal gap between them: the page's rows. */ +function bands(atomList: Atom[]): Atom[][] { + const out: Atom[][] = []; + let current: Atom[] = []; + let bottom = -Infinity; + for (const a of [...atomList].sort((p, q) => p.y0 - q.y0)) { + if (current.length > 0 && a.y0 >= bottom) { + out.push(current); + current = []; + bottom = -Infinity; + } + current.push(a); + bottom = Math.max(bottom, a.y1); + } + if (current.length > 0) out.push(current); + return out; +} + +/** The widest empty vertical band with content on both sides, or null. */ +function gutter(row: Atom[]): [number, number] | null { + if (row.length < 2) return null; + const edges = [...new Set(row.flatMap((a) => [a.x0, a.x1]))].sort((p, q) => p - q); + let best: [number, number] | null = null; + for (let i = 0; i + 1 < edges.length; i++) { + const a = edges[i] as number; + const b = edges[i + 1] as number; + if (b - a < MIN_GUTTER) continue; + if (row.some((it) => it.x0 < b && it.x1 > a)) continue; + if (best === null || b - a > best[1] - best[0]) best = [a, b]; + } + if (best === null) return null; + const [a, b] = best; + if (!row.some((it) => it.x1 <= a) || !row.some((it) => it.x0 >= b)) return null; + return best; +} + +/** The page, or a part of it, as slots. */ +function cut(atomList: Atom[], depth: number, rtl: boolean): Slot[] { + const out: Slot[] = []; + let run: Atom[] = []; + const flush = () => { + if (run.length === 0) return; + out.push({ kind: "flows", flows: flowsOf(run) }); + run = []; + }; + + for (const row of bands(atomList)) { + let g = depth < MAX_DEPTH ? gutter(row) : null; + if (g !== null && depth > 0) { + // Below the top level only a picture earns a cut; see the note on placeOnPage. + const left = row.filter((a) => a.x1 <= (g as [number, number])[0]); + const right = row.filter((a) => a.x0 >= (g as [number, number])[1]); + if (!left.every((a) => a.picture) && !right.every((a) => a.picture)) g = null; + } + if (g === null) { + // Merged with the row before it: a list split across two rows is still one list. + run.push(...row); + continue; + } + flush(); + const [a, b] = g; + const sides = [row.filter((it) => it.x1 <= a), row.filter((it) => it.x0 >= b)]; + // The document's own direction decides which side is read first, and the DOM order + // is that order, so `dir` on the container lays it out without a second rule. + if (rtl) sides.reverse(); + const columns: Column[] = sides.map((side) => ({ + slots: cut(side, depth + 1, rtl), + width: Math.max(...side.map((it) => it.x1)) - Math.min(...side.map((it) => it.x0)), + })); + out.push({ + kind: "beside", + strip: row.every((it) => it.picture), + rtl, + columns, + }); + } + flush(); + return out; +} + +/** + * A run of atoms as flows, through the same merge and grouping as before. + * + * Document order is restored first. Placing shuffles atoms by geometry, and handing + * [mergePage] its blocks in any other order would break the one thing it is allowed to + * assume — that blocks already read correctly and only the figures need placing. + */ +function flowsOf(atomList: Atom[]): Flow[] { + const ordered = [...atomList].sort((p, q) => p.seq - q.seq); + return group( + mergePage( + ordered.flatMap((a) => a.blocks), + ordered.flatMap((a) => a.figures), + ), + ); +} + +type Item = { block: Block } | { figure: Figure }; + +/** + * One page's blocks and figures in the order they are printed down the page. + * + * Only the figures are placed, and they are placed by their top edge against each + * block's. Both lists are measured in the same space — a figure's box and a block's + * box both come back in the 892-unit space of the 108 dpi render, verified against + * the stored pixel sizes at 2.00 pixels per unit — so the comparison is direct and + * needs no scaling. + * + * A tie keeps the block first, so a picture and the paragraph introducing it stay in + * that order. + */ +export function mergePage(blocks: Block[], figures: Figure[]): Item[] { + const sorted = [...figures].sort((a, b) => a.y0 - b.y0 || a.index - b.index); + const out: Item[] = []; + let f = 0; + for (const block of blocks) { + while (f < sorted.length && (sorted[f] as Figure).y0 < block.y0) { + out.push({ figure: sorted[f] as Figure }); + f++; + } + out.push({ block }); + } + for (; f < sorted.length; f++) out.push({ figure: sorted[f] as Figure }); + return out; +} + +/** + * Runs of list items become one list, runs of table cells become tables. + * + * A run is not broken by a figure that happens to sit inside its vertical span, and + * that is measured rather than tidy-minded. Page 52 of the columns manual prints a + * nine-row table down the left of the measure and a photograph to its right, whose + * top edge falls between two of the rows: placing the picture strictly by height + * split one printed table into two, one of them a single stray cell reading + * "Fugendüse". The same happens to a two-line heading — page 14's "Trockensaugen mit + * der DryBOX / (Zyklon-Filtertechnologie)" arrives as two level-1 blocks 24 units + * apart, with a photograph's top edge one unit inside that gap. So a figure met while + * a run is being consumed is held and emitted directly after it, which moves a + * picture by at most the height of the thing it landed in. + */ +function group(items: Item[]): Flow[] { + const flows: Flow[] = []; + // Figures met while a run is being consumed, emitted when the run ends. + let held: Figure[] = []; + const release = () => { + for (const figure of held) flows.push({ kind: "figure", figure }); + held = []; + }; + + let i = 0; + while (i < items.length) { + const item = items[i] as Item; + if ("figure" in item) { + flows.push({ kind: "figure", figure: item.figure }); + i++; + continue; + } + const block = item.block; + + /** Consumes the rest of a run of blocks this predicate accepts, holding figures. */ + const run = (accepts: (b: Block) => boolean): Block[] => { + const out: Block[] = []; + const holding: Figure[] = []; + let j = i; + while (j < items.length) { + const next = items[j] as Item; + if ("figure" in next) { + holding.push(next.figure); + j++; + continue; + } + if (!accepts(next.block)) break; + // Only now are the figures passed over known to be inside the run rather + // than after its last block. + for (const figure of holding.splice(0)) held.push(figure); + out.push(next.block); + j++; + i = j; + } + return out; + }; + + // A contents entry is a list item whose note says it carries a dot leader, and + // internal/doc explains why it is not a kind of its own: BlockKind reaches a + // database column whose CHECK lists five kinds, and a sixth costs a rebuild of + // the table the search index is external-content over. + if (isContentsEntry(block)) { + const cells = run(isContentsEntry); + flows.push({ kind: "contents", entries: cells.map(splitEntry) }); + release(); + continue; + } + + if (block.kind === "list-item") { + // Contents entries are list items too and are taken above this, deliberately: + // asking about the kind first swallows them into an ordinary list, which is + // what happened the first time this was wired and what the leader note is for. + const cells = run((b) => b.kind === "list-item" && !isContentsEntry(b)); + flows.push({ kind: "list", items: cells.map(splitMarker) }); + release(); + continue; + } + + if (block.kind === "table") { + const cells = run((b) => b.kind === "table"); + for (const flow of tables(cells)) flows.push(flow); + release(); + continue; + } + + if (block.kind === "heading") { + // conversion.md: a heading is level 1 or 2 and never more, so this clamps + // rather than building a hierarchy that cannot arrive. + const level = block.level === 2 ? 2 : 1; + const heads = run((b) => b.kind === "heading" && (b.level === 2 ? 2 : 1) === level); + for (const head of heads) flows.push({ kind: "heading", block: head, level }); + release(); + continue; + } + + flows.push({ kind: "paragraph", block }); + i++; + } + return flows; +} + +/** + * The marker a list item opens with, and the item's text without it. + * + * The marker is only lifted out when whitespace follows it, and that guard is not + * theoretical. 12 of the columns manual's 113 list items read "*) modellabhängig", + * where the recorded marker is `*` and removing it leaves a stray ") ". All ten of + * the Hebrew section's items are worse: `-'א רויא` records the marker `-`, which is + * the *last* character of the figure reference "איור א'-" and only looks like a + * leading marker because the stored text is in visual order. Both keep their text + * whole and print no marker of their own — a marker is never invented, so an item + * whose marker cannot be lifted out simply shows the line as it is printed. + */ +export function splitMarker(block: Block): { block: Block; marker: string; text: string } { + const match = MARKER_NOTE.exec(block.note ?? ""); + const marker = match?.[1] ?? ""; + const text = block.text; + if (!marker || !text.startsWith(marker)) return { block, marker: "", text }; + const rest = text.slice(marker.length); + if (rest !== "" && !/^\s/.test(rest)) return { block, marker: "", text }; + return { block, marker, text: rest.trimStart() }; +} + +/** The note internal/doc writes on one entry of a printed table of contents. */ +const CONTENTS_NOTE = "a dot leader of "; + +/** Whether a block is one entry of a printed table of contents. */ +function isContentsEntry(block: Block): boolean { + return block.kind === "list-item" && (block.note ?? "").startsWith(CONTENTS_NOTE); +} + +/** + * A contents entry split into the title and the page it points at. + * + * The dot leader is the document's own typesetting and is dropped from the DOM + * rather than rendered: a row of literal periods read by a screen reader is noise, + * and the leader is drawn with a rule instead. The dots are still in the block's + * text, which is what search and the coverage check see, so nothing is lost — this + * is presentation only. + * + * A line that does not split — no leader of four or more, or nothing after it — + * keeps its whole text as the title and shows no page number, rather than guessing. + * internal/doc will not classify such a line as an entry in the first place, so this + * is the belt to that brace. + */ +export function splitEntry(block: Block): { block: Block; title: string; page: string } { + const match = /^(.*?)[\s.]*\.{4,}\s*([\d\s\u2013\u2014-]*\d)\s*$/.exec(block.text); + const title = match?.[1]; + const page = match?.[2]; + if (title === undefined || page === undefined) { + return { block, title: block.text.trim(), page: "" }; + } + return { block, title: title.trim(), page: page.replace(/\s+/g, " ").trim() }; +} + +/** + * The page of the PDF a contents entry points at, or null where it must not be a + * link at all. + * + * `printed` is what splitEntry pulled off the line -- the number the paper prints -- + * and `folioOffset` is the document's one constant, from the conversion response. + * The map is `pdf = printed + offset`; internal/registry's folio.go carries the + * measurement behind that being one constant per document and the rule for when + * there is no honest answer. + * + * A range entry links to its first page. "Сухая уборка ... 15 - 23" goes to 15, + * because that is where the section starts and where a reader following the entry + * expects to arrive; the end of the range is a fact about the section's length, not + * a second destination. + * + * Three things make it not a link, and each falls back to plain text rather than a + * link that goes nowhere: + * + * - No offset was served. The folios did not agree on one, so there is nothing to + * add. `folioOffset` is optional for exactly this reason and must never be + * defaulted to 0 -- the columns manual's real offset IS 0. + * - The line carries no page number, or none this can read. + * - The target is not a page this language's conversion holds. That is not a + * defensive check: the columns manual prints five languages' contents pages, so + * a German reader's entry can point at a page that is entirely Russian. The same + * check covers a target outside the document altogether -- `pages` holds only + * pages that exist and were converted, so page 0 and page 9999 fail it for the + * same reason and need no separate test. + */ +export function contentsTarget( + printed: string, + folioOffset: number | undefined, + pages: ReadonlySet, +): number | null { + if (folioOffset === undefined) return null; + // The first run of digits: a range entry links to where the section starts. + const first = /\d+/.exec(printed); + if (!first) return null; + const target = Number(first[0]) + folioOffset; + return pages.has(target) ? target : null; +} + +/** + * A run of adjacent table cells, as one flow per printed table. + * + * A new table starts where the row number stops advancing. That is what separates + * the two troubleshooting tables printed side by side on page 57 of the columns + * manual: the second opens with its row 1 directly after the first's row 7. Doing it + * by geometry instead would have to tell two tables apart by their cells' text + * boxes, which do not line up with the ruled columns at all. + */ +function tables(cells: Block[]): Flow[] { + const flows: Flow[] = []; + let current: Array<{ block: Block; row: number; col: number; span: number }> = []; + let columns = 0; + let prevRow = 0; + let prevCol = 0; + let lang = ""; + + const flush = () => { + if (current.length === 0) return; + flows.push({ kind: "table", rows: grid(current, columns, lang), columns, lang }); + current = []; + }; + + for (const block of cells) { + const match = CELL_NOTE.exec(block.note ?? ""); + if (!match) { + // Not a cell this reader can place. Shown as prose rather than dropped: the + // text is real and losing it silently is the one failure that must not happen. + flush(); + flows.push({ kind: "paragraph", block }); + prevRow = 0; + prevCol = 0; + continue; + } + const row = Number(match[1]); + const col = Number(match[3]); + const cols = Number(match[4]); + const span = match[5] ? Number(match[5]) : 1; + if ( + current.length > 0 && + (cols !== columns || row < prevRow || (row === prevRow && col <= prevCol)) + ) { + flush(); + } + columns = cols; + lang = block.lang ?? ""; + current.push({ block, row, col, span }); + prevRow = row; + prevCol = col; + } + flush(); + return flows; +} + +/** + * The cells of one table as rows of a rectangular grid. + * + * Rows the pipeline recovered no cell for are absent rather than blank: page 52 of + * the columns manual returns rows 1-6 and 9 of a nine-row table, and conversion.md + * records why — a vertically merged cell is dropped by the row walk. Printing two + * empty rows would be inventing evidence of something that is simply not there. + * + * A right-to-left table has its cells emitted in reverse column order, because + * column 1 of the ruled grid is the leftmost and under `dir="rtl"` the first cell in + * the markup is laid out on the right. Checked against the Hebrew section: the + * header row's column 3 is "part" and column 1 is "replacement period", which is the + * order the page prints them in when read from the right. + */ +function grid( + cells: Array<{ block: Block; row: number; col: number; span: number }>, + columns: number, + lang: string, +): TableCell[][] { + const byRow = new Map>(); + const order: number[] = []; + for (const cell of cells) { + const row = byRow.get(cell.row); + if (row) row.push(cell); + else { + byRow.set(cell.row, [cell]); + order.push(cell.row); + } + } + + const rtl = isRTL(lang); + return order.map((rowNumber) => { + const slots: TableCell[] = []; + const placed = byRow.get(rowNumber) ?? []; + let col = 1; + while (col <= columns) { + const cell = placed.find((c) => c.col === col); + if (cell) { + const span = Math.max(1, Math.min(cell.span, columns - col + 1)); + slots.push({ block: cell.block, colSpan: span }); + col += span; + } else { + slots.push({ block: null, colSpan: 1 }); + col++; + } + } + return rtl ? slots.reverse() : slots; + }); +}