Fix parsing bugs, improve tests, update deps and CI

.github/workflows/docd.yml

@@ -11,18 +11,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
       - id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: sajari/docd
           labels: |
@@ -34,7 +34,7 @@ jobs:
             type=semver,pattern={{major}}
             type=sha,format=long
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         with:
           context: .
           file: docd/Dockerfile

.github/workflows/go.yml

@@ -9,22 +9,19 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go: ["1.21"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
 
       - name: Install dependencies
-        run: sudo apt install wv unrtf tidy
+        run: sudo apt install wv unrtf tidy poppler-utils
 
-      - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v2
+      - name: Set up Go
+        uses: actions/setup-go@v6
         with:
-          go-version: ${{ matrix.go }}
+          go-version-file: go.mod
 
-      - name: Build ${{ matrix.go }}
+      - name: Build
         run: go build -v ./...
 
-      - name: Test ${{ matrix.go }}
+      - name: Test
         run: go test -v -race ./...

CLAUDE.md

@@ -0,0 +1,65 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Overview
+
+`docconv` (module: `code.sajari.com/docconv/v2`) is a Go library that converts documents (PDF, DOC, DOCX, RTF, ODT, HTML, XML, PPTX, Apple Pages, images) to plain text. It wraps external CLI tools and provides a structured `Response{Body, Meta, MSecs, Error}`.
+
+The `docd` subdirectory is an HTTP service (port 8888) built on top of the library.
+
+## System Dependencies
+
+Tests and some converters require system packages. On Debian/Ubuntu:
+```bash
+sudo apt-get install wv unrtf tidy poppler-utils
+```
+OCR support additionally requires `tesseract`.
+
+## Build & Test
+
+```bash
+# Build
+go build -v ./...
+
+# Test (requires system deps above)
+go test -v -race ./...
+
+# Single package
+go test -v -race ./docx_test/...
+
+# With OCR support (requires gosseract/tesseract)
+go build -tags ocr -v ./...
+go test -tags ocr -v -race ./...
+
+# Build the docd HTTP service
+go build -v ./docd
+```
+
+## Architecture
+
+**Dispatch pattern:** `Convert(r io.Reader, mimeType string, readability bool)` in `docconv.go` routes to format-specific converters based on MIME type. `ConvertPath` detects MIME via file extension.
+
+**Format converters** (each returns `(string, map[string]string, error)`):
+- Pure Go: `docx.go`, `odt.go`, `pages.go`, `pptx.go`, `xml.go`
+- Wraps external CLI: `doc.go` (`wvText`), `pdf.go` (`pdftotext`/`pdfinfo`), `rtf.go` (`unrtf`), `html.go` (`tidy`)
+- OCR variants: `pdf_ocr.go`, `image_ocr.go` — only compiled with `-tags ocr`
+- `image.go`: base image handling without OCR
+
+**Key utilities:**
+- `local.go` — `LocalFile` ensures input is on disk (creates temp files for streams); converters that shell out need a real file path
+- `limit.go` — wraps readers with 20MB cap
+- `tidy.go` — wraps the `tidy` CLI for HTML/XML sanitization
+
+**iWork/Pages format:** Uses a custom snappy decompressor (`snappy/`) and protobuf definitions (`iWork/`) to parse Apple's binary format. The `snappy/` package is a fork of snappy-go modified for the `.pages` `.iwa` format: compressed chunks do **not** include the 4-byte checksum prefix that standard framing includes (the checksum is only present on uncompressed chunks). `NewWriter` and `NewReader` are kept consistent with this invariant.
+
+**`docd` service:** HTTP handlers in `docd/convert.go` accept multipart form, path, or streaming input and return JSON. Routing via `gorilla/mux`.
+
+**`client/`:** HTTP client package for talking to a remote `docd` instance.
+
+## Adding a New Format
+
+1. Create `<format>.go` with a `Convert<Format>(r io.Reader) (string, map[string]string, error)` function
+2. Add a MIME type case in the `Convert` switch in `docconv.go`
+3. Add the extension mapping in `MimeTypeByExtension`
+4. Add test data in `<format>_test/testdata/` and a `_test.go` file

README.md

@@ -31,24 +31,22 @@ See `go help install` for details on the installation location of the installed
 
 ```console
 $ sudo apt-get install poppler-utils wv unrtf tidy
-$ go get github.com/JalfResi/justext
 ```
 
 ### macOS
 
 ```console
-$ brew install poppler-qt5 wv unrtf tidy-html5
-$ go get github.com/JalfResi/justext
+$ brew install poppler wv unrtf tidy-html5
 ```
 
 ### Optional dependencies
 
-To add image support to the `docconv` library you first need to [install and build gosseract](https://github.com/otiai10/gosseract/tree/v2.2.4).
+To add image support to the `docconv` library you first need to [install and build gosseract](https://github.com/otiai10/gosseract).
 
 Now you can add `-tags ocr` to any `go` command when building/fetching/testing `docconv` to include support for processing images:
 
 ```console
-$ go get -tags ocr code.sajari.com/docconv/v2/...
+$ go build -tags ocr code.sajari.com/docconv/v2/...
 ```
 
 This may complain on macOS, which you can fix by installing [tesseract](https://tesseract-ocr.github.io) via brew:

doc.go

@@ -12,6 +12,16 @@ import (
 	"github.com/richardlehane/msoleps"
 )
 
+type docMetaResult struct {
+	meta map[string]string
+	err  error
+}
+
+type docBodyResult struct {
+	body string
+	err  error
+}
+
 // ConvertDoc converts an MS Word .doc to text.
 func ConvertDoc(r io.Reader) (string, map[string]string, error) {
 	f, err := NewLocalFile(r)
@@ -21,29 +31,28 @@ func ConvertDoc(r io.Reader) (string, map[string]string, error) {
 	defer f.Done()
 
 	// Meta data
-	mc := make(chan map[string]string, 1)
+	mc := make(chan docMetaResult, 1)
 	go func() {
+		meta := make(map[string]string)
+
 		defer func() {
 			if e := recover(); e != nil {
-				// TODO: Propagate error.
+				mc <- docMetaResult{meta: meta, err: fmt.Errorf("panic reading doc metadata: %v", e)}
 			}
 		}()
 
-		meta := make(map[string]string)
-
 		doc, err := mscfb.New(f)
 		if err != nil {
-			// TODO: Propagate error.
-			mc <- meta
+			mc <- docMetaResult{meta: meta, err: fmt.Errorf("error reading doc metadata: %v", err)}
 			return
 		}
 
 		props := msoleps.New()
 		for entry, err := doc.Next(); err == nil; entry, err = doc.Next() {
 			if msoleps.IsMSOLEPS(entry.Initial) {
 				if err := props.Reset(doc); err != nil {
-					// TODO: Propagate error.
-					break
+					mc <- docMetaResult{meta: meta, err: fmt.Errorf("error reading doc properties: %v", err)}
+					return
 				}
 
 				for _, prop := range props.Property {
@@ -66,42 +75,44 @@ func ConvertDoc(r io.Reader) (string, map[string]string, error) {
 			}
 		}
 
-		mc <- meta
+		mc <- docMetaResult{meta: meta}
 	}()
 
 	// Document body
-	bc := make(chan string, 1)
+	bc := make(chan docBodyResult, 1)
 	go func() {
-		// Save output to a file
 		var buf bytes.Buffer
 		outputFile, err := os.CreateTemp("/tmp", "sajari-convert-")
 		if err != nil {
-			bc <- buf.String()
+			bc <- docBodyResult{err: fmt.Errorf("error creating temp file: %v", err)}
 			return
 		}
 		defer os.Remove(outputFile.Name())
+		defer outputFile.Close()
 
-		err = exec.Command("wvText", f.Name(), outputFile.Name()).Run()
-		if err != nil {
-			// TODO: Propagate error.
+		if err = exec.Command("wvText", f.Name(), outputFile.Name()).Run(); err != nil {
+			bc <- docBodyResult{err: fmt.Errorf("wvText error: %v", err)}
+			return
 		}
 
-		_, err = buf.ReadFrom(outputFile)
-		if err != nil {
-			// TODO: Propagate error.
+		if _, err = buf.ReadFrom(outputFile); err != nil {
+			bc <- docBodyResult{err: fmt.Errorf("error reading wvText output: %v", err)}
+			return
 		}
 
-		bc <- buf.String()
+		bc <- docBodyResult{body: buf.String()}
 	}()
 
-	// TODO: Should errors in either of the above Goroutines stop things from progressing?
-	body := <-bc
-	meta := <-mc
+	br := <-bc
+	mr := <-mc
 
-	// TODO: Check for errors instead of len(body) == 0?
-	if len(body) == 0 {
+	// If wvText failed or produced no output, fall back to DOCX parsing.
+	// Some .doc files are actually DOCX-compatible (e.g. doc saved as docx).
+	if br.err != nil || len(br.body) == 0 {
 		f.Seek(0, 0)
 		return ConvertDocx(f)
 	}
-	return body, meta, nil
+	// Metadata errors are non-fatal: return body with whatever meta we have.
+	_ = mr.err
+	return br.body, mr.meta, nil
 }

docconv_test.go

@@ -1,6 +1,7 @@
 package docconv
 
 import (
+	"os/exec"
 	"strings"
 	"testing"
 )
@@ -18,3 +19,66 @@ func TestConvertTrimsSpace(t *testing.T) {
 		t.Errorf("body = %v, want %v", resp.Body, want)
 	}
 }
+
+func TestConvertUnknownMimeType(t *testing.T) {
+	// Unknown MIME types should return an empty body with no error.
+	resp, err := Convert(strings.NewReader("data"), "application/octet-stream", false)
+	if err != nil {
+		t.Fatalf("got error = %v, want nil", err)
+	}
+	if resp.Body != "" {
+		t.Errorf("body = %q, want empty", resp.Body)
+	}
+}
+
+func TestMimeTypeByExtension(t *testing.T) {
+	tests := []struct {
+		filename string
+		want     string
+	}{
+		{"doc.doc", "application/msword"},
+		{"doc.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
+		{"doc.pdf", "application/pdf"},
+		{"doc.rtf", "application/rtf"},
+		{"doc.html", "text/html"},
+		{"doc.htm", "text/html"},
+		{"doc.xml", "text/xml"},
+		{"doc.txt", "text/plain"},
+		{"doc.pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
+		{"doc.odt", "application/vnd.oasis.opendocument.text"},
+		{"doc.pages", "application/vnd.apple.pages"},
+		{"doc.png", "image/png"},
+		{"doc.jpg", "image/jpeg"},
+		{"doc.jpeg", "image/jpeg"},
+		{"doc.tiff", "image/tiff"},
+		{"DOC.PDF", "application/pdf"}, // case-insensitive
+		{"noextension", "application/octet-stream"},
+		{"unknown.xyz", "application/octet-stream"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.filename, func(t *testing.T) {
+			got := MimeTypeByExtension(tt.filename)
+			if got != tt.want {
+				t.Errorf("MimeTypeByExtension(%q) = %q, want %q", tt.filename, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestConvertXMLViaConvert(t *testing.T) {
+	if _, err2 := exec.LookPath("tidy"); err2 != nil {
+		t.Skip("tidy not installed")
+	}
+
+	resp, err := Convert(
+		strings.NewReader(`<?xml version="1.0"?><root><item>hello</item></root>`),
+		"text/xml",
+		false,
+	)
+	if err != nil {
+		t.Fatalf("Convert(text/xml) error = %v", err)
+	}
+	if !strings.Contains(resp.Body, "hello") {
+		t.Errorf("body = %q, want it to contain %q", resp.Body, "hello")
+	}
+}

docx.go

@@ -41,7 +41,7 @@ func ConvertDocx(r io.Reader) (string, map[string]string, error) {
 	} else {
 		b, err := io.ReadAll(io.LimitReader(r, maxBytes))
 		if err != nil {
-			return "", nil, nil
+			return "", nil, err
 		}
 		size = int64(len(b))
 		ra = bytes.NewReader(b)