Add a measured raw-XDR extraction boundary

README.md

@@ -27,17 +27,12 @@ import (
 )
 
 func main() {
-    // From raw XDR bytes (streaming ingester, gRPC, protobuf)
-    input, err := extract.NewLedgerInputFromXDR(xdrBytes, "Test SDF Network ; September 2015")
-    if err != nil {
-        log.Fatal(err)
-    }
+    // Direct raw XDR to the complete typed surface.
+    data, errs := extract.ExtractAllFromXDR(xdrBytes, "Test SDF Network ; September 2015")
 
     // Or from an already-decoded LedgerCloseMeta (history loader, nebu)
     // input := extract.NewLedgerInput(lcm, "Test SDF Network ; September 2015")
-
-    // Extract everything at once (runs all 16 extractors concurrently)
-    data, errs := extract.ExtractAll(input)
+    // data, errs := extract.ExtractAll(input)
     for _, e := range errs {
         log.Printf("warning: %v", e)
     }
@@ -65,7 +60,16 @@ extract.NewLedgerInput(lcm xdr.LedgerCloseMeta, networkPassphrase string) *Ledge
 // Create input from raw XDR bytes
 extract.NewLedgerInputFromXDR(xdrBytes []byte, networkPassphrase string) (*LedgerInput, error)
 
-// Run all 16 extractors concurrently
+// Direct raw XDR to the complete typed LedgerData surface
+extract.ExtractAllFromXDR(xdrBytes []byte, networkPassphrase string) (*LedgerData, []error)
+
+// Create an explicit borrowed view input
+extract.NewLedgerViewInput(xdrBytes []byte, networkPassphrase string) (*LedgerViewInput, error)
+
+// View-backed contract events without a full ledger decode
+extract.ExtractContractEventsView(input *LedgerViewInput) ([]ContractEventData, error)
+
+// Run all extractors concurrently
 extract.ExtractAll(input *LedgerInput) (*LedgerData, []error)
 ```
 
@@ -107,6 +111,14 @@ type LedgerInput struct {
 
 `Sequence`, `ClosedAt`, and `LedgerRange` are populated automatically by `NewLedgerInput` / `NewLedgerInputFromXDR`. Override `LedgerRange` or set `EraID` after creation if needed.
 
+### LedgerViewInput
+
+`LedgerViewInput` is the explicit borrowed-XDR boundary for full-history work.
+Finish extraction before the upstream `LedgerStream` advances or reuses its
+buffer. View-backed typed rows do not retain the borrowed bytes. The full
+contract, differential gate, benchmarks, and migration status are documented
+in [docs/VIEW_EXTRACTION.md](docs/VIEW_EXTRACTION.md).
+
 ## Usage patterns
 
 ### Obsrvr Lake: history loader
@@ -176,6 +188,7 @@ types_state.go       ClaimableBalanceData, LiquidityPoolData, ConfigSettingData,
 types_tokens.go      TokenTransferData
 
 extract.go           LedgerInput, LedgerData, NewLedgerInput, NewLedgerInputFromXDR, ExtractAll
+view_input.go        borrowed LedgerViewInput, ExtractAllFromXDR, ExtractAllView
 ledgers.go           ExtractLedgers
 transactions.go      ExtractTransactions + helpers
 operations.go        ExtractOperations
@@ -199,7 +212,7 @@ Based on comparison with [stellar-etl](https://github.com/stellar/stellar-etl) (
 ## Design principles
 
 - **Extraction only.** The library converts `xdr.LedgerCloseMeta` into typed Go structs. It doesn't know about Parquet, PostgreSQL, gRPC, protobuf, or any output format. Callers own serialization.
-- **One input type.** Every extractor takes `*LedgerInput`. Callers construct it from XDR bytes or a decoded LCM.
-- **Concurrent by default.** `ExtractAll` runs all 16 extractors in goroutines. Individual extractors are also safe to call concurrently.
+- **Explicit input representations.** Stable parsed extractors take `*LedgerInput`; migrated zero-copy extractors take borrowed `*LedgerViewInput`. The boundary prevents accidental mixed lifetimes.
+- **Concurrent by default.** `ExtractAll` runs every table extractor in goroutines. Individual extractors are also safe to call concurrently.
 - **Single SDK pin.** All extraction logic uses one version of `go-stellar-sdk`. When the SDK is upgraded, every consumer gets the fix.
 - **Protocol changes must fail loudly.** Every extractor switches on XDR union discriminants, and Go does not warn when a protocol upgrade adds an arm an existing switch ignores — the build stays green and the columns go quietly wrong. `protocol_coverage_test.go` enumerates the discriminants the SDK declares valid and fails when one is unhandled.

docs/VIEW_EXTRACTION.md

@@ -0,0 +1,122 @@
+# View-backed extraction
+
+`go-stellar-sdk v0.7.1` introduces experimental zero-copy XDR views. This
+library adopts them behind an explicit `LedgerViewInput` boundary so a future
+SDK signature change is localized and parsed callers remain source-compatible.
+
+## Borrowed input contract
+
+`NewLedgerViewInput` borrows its XDR byte slice. The input, its cached
+transaction views, and every SDK view derived from it are valid only until the
+upstream ledger stream advances or reuses the buffer.
+
+Complete extraction before advancing:
+
+```go
+for ledgerXDR, err := range stream.RawLedgers(ctx, ledgerRange) {
+    if err != nil {
+        return err
+    }
+    input, err := extract.NewLedgerViewInput(ledgerXDR, networkPassphrase)
+    if err != nil {
+        return err
+    }
+    events, err := extract.ExtractContractEventsView(input)
+    if err != nil {
+        return err
+    }
+    if err := writer.AppendContractEvents(events); err != nil {
+        return err
+    }
+    // ledgerXDR may be reused after all rows have been consumed.
+}
+```
+
+Do not retain or copy `LedgerViewInput` after first use. Its synchronized cache
+ensures view-backed table extractors share one SDK transaction-view walk.
+
+Typed output rows do not retain the borrowed XDR bytes.
+
+## Full typed surface
+
+`ExtractAllFromXDR` is the direct raw-XDR-to-`LedgerData` entry point for the
+backfill writer. Today it decodes the ledger exactly once and calls the stable
+parsed extractors. This removes any need for JSON or `LedgerBatch` protobuf in
+the file-backfill path without claiming the full extraction surface is already
+zero-copy.
+
+`ExtractAllView` exists for callers that need view metadata, a custom
+`LedgerRange`, or `EraID` before requesting the complete typed surface. It also
+decodes exactly one compatibility input.
+
+The full surface intentionally does not run parsed and view implementations of
+the same table together. A real-ledger benchmark showed that migrating only
+contract events while retaining the parsed graph for every other table paid
+for both representations and made total extraction about 13% slower. The
+default was kept on the non-regressing one-decode path.
+
+## Current migration status
+
+| Table family | Input path | Notes |
+| --- | --- | --- |
+| Contract events | View-native | Cached `LedgerTransactionViewRange`; diagnostic, operation, and transaction-level events retain parsed-path semantics |
+| All other typed rows | One parsed ledger | Existing stable extractors; migrated incrementally behind `LedgerViewInput` |
+
+Contract events were selected first because they dominate row volume on recent
+Soroban ledgers and the SDK exposes all three event shapes through transaction
+views.
+
+`ExtractLedgerTxParts` is not the contract-event boundary yet because
+`EventsFromTxParts` intentionally omits diagnostic events and V3 event gating
+depends on the paired envelope. `LedgerTransactionViewRange` provides
+diagnostic, operation, and transaction-level events with that pairing in one
+cached API. The parts API remains a candidate when transaction and fee-derived
+tables migrate together.
+
+## Differential gate
+
+The repository does not commit another megabyte-scale ledger fixture. Point
+the parity test at a real `LedgerCloseMeta` binary instead:
+
+```bash
+STELLAR_EXTRACT_LEDGER_FIXTURE=/path/to/ledger.bin \
+go test -run TestExtractAllViewRealLedgerParity -v -count=1
+```
+
+The gate compares:
+
+- errors from every extractor;
+- row counts for every `LedgerData` table family; and
+- every contract-event field after normalizing the intentionally nondeterministic `CreatedAt` timestamp.
+
+The 2026-08-05 gate used SDK fixture ledger 58,752,000, a 1,278,080-byte
+mainnet ledger with 249 transactions and 2,229 extracted contract-event rows.
+It passed.
+
+## Benchmark
+
+```bash
+STELLAR_EXTRACT_LEDGER_FIXTURE=/path/to/ledger.bin \
+go test -run '^$' \
+  -bench 'Benchmark(ContractEvents|ExtractAll)FromXDR' \
+  -benchmem -count=5
+```
+
+On ledger 58,752,000 and an AMD Ryzen 9 7940HS:
+
+| Contract events from raw XDR | Time | Allocated bytes | Allocations |
+| --- | ---: | ---: | ---: |
+| Parsed | 11.40–11.78 ms | 14.90 MB | ~197.7k |
+| View-backed | 8.22–8.42 ms | 8.98 MB | ~129.7k |
+
+The view path reduced time by roughly 28%, allocated bytes by 40%, and
+allocations by 34%. `ExtractAllFromXDR` remains within benchmark noise of the
+manual `NewLedgerInputFromXDR` plus `ExtractAll` sequence because it is a
+convenience boundary over that same one-decode implementation.
+
+## Next table migrations
+
+Migrate a table only when its differential gate is exact and the full-surface
+benchmark improves. The next candidates are transactions/operations, followed
+by change-reader-backed state tables. Once enough families are view-native,
+`ExtractAllView` can stop materializing the parsed ledger entirely.

docs/handoffs/2026-08-04-sdk-0.7.1-upgrade.md

@@ -118,6 +118,13 @@ edge. Total guard runtime is ~15ms.
    the APIs are marked **experimental, signatures may still change**, and it is
    a rewrite of every extractor's read path, not a drop-in. Recommend waiting
    for the experimental marker to come off, then shaping it as its own cycle.
+
+   **Decision recorded 2026-08-05:** adopt incrementally behind this
+   repository's explicit `LedgerViewInput` boundary rather than exposing SDK
+   views throughout consumers. Contract events are the first view-native table
+   family. The complete `ExtractAllFromXDR` surface remains on one parsed decode
+   until enough families migrate to remove the parsed graph; benchmarking
+   rejected a mixed full-surface path that paid for both representations.
 4. **`ScvContractInstance` is still lossy.** `ConvertScValToJSON` returns the
    literal string `"complex_contract_instance"` for it. Pre-existing, unrelated
    to this upgrade, but it is a real hole in `contract_events_stream_v1`.

soroban.go

@@ -79,6 +79,87 @@ func ExtractContractEvents(input *LedgerInput) ([]ContractEventData, error) {
 	return events, nil
 }
 
+// ExtractContractEventsView extracts the same typed contract-event rows from
+// a borrowed LedgerCloseMeta view. The SDK pairs envelopes and transaction
+// processing in one cached walk; only the event payloads retained in output
+// rows are decoded.
+func ExtractContractEventsView(input *LedgerViewInput) ([]ContractEventData, error) {
+	transactions, err := input.transactionViews()
+	if err != nil {
+		return nil, fmt.Errorf("extract ledger transaction views: %w", err)
+	}
+	return extractContractEventsFromViews(input, transactions)
+}
+
+func extractContractEventsFromViews(input *LedgerViewInput, transactions []ingest.LedgerTransactionView) ([]ContractEventData, error) {
+	var events []ContractEventData
+	for txIndex := range transactions {
+		transaction := transactions[txIndex]
+		txHash := hex.EncodeToString(transaction.Hash[:])
+
+		for diagnosticIndex, raw := range transaction.DiagnosticEvents {
+			var diagnostic xdr.DiagnosticEvent
+			if err := diagnostic.UnmarshalBinary(raw); err != nil {
+				return nil, fmt.Errorf("transaction %s diagnostic event %d: %w", txHash, diagnosticIndex, err)
+			}
+			row := extractDiagnosticEvent(
+				diagnostic,
+				txHash,
+				input.Sequence,
+				input.ClosedAt,
+				input.LedgerRange,
+				uint32(diagnosticIndex),
+				transaction.Successful,
+			)
+			row.EraID = input.EraID
+			events = append(events, row)
+		}
+
+		for operationIndex, operationEvents := range transaction.ContractEvents {
+			for eventIndex, raw := range operationEvents {
+				var event xdr.ContractEvent
+				if err := event.UnmarshalBinary(raw); err != nil {
+					return nil, fmt.Errorf("transaction %s operation %d event %d: %w", txHash, operationIndex, eventIndex, err)
+				}
+				row := extractContractEvent(
+					event,
+					txHash,
+					input.Sequence,
+					input.ClosedAt,
+					input.LedgerRange,
+					uint32(operationIndex),
+					uint32(eventIndex),
+					false,
+					transaction.Successful,
+				)
+				row.EraID = input.EraID
+				events = append(events, row)
+			}
+		}
+
+		for eventIndex, raw := range transaction.TransactionEvents {
+			var transactionEvent xdr.TransactionEvent
+			if err := transactionEvent.UnmarshalBinary(raw); err != nil {
+				return nil, fmt.Errorf("transaction %s transaction event %d: %w", txHash, eventIndex, err)
+			}
+			row := extractContractEvent(
+				transactionEvent.Event,
+				txHash,
+				input.Sequence,
+				input.ClosedAt,
+				input.LedgerRange,
+				0,
+				uint32(eventIndex),
+				true,
+				transaction.Successful,
+			)
+			row.EraID = input.EraID
+			events = append(events, row)
+		}
+	}
+	return events, nil
+}
+
 // extractDiagnosticEvent extracts data from a diagnostic event.
 func extractDiagnosticEvent(diagEvent xdr.DiagnosticEvent, txHash string, ledgerSeq uint32, closedAt time.Time, ledgerRange uint32, diagIdx uint32, txSuccessful bool) ContractEventData {
 	eventData := extractContractEvent(

view_benchmark_test.go

@@ -0,0 +1,91 @@
+package extract
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stellar/go-stellar-sdk/network"
+)
+
+var benchmarkContractEvents []ContractEventData
+var benchmarkLedgerData *LedgerData
+
+func BenchmarkContractEventsFromXDR(b *testing.B) {
+	fixturePath := os.Getenv("STELLAR_EXTRACT_LEDGER_FIXTURE")
+	if fixturePath == "" {
+		b.Skip("set STELLAR_EXTRACT_LEDGER_FIXTURE to benchmark a real ledger")
+	}
+	raw, err := os.ReadFile(fixturePath)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.SetBytes(int64(len(raw)))
+
+	b.Run("parsed", func(b *testing.B) {
+		b.ReportAllocs()
+		for b.Loop() {
+			input, err := NewLedgerInputFromXDR(raw, network.PublicNetworkPassphrase)
+			if err != nil {
+				b.Fatal(err)
+			}
+			benchmarkContractEvents, err = ExtractContractEvents(input)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.ReportMetric(float64(len(benchmarkContractEvents)), "events/op")
+	})
+
+	b.Run("view", func(b *testing.B) {
+		b.ReportAllocs()
+		for b.Loop() {
+			input, err := NewLedgerViewInput(raw, network.PublicNetworkPassphrase)
+			if err != nil {
+				b.Fatal(err)
+			}
+			benchmarkContractEvents, err = ExtractContractEventsView(input)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.ReportMetric(float64(len(benchmarkContractEvents)), "events/op")
+	})
+}
+
+func BenchmarkExtractAllFromXDR(b *testing.B) {
+	fixturePath := os.Getenv("STELLAR_EXTRACT_LEDGER_FIXTURE")
+	if fixturePath == "" {
+		b.Skip("set STELLAR_EXTRACT_LEDGER_FIXTURE to benchmark a real ledger")
+	}
+	raw, err := os.ReadFile(fixturePath)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.SetBytes(int64(len(raw)))
+
+	b.Run("parsed", func(b *testing.B) {
+		b.ReportAllocs()
+		for b.Loop() {
+			input, err := NewLedgerInputFromXDR(raw, network.PublicNetworkPassphrase)
+			if err != nil {
+				b.Fatal(err)
+			}
+			var errs []error
+			benchmarkLedgerData, errs = ExtractAll(input)
+			if len(errs) != 1 || errs[0].Error() != "token_transfers: failed to extract token transfer events: error reading from unified events stream, expected version 4 got 3" {
+				b.Fatalf("unexpected extraction errors: %v", errs)
+			}
+		}
+	})
+
+	b.Run("direct", func(b *testing.B) {
+		b.ReportAllocs()
+		for b.Loop() {
+			var errs []error
+			benchmarkLedgerData, errs = ExtractAllFromXDR(raw, network.PublicNetworkPassphrase)
+			if len(errs) != 1 || errs[0].Error() != "token_transfers: failed to extract token transfer events: error reading from unified events stream, expected version 4 got 3" {
+				b.Fatalf("unexpected extraction errors: %v", errs)
+			}
+		}
+	})
+}