perf(storage): O(1) storage_tail append via newest-first batch list #223
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| env: | |
| DATASCRIPT_UPSTREAM_REPO: https://github.com/logseq/datascript.git | |
| DATASCRIPT_UPSTREAM_COMMIT: 3f141af97b70e1f14c65eaa119acd822ebece37e | |
| jobs: | |
| test: | |
| name: OCaml ${{ matrix.ocaml-compiler }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ocaml-compiler: ["5.5.1"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # pin-depends float on `#main`: resolve their current commits so the | |
| # opam cache key changes when a pinned repo moves. Without this the | |
| # restored switch is always stale (its pins predate upstream HEAD), | |
| # `opam install` rebuilds them on every run, and the rebuilt result is | |
| # never cached because the save step is skipped on cache hits. | |
| - name: Resolve pinned dependency commits | |
| id: pins | |
| run: | | |
| export GIT_TERMINAL_PROMPT=0 | |
| sha=$( | |
| grep -hoE 'git\+[^"]+#[^"]+' ./*.opam | sort -u | while IFS= read -r pin; do | |
| repo="${pin%%#*}"; ref="${pin##*#}" | |
| git ls-remote "${repo#git+}" "$ref" || echo "unresolved $pin" | |
| done | shasum | awk '{print $1}' | |
| ) | |
| echo "resolved pins sha: $sha" | |
| echo "sha=$sha" >> "$GITHUB_OUTPUT" | |
| # setup-ocaml creates a *local* switch: the compiler and installed | |
| # packages live in `_opam/` (~1GB+), while `~/.opam` is only the opam | |
| # root skeleton (~40MB) — caching `~/.opam` alone never restores deps. | |
| - name: Restore opam root | |
| id: opam-root | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.opam | |
| key: opam-root-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }} | |
| - name: Restore opam switch | |
| id: opam-switch | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: _opam | |
| key: opam-switch-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-${{ hashFiles('*.opam') }}-${{ steps.pins.outputs.sha }} | |
| restore-keys: | | |
| opam-switch-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}- | |
| # A restored root + local switch is already complete; only the opam | |
| # binary is needed. This skips ocaml/setup-ocaml, whose unconditional | |
| # `opam update` costs ~1m30s on cache hits. | |
| - name: Check for restored opam environment | |
| id: opam-env | |
| run: | | |
| if [ -f "$HOME/.opam/config" ] && [ -d "_opam" ]; then | |
| echo "restored=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "restored=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Install opam binary | |
| if: steps.opam-env.outputs.restored == 'true' | |
| run: | | |
| mkdir -p ~/.local/bin | |
| curl -fsSL https://github.com/ocaml/opam/releases/download/2.6.0/opam-2.6.0-x86_64-linux -o ~/.local/bin/opam | |
| chmod +x ~/.local/bin/opam | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| eval "$(opam env)" | |
| opam --version && ocaml -version | |
| - name: Set up OCaml | |
| if: steps.opam-env.outputs.restored != 'true' | |
| uses: ocaml/setup-ocaml@v3 | |
| with: | |
| ocaml-compiler: ${{ matrix.ocaml-compiler }} | |
| - name: Set up Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: "17" | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24" | |
| - name: Set up Clojure | |
| uses: DeLaGuardo/setup-clojure@13.4 | |
| with: | |
| cli: latest | |
| lein: latest | |
| # `lein cljsbuild` (upstream bundle build) and `clojure -M:cljs` | |
| # (cross-runtime fuzz runner inside `dune runtest`) otherwise | |
| # re-download their Maven/git dependencies on every cold run. | |
| - name: Cache Clojure dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2 | |
| ~/.gitlibs | |
| ~/.clojure | |
| ~/.lein | |
| key: clojure-deps-${{ runner.os }}-${{ env.DATASCRIPT_UPSTREAM_COMMIT }} | |
| restore-keys: | | |
| clojure-deps-${{ runner.os }}- | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev pkg-config | |
| # On an exact switch-cache hit this only verifies and re-fetches the | |
| # (already matching) pins; on a partial restore it rebuilds just the | |
| # moved pins; on a miss it installs everything. The retry covers a | |
| # stale opam-repository snapshot in a restored root failing to resolve | |
| # newly released versions. | |
| - name: Install OCaml dependencies | |
| run: | | |
| # A partially-restored switch keeps its recorded pins, and opam | |
| # does not override them from pin-depends — re-pin each | |
| # pin-depends entry so a moved ref or URL actually rebuilds. | |
| grep -hoE '\["[A-Za-z0-9_.-]+" +"git\+[^"]+"\]' ./*.opam \ | |
| | sed 's/\["//; s/"\]$//; s/" "/ /' \ | |
| | sort -u \ | |
| | while read -r pkg url; do | |
| opam pin add "$pkg" "$url" -y | |
| done | |
| opam install . --deps-only --with-test -y || { | |
| opam update -y | |
| opam install . --deps-only --with-test -y | |
| } | |
| - name: Save opam root | |
| if: steps.opam-root.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ~/.opam | |
| key: opam-root-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }} | |
| - name: Save opam switch | |
| if: steps.opam-switch.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: _opam | |
| key: ${{ steps.opam-switch.outputs.cache-primary-key }} | |
| - name: Restore dune build cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.cache/dune | |
| key: dune-cache-${{ github.ref_name }}-${{ github.sha }} | |
| restore-keys: | | |
| dune-cache-${{ github.ref_name }}- | |
| dune-cache-main- | |
| - name: Enable dune shared cache | |
| run: | | |
| mkdir -p ~/.config/dune | |
| printf '(lang dune 3.0)\n(cache enabled)\n' > ~/.config/dune/config | |
| - name: Cache upstream DataScript | |
| id: cache-upstream-datascript | |
| uses: actions/cache@v4 | |
| with: | |
| path: _ci/datascript | |
| key: datascript-upstream-${{ runner.os }}-${{ env.DATASCRIPT_UPSTREAM_COMMIT }} | |
| - name: Checkout upstream DataScript | |
| if: steps.cache-upstream-datascript.outputs.cache-hit != 'true' | |
| run: | | |
| git clone "$DATASCRIPT_UPSTREAM_REPO" _ci/datascript | |
| git -C _ci/datascript checkout "$DATASCRIPT_UPSTREAM_COMMIT" | |
| - name: Build upstream DataScript JS bundle | |
| if: steps.cache-upstream-datascript.outputs.cache-hit != 'true' | |
| working-directory: _ci/datascript | |
| run: lein with-profile test cljsbuild once release | |
| - name: Verify upstream DataScript cache | |
| run: | | |
| test "$(git -C _ci/datascript rev-parse HEAD)" = "$DATASCRIPT_UPSTREAM_COMMIT" | |
| test -f _ci/datascript/release-js/datascript.js | |
| - name: Run tests | |
| env: | |
| UPSTREAM_DATASCRIPT_REPO: ${{ github.workspace }}/_ci/datascript | |
| UPSTREAM_DATASCRIPT_JS: ${{ github.workspace }}/_ci/datascript/release-js/datascript.js | |
| run: opam exec -- dune runtest | |
| - name: Run memory benchmark | |
| env: | |
| UPSTREAM_DATASCRIPT_JS: ${{ github.workspace }}/_ci/datascript/release-js/datascript.js | |
| run: opam exec -- script/memory_benchmark_vs_cljs.sh | |
| - name: Run benchmark comparison | |
| env: | |
| UPSTREAM_DATASCRIPT_JS: ${{ github.workspace }}/_ci/datascript/release-js/datascript.js | |
| run: opam exec -- script/benchmark_vs_cljs.sh | |
| - name: Save dune build cache | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ~/.cache/dune | |
| key: dune-cache-${{ github.ref_name }}-${{ github.sha }} |