e2e.yml

name: E2E

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:
    inputs:
      ios_module:
        description: "iOS e2e module (scripts/test-ios-e2e-suite.sh --list)"
        required: false
        default: "smoke"
      android_module:
        description: "Android e2e module (scripts/test-android-e2e.sh --list)"
        required: false
        default: "smoke"
  schedule:
    # Nightly full-suite coverage; PRs and pushes run the small smoke modules.
    - cron: "37 3 * * *"

concurrency:
  group: e2e-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  IOS_MODULE: ${{ inputs.ios_module || (github.event_name == 'schedule' && 'all' || 'smoke') }}
  ANDROID_MODULE: ${{ inputs.android_module || (github.event_name == 'schedule' && 'all' || 'smoke') }}

jobs:
  # opam-monorepo.0.3.6 (needed for `monorepo pull`) vendors an old base whose
  # -mpopcnt C flag fails to compile on macOS arm64. duniverse/ is pure source
  # — platform-independent — so vendor it once on Linux and share via cache.
  duniverse:
    name: Vendor duniverse
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
    steps:
      - uses: actions/checkout@v4

      - name: Restore opam root
        id: opam-root
        uses: actions/cache/restore@v4
        with:
          path: ~/.opam
          key: opam-root-${{ runner.os }}-ocaml-5.5

      - name: Restore opam switch
        id: opam-switch
        uses: actions/cache/restore@v4
        with:
          path: _opam
          key: opam-switch-${{ runner.os }}-ocaml-5.5-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam') }}

      - name: Install opam binary
        if: steps.opam-root.outputs.cache-hit == 'true' && steps.opam-switch.outputs.cache-hit == 'true'
        run: |
          mkdir -p ~/.local/bin
          curl -fsSL https://github.com/ocaml/opam/releases/download/2.6.0/opam-2.6.0-x86_64-linux -o ~/.local/bin/opam
          chmod +x ~/.local/bin/opam
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          eval $(opam env)
          opam --version && ocaml -version

      - uses: ocaml/setup-ocaml@v3
        if: steps.opam-root.outputs.cache-hit != 'true' || steps.opam-switch.outputs.cache-hit != 'true'
        with:
          ocaml-compiler: "5.5"
          opam-pin: false

      - name: Save opam root
        if: steps.opam-root.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: ~/.opam
          key: opam-root-${{ runner.os }}-ocaml-5.5

      - name: Install opam dependencies
        if: steps.opam-switch.outputs.cache-hit != 'true'
        run: |
          eval $(opam env)
          export GIT_TERMINAL_PROMPT=0
          export GIT_CONFIG_COUNT=3
          export GIT_CONFIG_KEY_0="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_0="git@github.com:"
          export GIT_CONFIG_KEY_1="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_1="ssh://git@github.com/"
          export GIT_CONFIG_KEY_2="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_2="git+ssh://git@github.com/"
          opam pin add -n -y melange-edn-core git+https://github.com/logseq/melange-edn.git#main
          opam pin add -n -y melange-edn-native git+https://github.com/logseq/melange-edn.git#main
          opam pin add -n -y melange-transit-core git+https://github.com/logseq/melange-transit.git#main
          opam pin add -n -y melange-transit-native git+https://github.com/logseq/melange-transit.git#main
          opam install -y melange.7.0.1-55
          opam install . --deps-only --yes --with-test --assume-depexts
          # 0.3.x matches the lockfile's x-opam-monorepo-version.
          opam install -y opam-monorepo.0.3.6

      - name: Save opam switch
        if: steps.opam-switch.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: _opam
          key: opam-switch-${{ runner.os }}-ocaml-5.5-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam') }}

      - name: Restore duniverse
        id: duniverse
        uses: actions/cache/restore@v4
        with:
          path: duniverse
          key: duniverse-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam', '.github/workflows/ci.yml', '.github/workflows/e2e.yml') }}
          enableCrossOsArchive: true

      - name: Pull vendored dependencies
        if: steps.duniverse.outputs.cache-hit != 'true'
        run: |
          set -e
          eval $(opam env)
          # All pins are public; ssh URLs just need rewriting to https.
          git config --global url."https://github.com/".insteadOf "git@github.com:"
          git config --global url."https://github.com/".insteadOf "ssh://git@github.com/"
          git config --global url."https://github.com/".insteadOf "git+ssh://git@github.com/"
          # Call the binary directly: the `opam monorepo` plugin shim rejects
          # plugins built under a different opam version (restored caches).
          # Retry: release-tarball downloads occasionally fail with HTTP 5xx.
          for attempt in 1 2 3; do
            opam exec -- opam-monorepo pull && break
            [ "$attempt" -lt 3 ] || exit 1
            echo "opam-monorepo pull failed (attempt $attempt); retrying" >&2
            sleep 10
          done
          for pin in $(grep -oE 'logseq/[A-Za-z0-9_-]+\.git#[A-Za-z0-9._/-]+' logseq_chat.opam | sort -u); do
            url="${pin%%#*}"; sha="${pin##*#}"; name="$(basename "$url" .git)"
            if [ -d "duniverse/$name" ]; then
              continue
            fi
            git clone --quiet "https://github.com/$url" "duniverse/$name"
            if ! git -C "duniverse/$name" checkout --quiet "$sha"; then
              git -C "duniverse/$name" fetch --quiet "https://github.com/$url" "$sha"
              git -C "duniverse/$name" checkout --quiet FETCH_HEAD
            fi
          done

      - name: Save duniverse
        if: steps.duniverse.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: duniverse
          key: duniverse-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam', '.github/workflows/ci.yml', '.github/workflows/e2e.yml') }}
          enableCrossOsArchive: true

  ios-e2e:
    name: iOS e2e
    needs: duniverse
    runs-on: macos-15
    timeout-minutes: 90
    env:
      LOGSEQ_CHAT_OPAM_SWITCH: ${{ github.workspace }}
      # SwiftPM clones lui over ssh://; rewrite to https (repos are public).
      GIT_CONFIG_COUNT: 3
      GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf
      GIT_CONFIG_VALUE_0: "git@github.com:"
      GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf
      GIT_CONFIG_VALUE_1: "ssh://git@github.com/"
      GIT_CONFIG_KEY_2: url.https://github.com/.insteadOf
      GIT_CONFIG_VALUE_2: "git+ssh://git@github.com/"
    steps:
      - uses: actions/checkout@v4

      - name: Install Maestro
        run: |
          curl -fsSL "https://get.maestro.mobile.dev" | bash
          echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"

      - name: Restore opam root
        id: opam-root
        uses: actions/cache/restore@v4
        with:
          path: ~/.opam
          key: opam-root-${{ runner.os }}-ocaml-5.5

      - name: Restore opam switch
        id: opam-switch
        uses: actions/cache/restore@v4
        with:
          path: _opam
          key: opam-switch-${{ runner.os }}-ocaml-5.5-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam') }}

      # See ci.yml: on a dual cache hit only the opam binary is needed.
      - name: Install opam binary
        if: steps.opam-root.outputs.cache-hit == 'true' && steps.opam-switch.outputs.cache-hit == 'true'
        run: |
          brew install opam
          eval $(opam env)
          opam --version && ocaml -version

      - uses: ocaml/setup-ocaml@v3
        if: steps.opam-root.outputs.cache-hit != 'true' || steps.opam-switch.outputs.cache-hit != 'true'
        with:
          ocaml-compiler: "5.5"
          opam-pin: false

      - name: Save opam root
        if: steps.opam-root.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: ~/.opam
          key: opam-root-${{ runner.os }}-ocaml-5.5

      - name: Install opam dependencies
        if: steps.opam-switch.outputs.cache-hit != 'true'
        run: |
          eval $(opam env)
          export GIT_TERMINAL_PROMPT=0
          export GIT_CONFIG_COUNT=3
          export GIT_CONFIG_KEY_0="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_0="git@github.com:"
          export GIT_CONFIG_KEY_1="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_1="ssh://git@github.com/"
          export GIT_CONFIG_KEY_2="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_2="git+ssh://git@github.com/"
          opam pin add -n -y melange-edn-core git+https://github.com/logseq/melange-edn.git#main
          opam pin add -n -y melange-edn-native git+https://github.com/logseq/melange-edn.git#main
          opam pin add -n -y melange-transit-core git+https://github.com/logseq/melange-transit.git#main
          opam pin add -n -y melange-transit-native git+https://github.com/logseq/melange-transit.git#main
          opam install -y melange.7.0.1-55
          opam install . --deps-only --yes --with-test --assume-depexts

      - name: Save opam switch
        if: steps.opam-switch.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: _opam
          key: opam-switch-${{ runner.os }}-ocaml-5.5-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam') }}

      # Produced by the linux `duniverse` job — opam-monorepo doesn't build on
      # macOS arm64, and a plain restore keeps this job off that toolchain.
      - name: Restore duniverse
        id: duniverse
        uses: actions/cache/restore@v4
        with:
          path: duniverse
          key: duniverse-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam', '.github/workflows/ci.yml', '.github/workflows/e2e.yml') }}
          enableCrossOsArchive: true

      - name: Require vendored dependencies
        if: steps.duniverse.outputs.cache-hit != 'true'
        run: echo "duniverse cache miss; the duniverse job should have populated it" >&2 && exit 1

      - name: Select Xcode 26
        # lui declares swift-tools-version 6.2; the image default (16.4) ships
        # Swift 6.1, so select the Xcode 26 toolchain (Swift 6.2).
        run: sudo xcode-select --switch /Applications/Xcode_26.3.app

      - name: Restore iOS OCaml toolchain
        id: ios-toolchain
        uses: actions/cache/restore@v4
        with:
          path: _build/apple-toolchains
          key: apple-toolchain-${{ runner.os }}-xcode26.3-ios-simulator-5.5.0-${{ hashFiles('scripts/bootstrap-ios-ocaml.sh', 'scripts/build-mobile-libffi.sh') }}

      - name: Boot an iOS simulator
        # simctl boot returns as soon as the boot is initiated; the device
        # finishes booting while the app builds, and bootstatus gates later.
        run: |
          runtime_id=$(xcrun simctl list -j runtimes | python3 -c 'import json,sys; rs=[r for r in json.load(sys.stdin)["runtimes"] if r.get("isAvailable") and ".iOS-" in r["identifier"]]; rs.sort(key=lambda r: [int(x) for x in r["version"].split(".")]); print(rs[-1]["identifier"])')
          udid=$(xcrun simctl create "CI iPhone" \
            "com.apple.CoreSimulator.SimDeviceType.iPhone-16" "$runtime_id")
          xcrun simctl boot "$udid"
          echo "LOGSEQ_CHAT_IOS_SIMULATOR_UDID=$udid" >> "$GITHUB_ENV"

      - name: Start db-sync server (background)
        # Builds while the iOS app build runs; "Wait for db-sync" polls it.
        uses: ./.github/actions/db-sync-server
        with:
          background: "true"

      - name: Restore iOS build artifacts
        # SwiftPM + xcodebuild products; stale objects are harmless — the
        # build only reuses what still matches.
        id: ios-build-artifacts
        uses: actions/cache/restore@v4
        with:
          path: apple/.build
          key: ios-build-artifacts-${{ runner.os }}-xcode26.3-${{ hashFiles('apple/Package.swift', 'apple/Package.resolved') }}
          restore-keys: ios-build-artifacts-${{ runner.os }}-xcode26.3-

      - name: Build iOS app for the simulator
        run: |
          eval $(opam env)
          export LOGSEQ_CHAT_OPAM_SWITCH="$(opam switch show)"
          export DUNE="$(command -v dune)"
          ./scripts/build-mobile-ios-simulator.sh

      - name: Save iOS build artifacts
        if: always()
        uses: actions/cache/save@v4
        with:
          path: apple/.build
          key: ios-build-artifacts-${{ runner.os }}-xcode26.3-${{ hashFiles('apple/Package.swift', 'apple/Package.resolved') }}

      - name: Save iOS OCaml toolchain
        # always(): even when e2e fails, preserve the expensive toolchain
        # build — bootstrap stamp files keep partial builds safe to cache.
        if: always() && steps.ios-toolchain.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: _build/apple-toolchains
          key: apple-toolchain-${{ runner.os }}-xcode26.3-ios-simulator-5.5.0-${{ hashFiles('scripts/bootstrap-ios-ocaml.sh', 'scripts/build-mobile-libffi.sh') }}

      - name: Wait for db-sync and the simulator
        run: |
          for attempt in $(seq 1 180); do
            if curl -s -o /dev/null "http://127.0.0.1:8787/"; then
              echo "db-sync is listening on :8787"
              break
            fi
            if [[ $attempt == 180 ]]; then
              echo "db-sync did not come up; build log:" >&2
              tail -100 logseq-server/deps/db-sync/build.log >&2 || true
              tail -100 logseq-server/deps/db-sync/db-sync.log >&2 || true
              exit 1
            fi
            sleep 2
          done
          xcrun simctl bootstatus "$LOGSEQ_CHAT_IOS_SIMULATOR_UDID" -b

      - name: Run iOS e2e module
        run: |
          eval $(opam env)
          export LOGSEQ_CHAT_OPAM_SWITCH="$(opam switch show)"
          export DUNE="$(command -v dune)"
          LOGSEQ_CHAT_IOS_SKIP_BUILD=1 ./scripts/test-ios-e2e-suite.sh "$IOS_MODULE"

      - name: Upload Maestro debug artifacts
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ios-maestro-debug
          path: ~/.maestro/tests
          retention-days: 7
          if-no-files-found: ignore

  android-e2e:
    name: Android e2e
    needs: duniverse
    runs-on: ubuntu-latest
    timeout-minutes: 90
    env:
      LOGSEQ_CHAT_OPAM_SWITCH: ${{ github.workspace }}
      LOGSEQ_CHAT_ANDROID_ABI: x86_64
      LOGSEQ_CHAT_E2E_BASE_URL: http://127.0.0.1:8787
    steps:
      - uses: actions/checkout@v4

      - name: System dependencies
        run: sudo apt-get install -y libsqlite3-dev libffi-dev pkg-config

      - name: Restore opam root
        id: opam-root
        uses: actions/cache/restore@v4
        with:
          path: ~/.opam
          key: opam-root-${{ runner.os }}-ocaml-5.5

      - name: Restore opam switch
        id: opam-switch
        uses: actions/cache/restore@v4
        with:
          path: _opam
          key: opam-switch-${{ runner.os }}-ocaml-5.5-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam') }}

      - name: Install opam binary
        if: steps.opam-root.outputs.cache-hit == 'true' && steps.opam-switch.outputs.cache-hit == 'true'
        run: |
          mkdir -p ~/.local/bin
          curl -fsSL https://github.com/ocaml/opam/releases/download/2.6.0/opam-2.6.0-x86_64-linux -o ~/.local/bin/opam
          chmod +x ~/.local/bin/opam
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          eval $(opam env)
          opam --version && ocaml -version

      - uses: ocaml/setup-ocaml@v3
        if: steps.opam-root.outputs.cache-hit != 'true' || steps.opam-switch.outputs.cache-hit != 'true'
        with:
          ocaml-compiler: "5.5"
          opam-pin: false

      - name: Save opam root
        if: steps.opam-root.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: ~/.opam
          key: opam-root-${{ runner.os }}-ocaml-5.5

      - name: Install opam dependencies
        if: steps.opam-switch.outputs.cache-hit != 'true'
        run: |
          eval $(opam env)
          export GIT_TERMINAL_PROMPT=0
          export GIT_CONFIG_COUNT=3
          export GIT_CONFIG_KEY_0="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_0="git@github.com:"
          export GIT_CONFIG_KEY_1="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_1="ssh://git@github.com/"
          export GIT_CONFIG_KEY_2="url.https://github.com/.insteadOf" GIT_CONFIG_VALUE_2="git+ssh://git@github.com/"
          opam pin add -n -y melange-edn-core git+https://github.com/logseq/melange-edn.git#main
          opam pin add -n -y melange-edn-native git+https://github.com/logseq/melange-edn.git#main
          opam pin add -n -y melange-transit-core git+https://github.com/logseq/melange-transit.git#main
          opam pin add -n -y melange-transit-native git+https://github.com/logseq/melange-transit.git#main
          opam install -y melange.7.0.1-55
          opam install . --deps-only --yes --with-test --assume-depexts

      - name: Save opam switch
        if: steps.opam-switch.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: _opam
          key: opam-switch-${{ runner.os }}-ocaml-5.5-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam') }}

      # Produced by the `duniverse` job — shared via cross-OS cache.
      - name: Restore duniverse
        id: duniverse
        uses: actions/cache/restore@v4
        with:
          path: duniverse
          key: duniverse-${{ hashFiles('logseq_chat.opam.locked', 'logseq_chat.opam', '.github/workflows/ci.yml', '.github/workflows/e2e.yml') }}
          enableCrossOsArchive: true

      - name: Require vendored dependencies
        if: steps.duniverse.outputs.cache-hit != 'true'
        run: echo "duniverse cache miss; the duniverse job should have populated it" >&2 && exit 1

      - name: Set up Gradle
        uses: gradle/actions/setup-gradle@v4

      - name: Accept Android SDK licenses
        run: yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses || true

      - name: Install Maestro
        run: |
          curl -fsSL "https://get.maestro.mobile.dev" | bash
          echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"

      - name: Restore Android OCaml toolchain
        id: android-toolchain
        uses: actions/cache/restore@v4
        with:
          path: |
            _build/android-toolchain
            _build/android-dependency-sources
            _build/android-sqlite-host
          key: android-toolchain-${{ runner.os }}-x86_64-5.5.0-${{ hashFiles('scripts/bootstrap-android-ocaml.sh', 'scripts/build-mobile-libffi.sh') }}

      - name: Enable KVM
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm

      - name: Start db-sync server (background)
        # Builds while the emulator boots and the APK compiles; the next
        # step polls the port before the flows run.
        uses: ./.github/actions/db-sync-server
        with:
          background: "true"

      - name: Run Android e2e module
        uses: reactivecircus/android-emulator-runner@v2
        env:
          # pub get clones the lui git dep over ssh; rewrite to https
          # (file-based git config doesn't reliably reach the
          # emulator-runner's script environment).
          GIT_CONFIG_COUNT: 3
          GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf
          GIT_CONFIG_VALUE_0: "git@github.com:"
          GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf
          GIT_CONFIG_VALUE_1: ssh://git@github.com/
          GIT_CONFIG_KEY_2: url.https://github.com/.insteadOf
          GIT_CONFIG_VALUE_2: git+ssh://git@github.com/
        with:
          api-level: 34
          target: google_apis
          arch: x86_64
          profile: pixel_6
          disable-animations: true
          emulator-boot-timeout: 600
          script: |
            eval $(opam env)
            export LOGSEQ_CHAT_OPAM_SWITCH="$(opam switch show)"
            export DUNE="$(command -v dune)"
            ./scripts/test-android-e2e.sh "$ANDROID_MODULE"

      - name: Upload Maestro debug artifacts
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: android-maestro-debug
          path: ~/.maestro/tests
          retention-days: 7
          if-no-files-found: ignore

      - name: Save Android OCaml toolchain
        if: always() && steps.android-toolchain.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: |
            _build/android-toolchain
            _build/android-dependency-sources
            _build/android-sqlite-host
          key: android-toolchain-${{ runner.os }}-x86_64-5.5.0-${{ hashFiles('scripts/bootstrap-android-ocaml.sh', 'scripts/build-mobile-libffi.sh') }}