diff --git a/.github/workflows/build-ollama-serve.yml b/.github/workflows/build-ollama-serve.yml index c023393..2a5d58a 100644 --- a/.github/workflows/build-ollama-serve.yml +++ b/.github/workflows/build-ollama-serve.yml @@ -37,7 +37,7 @@ jobs: short_sha: ${{ steps.versions.outputs.SHORT_SHA }} steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set version strings id: versions run: | diff --git a/.github/workflows/build-vector-serve.yml b/.github/workflows/build-vector-serve.yml index 131ddd5..08b1eb1 100644 --- a/.github/workflows/build-vector-serve.yml +++ b/.github/workflows/build-vector-serve.yml @@ -28,7 +28,7 @@ jobs: tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v5 with: @@ -55,7 +55,7 @@ jobs: short_sha: ${{ steps.versions.outputs.SHORT_SHA }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set version strings id: versions @@ -92,7 +92,7 @@ jobs: short_sha: ${{ steps.versions.outputs.SHORT_SHA }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set version strings id: versions @@ -134,7 +134,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Login to GHCR uses: docker/login-action@v2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e2a4975..1ee6874 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Install uv uses: astral-sh/setup-uv@v6 diff --git a/.github/workflows/extension_ci.yml b/.github/workflows/extension_ci.yml index 677c3d6..fc11c9c 100644 --- a/.github/workflows/extension_ci.yml +++ b/.github/workflows/extension_ci.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Rust minimal nightly with clippy and rustfmt uses: dtolnay/rust-toolchain@stable with: @@ -53,7 +53,7 @@ jobs: ports: - 3000:3000 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Rust stable toolchain uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/extension_upgrade.yml b/.github/workflows/extension_upgrade.yml index 413ba37..acb19b8 100644 --- a/.github/workflows/extension_upgrade.yml +++ b/.github/workflows/extension_upgrade.yml @@ -24,7 +24,7 @@ jobs: - 3000:3000 steps: - name: Checkout repository content - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1 diff --git a/.github/workflows/pg-image-build.yml b/.github/workflows/pg-image-build.yml index 75021f3..c6d7802 100644 --- a/.github/workflows/pg-image-build.yml +++ b/.github/workflows/pg-image-build.yml @@ -30,7 +30,7 @@ jobs: short_sha: ${{ steps.versions.outputs.SHORT_SHA }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install stoml shell: bash @@ -89,7 +89,7 @@ jobs: short_sha: ${{ steps.versions.outputs.SHORT_SHA }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -163,7 +163,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/proxy_ci.yml b/.github/workflows/proxy_ci.yml new file mode 100644 index 0000000..3b20aa4 --- /dev/null +++ b/.github/workflows/proxy_ci.yml @@ -0,0 +1,110 @@ +name: Proxy CI + +on: + pull_request: + branches: + - main + paths: + - "proxy/**" + - "core/**" + - ".github/workflows/proxy_ci.yml" + - "Cargo.lock" + - "Cargo.toml" + push: + branches: + - main + paths: + - "proxy/**" + - "core/**" + - ".github/workflows/proxy_ci.yml" + - "Cargo.lock" + - "Cargo.toml" + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Install Rust stable with clippy and rustfmt + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@v2 + - name: Cargo format + run: cargo fmt --all --check + - name: Clippy + run: cargo clippy -p vectorize-proxy + + test-unit: + name: Unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Run unit tests + run: cargo test --lib -p vectorize-proxy + + test-integration: + name: Integration tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install postgresql-client + run: sudo apt-get update && sudo apt-get install -y postgresql-client + - name: Start Postgres and vector-serve + run: docker compose up postgres vector-serve -d + - name: Wait for Postgres + run: until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done + - name: Wait for vector-serve + run: until curl -sf http://localhost:3000/alive; do sleep 2; done + timeout-minutes: 3 + - name: Load example dataset + run: psql postgresql://postgres:postgres@localhost:5432/postgres -f server/sql/example.sql + - name: Build proxy binaries + run: cargo build --bin vectorize-server --bin vectorize-proxy + - name: Start vectorize-server + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + EMBEDDING_SVC_URL: http://localhost:3000/v1 + RUST_LOG: info + run: ./target/debug/vectorize-server & + - name: Wait for vectorize-server + run: until curl -sf http://localhost:8080/health/live; do sleep 2; done + timeout-minutes: 2 + - name: Create embedding job + run: | + curl -sf -X POST http://localhost:8080/api/v1/table \ + -H "Content-Type: application/json" \ + -d '{"job_name":"my_job","src_table":"my_products","src_schema":"public","src_columns":["product_name","description"],"primary_key":"product_id","update_time_col":"updated_at","model":"sentence-transformers/all-MiniLM-L6-v2"}' + - name: Wait for embeddings to be generated + run: | + until psql postgresql://postgres:postgres@localhost:5432/postgres \ + -c "SELECT 1 FROM vectorize._embeddings_my_job LIMIT 1;" 2>/dev/null | grep -q "(1 row)"; do + sleep 5 + done + timeout-minutes: 5 + - name: Start vectorize-proxy + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + EMBEDDING_SVC_URL: http://localhost:3000/v1 + RUST_LOG: info + run: ./target/debug/vectorize-proxy & + - name: Wait for vectorize-proxy + run: until pg_isready -h localhost -p 5433; do sleep 2; done + timeout-minutes: 1 + - name: Run integration tests + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + EMBEDDING_SVC_URL: http://localhost:3000/v1 + run: cargo test --test proxy -p vectorize-proxy + - name: Debugging info + if: failure() + run: | + docker compose logs + docker ps diff --git a/.github/workflows/server_ci.yml b/.github/workflows/server_ci.yml index b3408c8..fe872cd 100644 --- a/.github/workflows/server_ci.yml +++ b/.github/workflows/server_ci.yml @@ -26,7 +26,7 @@ jobs: name: Run linters runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Rust minimal nightly with clippy and rustfmt uses: dtolnay/rust-toolchain@stable with: @@ -40,7 +40,7 @@ jobs: test-server: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Rust stable toolchain uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -78,7 +78,7 @@ jobs: short_sha: ${{ steps.versions.outputs.SHORT_SHA }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set version strings id: versions @@ -115,7 +115,7 @@ jobs: short_sha: ${{ steps.versions.outputs.SHORT_SHA }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set version strings id: versions @@ -152,7 +152,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Login to GHCR uses: docker/login-action@v3 diff --git a/.github/workflows/server_release.yml b/.github/workflows/server_release.yml index 9f91c18..b2e970e 100644 --- a/.github/workflows/server_release.yml +++ b/.github/workflows/server_release.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Extract tag id: tag @@ -46,7 +46,7 @@ jobs: runs-on: ubicloud-standard-2-arm-ubuntu-2204 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Extract tag id: tag @@ -81,7 +81,7 @@ jobs: needs: [build_and_push_arm64, build_and_push_amd64] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Login to GHCR uses: docker/login-action@v3