Jb/dev environment docker compose

.docker/app/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM node:22.22-bullseye-slim AS build
+FROM node:22.22-bullseye-slim AS base
 SHELL ["/bin/bash", "--login", "-c"]
 
 WORKDIR /app
@@ -13,6 +13,30 @@ COPY webapp/package.json webapp/yarn.lock ./
 # Using a custom node_modules location to avoid mounting it outside of docker
 RUN --mount=type=cache,target=/root/.cache/yarn yarn install --frozen-lockfile --modules-folder /node_modules
 
+# dev target: deps installed, source mounted at runtime — no production build.
+# Also installs the system libraries the Cypress test runner needs (GTK/X11
+# runtime libs plus xvfb for a virtual display), so the webapp test suites can
+# run inside the container.
+FROM base AS dev
+RUN apt update \
+    && apt install -y --no-install-recommends \
+        libgtk2.0-0 \
+        libgtk-3-0 \
+        libgbm-dev \
+        libnotify-dev \
+        libnss3 \
+        libxss1 \
+        libasound2 \
+        libxtst6 \
+        xauth \
+        xvfb \
+    && rm -rf /var/lib/apt/lists/*
+COPY webapp ./
+
+# build target: runs the production webpack bundle. Derives from `base` rather
+# than `dev` so production builds don't pull in the Cypress system libraries.
+FROM base AS build
+COPY webapp ./
 # These get replaced by the entrypoint script for production builds.
 # Set the real values in `.env` files or an external docker-compose.
 ENV NODE_ENV=production
@@ -28,7 +52,6 @@ ARG VUE_APP_AUTOMATICALLY_GENERATE_ID_DEFAULT=magic-generate-id-setting
 ARG VUE_APP_GIT_VERSION=0.0.0+ci
 ENV VUE_APP_GIT_VERSION=${VUE_APP_GIT_VERSION}
 
-COPY webapp ./
 RUN /node_modules/.bin/vue-cli-service build
 
 FROM node:22.22-bullseye-slim AS production

.github/workflows/ci.yml

@@ -196,6 +196,57 @@ jobs:
           # Test that plugin block is listed in info endpoint
           curl -s http://localhost:5000/info/blocks | jq '.data | any(.id == "example")'
 
+  docker-dev:
+    name: Test dev Docker builds
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build dev Docker images
+        uses: docker/bake-action@v7
+        with:
+          files: docker-compose.yml
+          load: true
+          source: .
+          targets: "app-dev,api-dev,database-dev"
+          set: |
+            app-dev.cache-from=type=gha,scope=${{ github.ref_name }}-build-app-dev
+            app-dev.cache-from=type=gha,scope=main-build-app-dev
+            app-dev.cache-to=type=gha,scope=${{ github.ref_name }}-build-app-dev,mode=max
+            api-dev.cache-from=type=gha,scope=${{ github.ref_name }}-build-api
+            api-dev.cache-from=type=gha,scope=main-build-api
+            database-dev.cache-from=type=gha,scope=${{ github.ref_name }}-build-database
+            database-dev.cache-from=type=gha,scope=main-build-database
+            api-dev.args.SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0+ci
+            app-dev.tags=datalab-app-dev:latest
+            api-dev.tags=datalab-api-dev:latest
+            database-dev.tags=datalab-database-dev:latest
+
+      - name: Start dev services
+        run: |
+          # Boot the hot-reloading dev profile and block on healthchecks
+          # (webpack dev server compiled, API answering /healthcheck/is_ready).
+          if ! docker compose --profile dev up --no-build --force-recreate -d --wait; then
+            echo "=== Docker compose startup failed ==="
+            echo "=== Container logs: ==="
+            docker compose --profile dev logs
+            echo "=== Container status: ==="
+            docker compose --profile dev ps
+            exit 1
+          fi
+
+      - name: Check the dev API and app respond
+        run: |
+          curl --fail http://localhost:5001/healthcheck/is_ready
+          curl --fail --silent --output /dev/null http://localhost:8081
+
   e2e:
     name: e2e tests
     runs-on: ubuntu-latest

docker-compose.yml

@@ -57,6 +57,83 @@ services:
     networks:
       - backend
 
+  # ---------------------------------------------------------------------------
+  # Development services (`docker compose --profile dev up`).
+  #
+  # These give a hot-reloading, source-mounted environment for local
+  # development. A small database can be populated
+  # separately with `invoke dev.seed`.
+  # ---------------------------------------------------------------------------
+  database-dev:
+    profiles: ["dev"]
+    build:
+      context: .
+      dockerfile: .docker/mongo/Dockerfile
+    volumes:
+      - datalab-dev-dbdata:/data/db
+    ports:
+      # Published on host port 27018 to avoid clashing with a
+      # developer's native MongoDB.
+      - "27018:27017"
+    networks:
+      - backend
+
+  api-dev:
+    profiles: ["dev"]
+    build:
+      context: .
+      dockerfile: .docker/server/Dockerfile
+      target: api
+    command: ["/opt/.venv/bin/invoke", "dev.serve", "--host", "0.0.0.0", "--port", "5001"]
+    depends_on:
+      - database-dev
+    volumes:
+      - ./pydatalab:/app
+      - datalab-dev-files:/app/files
+    ports:
+      - "5001:5001"
+    networks:
+      - backend
+    environment:
+      - PYDATALAB_MONGO_URI=mongodb://database-dev:27017/${DATALAB_DB_NAME:-datalabvue}
+      - PYDATALAB_FILE_DIRECTORY=/app/files
+
+  app-dev:
+    profiles: ["dev"]
+    build:
+      context: .
+      dockerfile: .docker/app/Dockerfile
+      target: dev
+    command:
+      ["/node_modules/.bin/vue-cli-service", "serve", "--host", "0.0.0.0", "--port", "8081"]
+    volumes:
+      - ./webapp:/app
+    ports:
+      - "8081:8081"
+    environment:
+      - NODE_ENV=development
+      - VUE_APP_API_URL=${VUE_APP_API_URL:-http://localhost:5001}
+      - CHOKIDAR_USEPOLLING=${CHOKIDAR_USEPOLLING:-true}
+    healthcheck:
+      # The dev image has no HEALTHCHECK (that lives in the production stage);
+      # this lets `docker compose up --wait` (used in CI) block until the
+      # webpack dev server has compiled and is serving.
+      test:
+        [
+          "CMD",
+          "node",
+          "-e",
+          "fetch('http://localhost:8081').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",
+        ]
+      interval: 15s
+      timeout: 5s
+      retries: 3
+      start_period: 180s
+
 networks:
   backend:
     driver: bridge
+
+volumes:
+  datalab-dev-dbdata:
+  datalab-dev-files:

pydatalab/.env.example

@@ -0,0 +1,47 @@
+# Development template: copy to `pydatalab/.env` (gitignored) and edit.
+# Loaded natively by `invoke dev.serve` and inside the Docker dev container.
+# Full settings reference: https://docs.datalab-org.io/en/latest/config/
+
+PYDATALAB_IDENTIFIER_PREFIX=dev
+
+PYDATALAB_SECRET_KEY=      # generate: python3 -c 'import secrets; print(secrets.token_hex(64))'
+GITHUB_OAUTH_CLIENT_ID=
+GITHUB_OAUTH_CLIENT_SECRET=
+PYDATALAB_BEHIND_REVERSE_PROXY=False
+PYDATALAB_AUTO_ACTIVATE_ACCOUNTS=True
+PYDATALAB_GITHUB_ORG_ALLOW_LIST=null
+OAUTHLIB_INSECURE_TRANSPORT=1          # local HTTP only; never set in production
+
+
+# PYDATALAB_TESTING=true   # dev only: disables auth; the web app UI has no login in this mode
+#
+# Other OAuth providers (same pattern as GitHub):
+#
+# GOOGLE_OAUTH_CLIENT_ID=
+# GOOGLE_OAUTH_CLIENT_SECRET=
+# MICROSOFT_OAUTH_CLIENT_ID=
+# MICROSOFT_OAUTH_CLIENT_SECRET=
+# ORCID_OAUTH_CLIENT_ID=
+# ORCID_OAUTH_CLIENT_SECRET=
+#
+# PYDATALAB_MONGO_URI=mongodb://localhost:27017/datalabvue
+# PYDATALAB_DEBUG=True
+# PYDATALAB_LOG_FILE=
+# PYDATALAB_FILE_DIRECTORY=
+# PYDATALAB_SESSION_LIFETIME=168
+# PYDATALAB_APP_URL=
+# PYDATALAB_ROOT_PATH=/
+# PYDATALAB_MAX_CONTENT_LENGTH=10000000000
+# PYDATALAB_MAX_BATCH_CREATE_SIZE=10000
+# PYDATALAB_DEPLOYMENT_METADATA=
+# PYDATALAB_EMAIL_DOMAIN_ALLOW_LIST=[]
+# PYDATALAB_EMAIL_AUTO_ACTIVATE_ACCOUNTS=False
+# PYDATALAB_EMAIL_AUTH_SMTP_SETTINGS=
+# PYDATALAB_ASYNC_BLOCK_TYPES=[]
+# PYDATALAB_REMOTE_FILESYSTEMS=[]
+# PYDATALAB_REMOTE_CACHE_MAX_AGE=60
+# PYDATALAB_REMOTE_CACHE_MIN_AGE=1
+# PYDATALAB_BACKUP_STRATEGIES=
+# PYDATALAB_REFCODE_GENERATOR=
+# PYDATALAB_USE_X_ACCEL_REDIRECT=False
+# PYDATALAB_ALLOW_INSECURE_SECRET_KEY=1  # dev only: accept the auto-generated key

pydatalab/docs/INSTALL.md

@@ -205,3 +205,48 @@ There are [pytest fixtures](https://docs.pytest.org/en/7.1.x/how-to/fixtures.htm
 test clients for unauthenticated, unauthorized, normal user and admin user
 access.
 As many authorisation cases should be tested as possible.
+
+## Docker development environment
+
+A hot-reloading *datalab* stack (API + web app + MongoDB); only
+[Docker](https://docs.docker.com/engine/install/) is required.
+
+```bash
+cp pydatalab/.env.example pydatalab/.env    # then fill in the OAuth values (see below)
+cp webapp/.env.example webapp/.env          # optional customisations
+docker compose --profile dev up --build     # first run builds; later just `up`
+```
+
+- Web app: http://localhost:8081
+- API: http://localhost:5001
+- Database: `mongodb://localhost:27018`
+
+Edits to `pydatalab/` and `webapp/` hot-reload; `.env` changes need a restart
+(`docker compose --profile dev restart api-dev` or `app-dev`). To log in to the
+web app, fill in the GitHub/Microsoft/Google OAuth block in `pydatalab/.env`.
+
+### `pydatalab/.env.example`
+
+```bash
+--8<-- "pydatalab/.env.example"
+```
+
+### `webapp/.env.example`
+
+```bash
+--8<-- "webapp/.env.example"
+```
+
+### Notes
+
+- The `dev` and `prod` compose profiles share the same Dockerfiles; `dev` services
+  bind-mount your checkout and keep their data in Docker volumes, with MongoDB on
+  host port 27018 to avoid clashing with a native install. Don't run both profiles
+  at once — they share ports 5001/8081.
+- The `dev` profile injects `PYDATALAB_MONGO_URI` into the API container (it must
+  point at the `database-dev` service), so setting it in `pydatalab/.env` only
+  affects native runs. To use a different database name in Docker, run
+  `export DATALAB_DB_NAME=my_project` before `docker compose up`.
+- Backend tests run natively: from `pydatalab/`, `uv sync --all-extras --dev` once,
+  then `uv run pytest`. Webapp component tests can run inside the container:
+  `docker compose exec app-dev /node_modules/.bin/cypress run --component`.

pydatalab/mkdocs.yml

@@ -61,7 +61,9 @@ markdown_extensions:
   - pymdownx.tabbed:
       alternate_style: true
   - pymdownx.tasklist
-  - pymdownx.snippets
+  - pymdownx.snippets:
+      base_path: [".", ".."]
+      check_paths: true
   - pymdownx.magiclink:
       repo_url_shorthand: true
       hide_protocol: true