Skip to content

Commit fda5e45

Browse files
authored
fix: added api and orch (#3454)
Ports the remaining build targets to release-please and prebuilt artifacts: orchestrator (one version covers the orchestrator, template-manager, and clean-nfs-cache binaries), nomad-nodepool-apm (both plugin binaries), dashboard-api, and api — which releases together with the infra db-migrator image so the api's startup migration guard and the migrator's baked-in migrations can never skew (both land in the `api` registry repo, tagged with one shared `API_VERSION`). Binaries publish to the public `e2b-artifact-binaries` bucket and images to public `e2b-artifacts` registries, each package Makefile gets the usual `<X>_VERSION` hook (unset = build from source, unchanged), and all four are seeded at 0.0.0 so the first releases come out as 0.0.1. Before merging the release PRs this opens, Atlantis-apply the new `api` and `dashboard-api` registry units in the terragrunt repo.
1 parent 6c72922 commit fda5e45

8 files changed

Lines changed: 278 additions & 13 deletions

File tree

‎.github/workflows/publish.yml‎

Lines changed: 89 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,18 @@ jobs:
5151
# tags.
5252
kind=image
5353
case "${TAG}" in
54+
api-v*)
55+
# api releases as one unit with the infra db-migrator: its
56+
# docker-bake.hcl builds both images (the api bakes in the
57+
# migration timestamp its startup guard expects, the migrator
58+
# bakes in the migrations, so one shared version makes schema
59+
# skew impossible). Both images land in the single `api`
60+
# registry repo (e2b-artifacts/api/api and
61+
# e2b-artifacts/api/db-migrator), which also avoids colliding
62+
# with belt's separate db-migrator repo.
63+
component=api
64+
kind=bake
65+
;;
5466
docker-reverse-proxy-v*)
5567
component=docker-reverse-proxy
5668
# Build context is packages/ (the Dockerfile needs shared/ and db/).
@@ -75,6 +87,13 @@ jobs:
7587
context=packages/clickhouse
7688
build_args=""
7789
;;
90+
dashboard-api-v*)
91+
component=dashboard-api
92+
# Build context is packages/ (the Dockerfile needs shared/).
93+
dockerfile=packages/dashboard-api/Dockerfile
94+
context=packages
95+
build_args="COMMIT_SHA EXPECTED_MIGRATION_TIMESTAMP"
96+
;;
7897
envd-v*)
7998
# envd is a raw static binary (CGO_ENABLED=0), not an image:
8099
# published as a versioned object in the public
@@ -84,6 +103,28 @@ jobs:
84103
component=envd
85104
kind=binary
86105
;;
106+
orchestrator-v*)
107+
# One release covers all three fc-env-pipeline binaries built
108+
# from packages/orchestrator: `make build` produces bin/
109+
# orchestrator and bin/clean-nfs-cache, and template-manager IS
110+
# the orchestrator binary uploaded under a second object name
111+
# (see the package Makefile's upload/template-manager), so no
112+
# separate template-manager object is published. The Makefile's
113+
# ORCHESTRATOR_VERSION flow downloads these and uploads them to
114+
# the client's fc-env-pipeline bucket under the usual names.
115+
component=orchestrator
116+
kind=binary
117+
binary_files="orchestrator clean-nfs-cache"
118+
;;
119+
nomad-nodepool-apm-v*)
120+
# Two Nomad plugin binaries built and released together from
121+
# one package; the Makefile's NOMAD_NODEPOOL_APM_VERSION flow
122+
# downloads both and uploads them to the client's
123+
# fc-env-pipeline bucket under the usual names.
124+
component=nomad-nodepool-apm
125+
kind=binary
126+
binary_files="nomad-nodepool-apm nomad-deployment-aware-target"
127+
;;
87128
*)
88129
# A stray tag caught by the loose glob is a no-op; an explicit
89130
# dispatch for one is a mistake and should fail loudly.
@@ -117,10 +158,15 @@ jobs:
117158
echo "dockerfile=${dockerfile}"
118159
echo "context=${context}"
119160
echo "build_args=${build_args}"
120-
else
121-
# <bucket>/<component>/<version>/<binary>
122-
echo "binary_uri=gs://e2b-artifact-binaries/${component}/v${version}/${component}"
161+
elif [ "${kind}" = "binary" ]; then
162+
# Versioned directory in the public binaries bucket; most
163+
# binary components ship one file named after the component,
164+
# multi-binary packages (orchestrator) list theirs explicitly.
165+
echo "binary_dir=gs://e2b-artifact-binaries/${component}/v${version}"
166+
echo "binary_files=${binary_files:-${component}}"
123167
fi
168+
# kind=bake needs no extra outputs: the bake step derives its
169+
# registry paths from the component and version.
124170
} >> "${GITHUB_OUTPUT}"
125171
126172
- name: Checkout repository
@@ -147,11 +193,11 @@ jobs:
147193
uses: google-github-actions/setup-gcloud@v3
148194

149195
- name: Configure Docker auth
150-
if: ${{ steps.pkg.outputs.kind == 'image' }}
196+
if: ${{ steps.pkg.outputs.kind == 'image' || steps.pkg.outputs.kind == 'bake' }}
151197
run: gcloud auth configure-docker us-docker.pkg.dev --quiet
152198

153199
- name: Set up Docker Buildx
154-
if: ${{ steps.pkg.outputs.kind == 'image' }}
200+
if: ${{ steps.pkg.outputs.kind == 'image' || steps.pkg.outputs.kind == 'bake' }}
155201
uses: docker/setup-buildx-action@v3
156202

157203
- name: Build and push released image
@@ -177,6 +223,11 @@ jobs:
177223
case "${arg}" in
178224
COMMIT_SHA) args+=( --build-arg COMMIT_SHA="$(git rev-parse --short HEAD)" ) ;;
179225
VERSION) args+=( --build-arg VERSION="${IMAGE_TAG}" ) ;;
226+
# The newest DB migration version in the tagged source — the
227+
# startup migration guard (api, dashboard-api). Derived from
228+
# the checkout, so it matches a from-source build; without it
229+
# the guard is silently disabled (empty parses to 0).
230+
EXPECTED_MIGRATION_TIMESTAMP) args+=( --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(scripts/get-latest-migration.sh)" ) ;;
180231
esac
181232
done
182233
docker buildx build \
@@ -189,6 +240,32 @@ jobs:
189240
-f "${DOCKERFILE}" \
190241
"${CONTEXT}"
191242
243+
# Bake components (kind=bake): api's docker-bake.hcl builds the api and
244+
# db-migrator images together; both are pushed into the `api` registry
245+
# repo under the shared version tag. Tag overrides replace the bake
246+
# file's REGISTRY_PREFIX-derived tags; the migration timestamp is
247+
# computed from the tag's checkout, matching a from-source build.
248+
- name: Build and push released images (bake)
249+
if: ${{ steps.pkg.outputs.kind == 'bake' }}
250+
env:
251+
COMPONENT: ${{ steps.pkg.outputs.component }}
252+
VERSION: ${{ steps.pkg.outputs.version }}
253+
run: |
254+
IMAGE_TAG="v${VERSION}"
255+
REGISTRY="us-docker.pkg.dev/e2b-artifacts/${COMPONENT}"
256+
EXPECTED_MIGRATION_TIMESTAMP="$(scripts/get-latest-migration.sh)"
257+
COMMIT_SHA="$(git rev-parse --short HEAD)"
258+
cd packages
259+
REGISTRY_PREFIX="${REGISTRY}" \
260+
COMMIT_SHA="${COMMIT_SHA}" \
261+
EXPECTED_MIGRATION_TIMESTAMP="${EXPECTED_MIGRATION_TIMESTAMP}" \
262+
docker buildx bake -f "${COMPONENT}/docker-bake.hcl" \
263+
--provenance=false \
264+
--sbom=false \
265+
--set "api.tags=${REGISTRY}/api:${IMAGE_TAG}" \
266+
--set "db-migrator.tags=${REGISTRY}/db-migrator:${IMAGE_TAG}" \
267+
--push
268+
192269
# Binary components (kind=binary): build with the repo Go toolchain and
193270
# upload the versioned object to the public e2b-artifact-binaries
194271
# bucket. The package Makefile's `build` target derives the commit sha
@@ -212,9 +289,13 @@ jobs:
212289
# consumers fetch via
213290
# https://storage.googleapis.com/e2b-artifact-binaries/... — the bucket
214291
# is public-read.
215-
- name: Upload released binary
292+
- name: Upload released binaries
216293
if: ${{ steps.pkg.outputs.kind == 'binary' }}
217294
env:
218295
COMPONENT: ${{ steps.pkg.outputs.component }}
219-
BINARY_URI: ${{ steps.pkg.outputs.binary_uri }}
220-
run: gcloud storage cp "packages/${COMPONENT}/bin/${COMPONENT}" "${BINARY_URI}"
296+
BINARY_DIR: ${{ steps.pkg.outputs.binary_dir }}
297+
BINARY_FILES: ${{ steps.pkg.outputs.binary_files }}
298+
run: |
299+
for f in ${BINARY_FILES}; do
300+
gcloud storage cp "packages/${COMPONENT}/bin/${f}" "${BINARY_DIR}/${f}"
301+
done

‎.github/workflows/release-please.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,10 @@ jobs:
117117
client-proxy) path=packages/client-proxy ;;
118118
clickhouse-migrator) path=packages/clickhouse ;;
119119
envd) path=packages/envd ;;
120+
orchestrator) path=packages/orchestrator ;;
121+
api) path=packages/api ;;
122+
dashboard-api) path=packages/dashboard-api ;;
123+
nomad-nodepool-apm) path=packages/nomad-nodepool-apm ;;
120124
*)
121125
echo "::error::release merge for unknown component '${component}' — add it to this workflow's package list"
122126
exit 1

‎.release-please-manifest.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,9 @@
22
"packages/docker-reverse-proxy": "0.0.1",
33
"packages/client-proxy": "0.0.1",
44
"packages/clickhouse": "0.0.1",
5-
"packages/envd": "0.0.1"
5+
"packages/envd": "0.0.1",
6+
"packages/orchestrator": "0.0.0",
7+
"packages/api": "0.0.0",
8+
"packages/dashboard-api": "0.0.0",
9+
"packages/nomad-nodepool-apm": "0.0.0"
610
}

‎packages/api/Makefile‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,27 @@ else
1414
REGISTRY_PREFIX := $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core
1515
endif
1616

17+
# E2B-hosted registry repo holding released, prebuilt api AND db-migrator
18+
# images — they release as one unit from this package's docker-bake.hcl.
19+
# Published to by the release-please workflow on tagging a release.
20+
E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/api
21+
22+
# Optional version "hook" for the released-image flow.
23+
# When empty (default), build-and-upload bakes both images from source and
24+
# pushes to the client's own core repo (the existing flow, unchanged).
25+
# When set (e.g. API_VERSION=v0.1.0), skip building: pull the released api
26+
# and db-migrator images from $(E2B_ARTIFACTS_REGISTRY) and retag/push them
27+
# into the client's core repo. One API_VERSION covers BOTH images on purpose:
28+
# the api bakes in the migration timestamp its startup guard expects and the
29+
# migrator bakes in the migrations, so a shared version makes schema skew
30+
# between them impossible.
31+
#
32+
# Set it either on the command line (make ... API_VERSION=v0.1.0) or in the
33+
# active .env.${ENV} file, which is included above. Quotes are stripped so
34+
# both `=v0.1.0` and `="v0.1.0"` work; a command-line value always wins over
35+
# the env file.
36+
API_VERSION := $(strip $(subst ",,$(API_VERSION)))
37+
1738
.PHONY: generate
1839
generate:
1940
go generate ./...
@@ -73,12 +94,31 @@ profiler:
7394

7495
.PHONY: build-and-upload
7596
build-and-upload:
97+
ifeq ($(strip $(API_VERSION)),)
98+
# Existing flow: bake both images from source and push to the client's
99+
# own core repo.
76100
$(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
77101
$(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration))
78102
cd .. && REGISTRY_PREFIX=$(REGISTRY_PREFIX) \
79103
COMMIT_SHA=$(COMMIT_SHA) \
80104
EXPECTED_MIGRATION_TIMESTAMP=$(EXPECTED_MIGRATION_TIMESTAMP) \
81105
docker buildx bake -f api/docker-bake.hcl --push
106+
else
107+
# Released-image flow: pull the prebuilt, versioned api + db-migrator
108+
# images from the E2B artifacts registry and retag/push them into the
109+
# client's own core repo.
110+
@echo "Using released api + db-migrator $(API_VERSION) from $(E2B_ARTIFACTS_REGISTRY)"
111+
docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY)/api:$(API_VERSION)
112+
docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY)/db-migrator:$(API_VERSION)
113+
docker tag $(E2B_ARTIFACTS_REGISTRY)/api:$(API_VERSION) $(REGISTRY_PREFIX)/api:latest
114+
docker tag $(E2B_ARTIFACTS_REGISTRY)/api:$(API_VERSION) $(REGISTRY_PREFIX)/api:$(API_VERSION)
115+
docker tag $(E2B_ARTIFACTS_REGISTRY)/db-migrator:$(API_VERSION) $(REGISTRY_PREFIX)/db-migrator:latest
116+
docker tag $(E2B_ARTIFACTS_REGISTRY)/db-migrator:$(API_VERSION) $(REGISTRY_PREFIX)/db-migrator:$(API_VERSION)
117+
docker push $(REGISTRY_PREFIX)/api:latest
118+
docker push $(REGISTRY_PREFIX)/api:$(API_VERSION)
119+
docker push $(REGISTRY_PREFIX)/db-migrator:latest
120+
docker push $(REGISTRY_PREFIX)/db-migrator:$(API_VERSION)
121+
endif
82122

83123
.PHONY: test
84124
test:

‎packages/dashboard-api/Makefile‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,29 @@ else
1010
IMAGE_REGISTRY := $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core/dashboard-api
1111
endif
1212

13+
# E2B-hosted registry that holds released, prebuilt dashboard-api images.
14+
# Published to by the release-please workflow on tagging a release.
15+
E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/dashboard-api/dashboard-api
16+
17+
# Optional version "hook" for the released-image flow.
18+
# When empty (default), build-and-upload builds from source and pushes to the
19+
# client's own core repo (the existing flow, unchanged).
20+
# When set (e.g. DASHBOARD_API_VERSION=v0.1.0), skip building: pull the
21+
# released image from $(E2B_ARTIFACTS_REGISTRY) and retag/push it into the
22+
# client's core repo, so the `dashboard-api:latest` lookup keeps working.
23+
#
24+
# Set it either on the command line (make ... DASHBOARD_API_VERSION=v0.1.0)
25+
# or in the active .env.${ENV} file, which is included above. Quotes are
26+
# stripped so both `=v0.1.0` and `="v0.1.0"` work; a command-line value
27+
# always wins over the env file.
28+
#
29+
# Bump in lockstep with API_VERSION: dashboard-api guards startup on the
30+
# same DB's migration timestamp, and the migrations come from the db-migrator
31+
# image that releases WITH the api — a dashboard-api pinned ahead of
32+
# API_VERSION expects migrations the pinned migrator doesn't carry and fails
33+
# its startup check.
34+
DASHBOARD_API_VERSION := $(strip $(subst ",,$(DASHBOARD_API_VERSION)))
35+
1336
HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null)
1437
$(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed))
1538

@@ -31,9 +54,21 @@ build-debug:
3154

3255
.PHONY: build-and-upload
3356
build-and-upload:
57+
ifeq ($(strip $(DASHBOARD_API_VERSION)),)
58+
# Existing flow: build from source and push to the client's own core repo.
3459
$(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
3560
$(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration))
3661
@docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --tag $(IMAGE_REGISTRY):$(COMMIT_SHA) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" -f ./Dockerfile ..
62+
else
63+
# Released-image flow: pull the prebuilt, versioned image from the E2B
64+
# artifacts registry and retag/push it into the client's own core repo.
65+
@echo "Using released dashboard-api $(DASHBOARD_API_VERSION) from $(E2B_ARTIFACTS_REGISTRY)"
66+
docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY):$(DASHBOARD_API_VERSION)
67+
docker tag $(E2B_ARTIFACTS_REGISTRY):$(DASHBOARD_API_VERSION) $(IMAGE_REGISTRY):latest
68+
docker tag $(E2B_ARTIFACTS_REGISTRY):$(DASHBOARD_API_VERSION) $(IMAGE_REGISTRY):$(DASHBOARD_API_VERSION)
69+
docker push $(IMAGE_REGISTRY):latest
70+
docker push $(IMAGE_REGISTRY):$(DASHBOARD_API_VERSION)
71+
endif
3772

3873
.PHONY: run
3974
run:

‎packages/nomad-nodepool-apm/Makefile‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,24 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set")
44
AWS_BUCKET_PREFIX ?= $(PREFIX)$(AWS_ACCOUNT_ID)-
55
GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)-
66

7+
# Public bucket holding released, prebuilt plugin binaries (objects are
8+
# <version>/nomad-nodepool-apm and <version>/nomad-deployment-aware-target).
9+
# Published to by the release-please workflow on tagging a release;
10+
# public-read, so downloads need no credentials.
11+
E2B_ARTIFACT_BINARIES_URL ?= https://storage.googleapis.com/e2b-artifact-binaries/nomad-nodepool-apm
12+
13+
# Optional version "hook" for the released-artifact flow.
14+
# When empty (default), build-and-upload builds from source and uploads the
15+
# binaries to the client's fc-env-pipeline bucket (the existing flow,
16+
# unchanged). When set (e.g. NOMAD_NODEPOOL_APM_VERSION=v0.1.0), skip
17+
# building: download the released binaries and upload those instead. Both
18+
# plugins release together under one version.
19+
#
20+
# Set it either on the command line or in the active .env.${ENV} file, which
21+
# is included above. Quotes are stripped so both `=v0.1.0` and `="v0.1.0"`
22+
# work; a command-line value always wins over the env file.
23+
NOMAD_NODEPOOL_APM_VERSION := $(strip $(subst ",,$(NOMAD_NODEPOOL_APM_VERSION)))
24+
725
PLUGIN_NAME := nomad-nodepool-apm
826
TARGET_PLUGIN_NAME := nomad-deployment-aware-target
927
BIN_DIR := bin
@@ -41,5 +59,23 @@ else
4159
gcloud storage cp --cache-control="no-cache, max-age=0" $(TARGET_BINARY) "gs://${GCP_BUCKET_PREFIX}fc-env-pipeline/nomad-deployment-aware-target.$(COMMIT_SHA)"
4260
endif
4361

62+
# Released-artifact flow: download the prebuilt, versioned binaries from the
63+
# public E2B artifact-binaries bucket.
64+
.PHONY: pull-released
65+
pull-released:
66+
@echo "Using released nomad-nodepool-apm $(NOMAD_NODEPOOL_APM_VERSION) from $(E2B_ARTIFACT_BINARIES_URL)"
67+
@mkdir -p $(BIN_DIR)
68+
curl -fL -o $(BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(PLUGIN_NAME)"
69+
curl -fL -o $(TARGET_BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(TARGET_PLUGIN_NAME)"
70+
71+
# The binaries come from source (default) or from released objects when
72+
# NOMAD_NODEPOOL_APM_VERSION is set. Either way they land in ./bin and the
73+
# upload target is identical.
74+
ifeq ($(strip $(NOMAD_NODEPOOL_APM_VERSION)),)
75+
BIN_SOURCE := build
76+
else
77+
BIN_SOURCE := pull-released
78+
endif
79+
4480
.PHONY: build-and-upload
45-
build-and-upload: build upload
81+
build-and-upload: $(BIN_SOURCE) upload

0 commit comments

Comments
 (0)