Skip to content

Commit 808f3d7

Browse files
robbycochranclaude
robbycochran
and
claude
committed
refactor: config-driven deploy — gateway.toml drives all behavior
Deploy now reads gateways/<name>/gateway.toml and derives all behavior from it: chart OCI/version, SCC grants, addon manifests, images, secrets, and launcher config. New deploy targets require only a new config directory — zero code changes. Key changes: - Add GatewayConfig struct + LoadConfig parser (internal/gateway/config.go) - Replace deployRemote with deployFromConfig (reads from GatewayConfig) - CLI accepts positional arg: harness deploy <ocp|local|kind> - Provider registration filtered by gateway config [providers] section - Local gateway supports custom provider enable/disable - Launcher Job parameterized from config (image, endpoint, SA, mTLS secret) - Teardown reads SCC/secret names from config instead of package vars - Delete deployRemote and package-level hardcoded vars Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent a1ffbf9 commit 808f3d7

11 files changed

Lines changed: 1121 additions & 247 deletions

File tree

‎cmd/deploy.go‎

Lines changed: 147 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,6 @@ import (
1616
"github.com/spf13/cobra"
1717
)
1818

19-
var (
20-
sccPrivilegedSAs = []string{"openshell", "openshell-sandbox", "default"}
21-
secretNames = []string{"openshell-gws", "openshell-atlassian"}
22-
)
23-
2419
func NewDeployCmd(harnessDir, cli string) *cobra.Command {
2520
var (
2621
local bool
@@ -29,31 +24,113 @@ func NewDeployCmd(harnessDir, cli string) *cobra.Command {
2924
)
3025

3126
cmd := &cobra.Command{
32-
Use: "deploy [--local|--remote]",
27+
Use: "deploy [gateway]",
3328
Short: "Deploy or verify the gateway",
29+
Long: "Deploy a gateway by name (e.g., local, ocp, kind). Reads configuration from gateways/<name>/gateway.toml.",
30+
Args: cobra.MaximumNArgs(1),
3431
RunE: func(cmd *cobra.Command, args []string) error {
35-
if remote {
36-
gw := gateway.New(cli)
37-
kc := k8s.New(kubeconfig, k8s.DefaultNamespace())
38-
clusterRunner := k8s.New(kubeconfig, "")
39-
return deployRemote(harnessDir, gw, kc, clusterRunner)
32+
gatewayName, err := resolveGatewayName(args, local, remote)
33+
if err != nil {
34+
return err
4035
}
41-
if local {
42-
gw := gateway.New(cli)
36+
37+
gwDir := filepath.Join(harnessDir, "gateways", gatewayName)
38+
gwCfg, err := gateway.LoadConfig(gwDir)
39+
if err != nil {
40+
return fmt.Errorf("loading gateway config %q: %w", gatewayName, err)
41+
}
42+
43+
gw := gateway.New(cli)
44+
45+
if gwCfg.IsLocal() {
4346
return deployLocal(gw)
4447
}
45-
return fmt.Errorf("specify --local or --remote")
48+
49+
kc := k8s.New(kubeconfig, k8s.DefaultNamespace())
50+
clusterRunner := k8s.New(kubeconfig, "")
51+
return deployFromConfig(harnessDir, gwCfg, gw, kc, clusterRunner)
4652
},
4753
}
4854

49-
cmd.Flags().BoolVar(&local, "local", false, "Verify local podman gateway")
50-
cmd.Flags().BoolVar(&remote, "remote", false, "Deploy to OpenShift cluster")
55+
cmd.Flags().BoolVar(&local, "local", false, "Alias for 'harness deploy local'")
56+
cmd.Flags().BoolVar(&remote, "remote", false, "Alias for 'harness deploy ocp'")
5157
cmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", "Path to kubeconfig (remote only)")
58+
cmd.Flags().MarkHidden("local")
59+
cmd.Flags().MarkHidden("remote")
5260

5361
return cmd
5462
}
5563

56-
func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.Runner) (retErr error) {
64+
func resolveGatewayName(args []string, local, remote bool) (string, error) {
65+
if len(args) > 0 {
66+
return args[0], nil
67+
}
68+
if local {
69+
return "local", nil
70+
}
71+
if remote {
72+
return "ocp", nil
73+
}
74+
return "", fmt.Errorf("specify a gateway: harness deploy <local|ocp|kind>")
75+
}
76+
77+
func deployLocal(gw gateway.Gateway) error {
78+
cliPath := gw.CLIPath()
79+
if cliPath == "" {
80+
return fmt.Errorf("openshell CLI not found. Install it first:\n curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh")
81+
}
82+
83+
status.Section("Container Runtime")
84+
if _, err := exec.LookPath("podman"); err != nil {
85+
status.Fail("Podman not found")
86+
return fmt.Errorf("podman is required")
87+
}
88+
out, _ := exec.Command("podman", "--version").Output()
89+
status.OKf("Podman: %s", strings.TrimSpace(string(out)))
90+
91+
status.Section("Gateway")
92+
gateways, err := gw.GatewayList()
93+
if err != nil {
94+
return fmt.Errorf("listing gateways: %w", err)
95+
}
96+
97+
var localGW string
98+
for _, g := range gateways {
99+
if strings.Contains(g.Endpoint, "127.0.0.1") {
100+
localGW = g.Name
101+
break
102+
}
103+
}
104+
105+
if localGW == "" {
106+
status.Fail("No local gateway found")
107+
fmt.Println()
108+
fmt.Println(" Install OpenShell (auto-registers the gateway):")
109+
fmt.Println(" curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh")
110+
return fmt.Errorf("no local gateway")
111+
}
112+
113+
if err := gw.GatewaySelect(localGW); err != nil {
114+
return fmt.Errorf("selecting gateway %s: %w", localGW, err)
115+
}
116+
117+
if gw.InferenceGet() == nil {
118+
status.OKf("%s (active, reachable)", localGW)
119+
} else {
120+
status.Failf("%s (not responding)", localGW)
121+
fmt.Println()
122+
fmt.Println(" Start the gateway:")
123+
fmt.Println(" macOS: brew services start openshell")
124+
fmt.Println(" Linux: systemctl --user start openshell")
125+
return fmt.Errorf("gateway not responding")
126+
}
127+
128+
fmt.Println()
129+
status.Done("Done.")
130+
return nil
131+
}
132+
133+
func deployFromConfig(harnessDir string, gwCfg *gateway.GatewayConfig, gw gateway.Gateway, kc, clusterRunner k8s.Runner) (retErr error) {
57134
defer func() {
58135
if retErr != nil {
59136
fmt.Fprintf(os.Stderr, "\nDeploy failed. Clean up with: harness teardown --k8s\n")
@@ -65,22 +142,21 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R
65142
chartVersion := os.Getenv("OPENSHELL_CHART_VERSION")
66143
if chartVersion == "" {
67144
cfg, _ := preflight.LoadConfig(filepath.Join(harnessDir, "openshell.toml"))
68-
if cfg != nil {
145+
if cfg != nil && cfg.Upstream.ChartVersion != "" {
69146
chartVersion = cfg.Upstream.ChartVersion
70147
}
71148
}
72149
if chartVersion == "" {
73-
chartVersion = "0.0.55"
150+
chartVersion = gwCfg.Chart.Version
74151
}
75-
chartOCI := "oci://ghcr.io/nvidia/openshell/helm-chart"
76152

77153
fmt.Printf("OpenShell chart: %s\n", chartVersion)
78154
if kbcfg := os.Getenv("KUBECONFIG"); kbcfg != "" {
79155
fmt.Printf("KUBECONFIG: %s\n", kbcfg)
80156
}
81157
fmt.Println()
82158

83-
// Step 1: Namespace (idempotent — ignore AlreadyExists)
159+
// Step 1: Namespace
84160
status.Step(1, "Creating namespace")
85161
clusterRunner.RunKubectl(ctx, "create", "ns", namespace)
86162
if _, err := clusterRunner.RunKubectl(ctx, "label", "ns", namespace,
@@ -92,42 +168,61 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R
92168

93169
// Step 2: Sandbox CRD
94170
status.Step(2, "Installing Sandbox CRD")
95-
if err := clusterRunner.RunKubectlPassthrough(ctx, "apply", "-f",
96-
"https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/manifest.yaml"); err != nil {
171+
if err := clusterRunner.RunKubectlPassthrough(ctx, "apply", "-f", gwCfg.Chart.CRD.URL); err != nil {
97172
return fmt.Errorf("installing sandbox CRD: %w", err)
98173
}
99174

100-
// Step 3: OpenShift SCCs (best-effort — oc may not exist on non-OpenShift)
101-
status.Step(3, "Granting OpenShift SCCs")
102-
for _, sa := range sccPrivilegedSAs {
103-
kc.RunOC(ctx, "adm", "policy", "add-scc-to-user", "privileged", "-z", sa, "-n", namespace)
175+
// Step 3: Platform-specific setup
176+
if gwCfg.IsOCP() {
177+
status.Step(3, "Granting OpenShift SCCs")
178+
for _, sa := range gwCfg.OCP.SCCPrivileged {
179+
kc.RunOC(ctx, "adm", "policy", "add-scc-to-user", "privileged", "-z", sa, "-n", namespace)
180+
}
181+
for _, sa := range gwCfg.OCP.SCCAnyuid {
182+
kc.RunOC(ctx, "adm", "policy", "add-scc-to-user", "anyuid", "-z", sa, "-n", namespace)
183+
}
184+
clusterRunner.RunKubectl(ctx, "create", "clusterrolebinding", "agent-sandbox-admin",
185+
"--clusterrole=cluster-admin",
186+
"--serviceaccount=agent-sandbox-system:agent-sandbox-controller")
104187
}
105-
kc.RunOC(ctx, "adm", "policy", "add-scc-to-user", "anyuid", "-z", "openshell", "-n", namespace)
106-
clusterRunner.RunKubectl(ctx, "create", "clusterrolebinding", "agent-sandbox-admin",
107-
"--clusterrole=cluster-admin",
108-
"--serviceaccount=agent-sandbox-system:agent-sandbox-controller")
109188

110-
// RBAC for launcher
111-
if err := kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "gateways", "ocp", "addons", "rbac.yaml")); err != nil {
112-
return fmt.Errorf("applying launcher RBAC: %w", err)
189+
// Addon manifests (RBAC, etc.)
190+
for _, manifestPath := range gwCfg.ManifestPaths() {
191+
if err := kc.RunKubectlPassthrough(ctx, "apply", "-f", manifestPath); err != nil {
192+
return fmt.Errorf("applying %s: %w", filepath.Base(manifestPath), err)
193+
}
113194
}
114195

115196
// Step 4: Helm install
116197
status.Step(4, "Deploying gateway via Helm")
117-
sandboxImage := envOr("SANDBOX_IMAGE", "ghcr.io/robbycochran/harness-openshell:sandbox")
118198

119-
appsDomain, err := clusterRunner.GetJSONPath(ctx, "ingresses.config.openshift.io/cluster", "{.spec.domain}")
120-
if err != nil || appsDomain == "" {
121-
return fmt.Errorf("could not determine OpenShift apps domain — is this an OpenShift cluster? (kubectl get ingresses.config.openshift.io cluster)")
199+
var gatewayURL string
200+
var routeHost string
201+
202+
switch gwCfg.Gateway.Service {
203+
case "route":
204+
appsDomain, err := clusterRunner.GetJSONPath(ctx, "ingresses.config.openshift.io/cluster", "{.spec.domain}")
205+
if err != nil || appsDomain == "" {
206+
return fmt.Errorf("could not determine OpenShift apps domain — is this an OpenShift cluster? (kubectl get ingresses.config.openshift.io cluster)")
207+
}
208+
routeHost = fmt.Sprintf("gateway-openshell.%s", appsDomain)
209+
gatewayURL = fmt.Sprintf("https://%s:443", routeHost)
210+
case "nodeport":
211+
gatewayURL = ""
212+
case "loadbalancer":
213+
gatewayURL = ""
122214
}
123-
routeHost := fmt.Sprintf("gateway-openshell.%s", appsDomain)
124215

125216
helmArgs := []string{
126-
"upgrade", "--install", "openshell", chartOCI,
217+
"upgrade", "--install", "openshell", gwCfg.Chart.OCI,
127218
"--version", chartVersion,
128-
"--values", filepath.Join(harnessDir, "gateways", "ocp", "helm", "values.yaml"),
129-
"--set", "server.sandboxImage=" + sandboxImage,
130-
"--set", "pkiInitJob.serverDnsNames[0]=" + routeHost,
219+
}
220+
if valuesPath := gwCfg.HelmValuesPath(); valuesPath != "" {
221+
helmArgs = append(helmArgs, "--values", valuesPath)
222+
}
223+
helmArgs = append(helmArgs, "--set", "server.sandboxImage="+gwCfg.Images.Sandbox)
224+
if routeHost != "" {
225+
helmArgs = append(helmArgs, "--set", "pkiInitJob.serverDnsNames[0]="+routeHost)
131226
}
132227
if ps := os.Getenv("PULL_SECRET"); ps != "" {
133228
helmArgs = append(helmArgs, "--set", "imagePullSecrets[0].name="+ps)
@@ -139,28 +234,22 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R
139234
return fmt.Errorf("helm install failed: %w", err)
140235
}
141236

142-
// Wait for gateway
143237
status.Section("Waiting for gateway")
144238
if err := kc.RunKubectlPassthrough(ctx, "rollout", "status", "statefulset/openshell", "--timeout=300s"); err != nil {
145239
return fmt.Errorf("gateway rollout failed: %w", err)
146240
}
147241

148-
// Step 5: Route
149-
status.Step(5, "Creating OpenShift route")
150-
if err := kc.RunKubectlQuiet(ctx, "get", "route", "gateway"); err != nil {
151-
kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "gateways", "ocp", "addons", "route.yaml"))
152-
}
153-
fmt.Printf(" Route: %s\n", routeHost)
242+
// Step 5: CLI gateway config
243+
status.Step(5, "Configuring CLI gateway")
244+
gatewayName := gwCfg.Gateway.Name
154245

155-
// Step 6: CLI gateway config
156-
status.Step(6, "Configuring CLI gateway")
157-
gatewayName := envOr("GATEWAY_NAME", "openshell-remote-ocp")
158-
gatewayURL := fmt.Sprintf("https://%s:443", routeHost)
246+
if gatewayURL == "" {
247+
return fmt.Errorf("service type %q endpoint resolution not yet implemented", gwCfg.Gateway.Service)
248+
}
159249

160-
// Remove existing gateways for this host
161250
existing, _ := gw.GatewayList()
162251
for _, g := range existing {
163-
if strings.Contains(g.Endpoint, routeHost) {
252+
if routeHost != "" && strings.Contains(g.Endpoint, routeHost) {
164253
gw.GatewayRemove(g.Name)
165254
}
166255
}
@@ -169,7 +258,6 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R
169258
return fmt.Errorf("registering gateway %s: %w", gatewayName, err)
170259
}
171260

172-
// Extract mTLS certs
173261
home, err := os.UserHomeDir()
174262
if err != nil {
175263
return fmt.Errorf("determining home directory: %w", err)
@@ -179,9 +267,9 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R
179267
return fmt.Errorf("creating mtls directory: %w", err)
180268
}
181269
for _, field := range []string{"ca.crt", "tls.crt", "tls.key"} {
182-
data, err := kc.GetSecretField(ctx, "openshell-client-tls", field)
270+
data, err := kc.GetSecretField(ctx, gwCfg.Secrets.MTLS, field)
183271
if err != nil {
184-
return fmt.Errorf("extracting %s from openshell-client-tls: %w", field, err)
272+
return fmt.Errorf("extracting %s from %s: %w", field, gwCfg.Secrets.MTLS, err)
185273
}
186274
if err := os.WriteFile(filepath.Join(mtlsDir, field), data, 0o600); err != nil {
187275
return fmt.Errorf("writing %s: %w", field, err)
@@ -193,7 +281,6 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R
193281
}
194282
status.OKf("%s registered (certs from cluster)", gatewayName)
195283

196-
// Wait for gateway to be reachable
197284
fmt.Print(" Waiting for gateway...")
198285
var gwReachable bool
199286
for range 30 {
@@ -214,59 +301,3 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R
214301
status.Done("Done.")
215302
return nil
216303
}
217-
218-
func deployLocal(gw gateway.Gateway) error {
219-
cliPath := gw.CLIPath()
220-
if cliPath == "" {
221-
return fmt.Errorf("openshell CLI not found. Install it first:\n curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh")
222-
}
223-
224-
status.Section("Container Runtime")
225-
if _, err := exec.LookPath("podman"); err != nil {
226-
status.Fail("Podman not found")
227-
return fmt.Errorf("podman is required")
228-
}
229-
out, _ := exec.Command("podman", "--version").Output()
230-
status.OKf("Podman: %s", strings.TrimSpace(string(out)))
231-
232-
status.Section("Gateway")
233-
gateways, err := gw.GatewayList()
234-
if err != nil {
235-
return fmt.Errorf("listing gateways: %w", err)
236-
}
237-
238-
var localGW string
239-
for _, g := range gateways {
240-
if strings.Contains(g.Endpoint, "127.0.0.1") {
241-
localGW = g.Name
242-
break
243-
}
244-
}
245-
246-
if localGW == "" {
247-
status.Fail("No local gateway found")
248-
fmt.Println()
249-
fmt.Println(" Install OpenShell (auto-registers the gateway):")
250-
fmt.Println(" curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh")
251-
return fmt.Errorf("no local gateway")
252-
}
253-
254-
if err := gw.GatewaySelect(localGW); err != nil {
255-
return fmt.Errorf("selecting gateway %s: %w", localGW, err)
256-
}
257-
258-
if gw.InferenceGet() == nil {
259-
status.OKf("%s (active, reachable)", localGW)
260-
} else {
261-
status.Failf("%s (not responding)", localGW)
262-
fmt.Println()
263-
fmt.Println(" Start the gateway:")
264-
fmt.Println(" macOS: brew services start openshell")
265-
fmt.Println(" Linux: systemctl --user start openshell")
266-
return fmt.Errorf("gateway not responding")
267-
}
268-
269-
fmt.Println()
270-
status.Done("Done.")
271-
return nil
272-
}

0 commit comments

Comments
 (0)