Let every Helm chart container set resource requests and limits

Clusters with an admission policy that requires requests and limits
rejected the substrate install, because most containers set neither.

Each container in the chart now has a `resources` value with requests
and limits, for example `ateApi.resources` and
`atenetRouter.agentgateway.resources`. The fixed requests on atelet and
tls-reloader are now values too. manifests/ate-install/rustfs.yaml is
regenerated from the chart.

The defaults come from a 10 minute load on kind: about 1000 req/s
through the router and one actor suspend or resume each second. No
container went above its limits. atelet's working set grew to 195 MiB,
so it requests 256Mi, up from 128Mi. The router's agentgateway used
about 270m CPU, so it requests 100m. The other containers used 3 to 10
times less than their requests. Those requests stay, because a small
actor on kind shows only the minimum use.

Signed-off-by: Danielle Mustillo <danielle.mustillo@solo.io>

Author: danielle-mustillo

charts/substrate/README.md

@@ -47,6 +47,7 @@ See `values.yaml` for the full set; the important keys:
 | `postgres.ownerRole` | `postgres` | Role assumed by migration and partition maintenance connections |
 | `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema |
 | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size |
+| `<component>.resources` | see `values.yaml` | Requests and limits for each container. Every container has a value, e.g. `ateApi.resources` and `atenetRouter.agentgateway.resources` |
 | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots |
 | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` |
 | `atelet.imageCredentialProviderConfig` | `""` | Host path to the kubelet credential provider config; set together with the bin directory |

charts/substrate/templates/ate-api-server.yaml

@@ -81,6 +81,8 @@ spec:
       containers:
       - name: ate-api-server
         image: {{ include "substrate.componentImage" (list "ateapi" .) }}
+        resources:
+{{ toYaml .Values.ateApi.resources | indent 10 }}
         args:
         - "--grpc-listen-addr=0.0.0.0:443"
         - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem"

charts/substrate/templates/ate-controller.yaml

@@ -71,6 +71,8 @@ spec:
       containers:
       - name: ate-controller
         image: {{ include "substrate.componentImage" (list "atecontroller" .) }}
+        resources:
+{{ toYaml .Values.ateController.resources | indent 10 }}
         args:
         # The atecontroller binary defaults --ateapi-conn-spec to
         # dns:///api.ate-system.svc:443, which is correct only for the

charts/substrate/templates/atelet.yaml

@@ -143,9 +143,7 @@ spec:
             add:
             - DAC_OVERRIDE
         resources:
-          requests:
-            cpu: 50m
-            memory: 128Mi
+{{ toYaml .Values.atelet.resources | indent 10 }}
         env:
         - name: NODE_NAME
           valueFrom:

charts/substrate/templates/atenet-egress.yaml

@@ -137,6 +137,8 @@ spec:
       containers:
       - name: agentgateway
         image: {{ include "substrate.thirdPartyImage" (list .Values.images.agentgateway .) }}
+        resources:
+{{ toYaml .Values.atenetEgress.agentgateway.resources | indent 10 }}
         args:
         - -f
         - /etc/agentgateway/config.yaml
@@ -179,6 +181,8 @@ spec:
           readOnly: true
       - name: ext-proc
         image: {{ include "substrate.componentImage" (list "atenet" .) }}
+        resources:
+{{ toYaml .Values.atenetEgress.extProc.resources | indent 10 }}
         args:
         - router
         - --mode=egress

charts/substrate/templates/atenet-router.yaml

@@ -179,6 +179,8 @@ spec:
       containers:
       - name: atenet-router
         image: {{ include "substrate.componentImage" (list "atenet" .) }}
+        resources:
+{{ toYaml .Values.atenetRouter.resources | indent 10 }}
         args:
         - "router"
         - "--mode=ingress"
@@ -225,6 +227,8 @@ spec:
         - { name: podidentity,   mountPath: /run/podidentity.podcert.ate.dev, readOnly: true }
       - name: agentgateway
         image: {{ include "substrate.thirdPartyImage" (list .Values.images.agentgateway .) }}
+        resources:
+{{ toYaml .Values.atenetRouter.agentgateway.resources | indent 10 }}
         args:
         - "-f"
         - "/etc/agentgateway/config.yaml"

charts/substrate/templates/k8s-credential-provider.yaml

@@ -75,6 +75,8 @@ spec:
       containers:
       - name: k8s-credential-provider
         image: {{ include "substrate.componentImage" (list "kubernetes-secrets" .) }}
+        resources:
+{{ toYaml .Values.credentialProvider.resources | indent 10 }}
         imagePullPolicy: {{ include "substrate.imagePullPolicy" . }}
         args:
         - "--listen-address=:50051"

charts/substrate/templates/pod-certificate-controller.yaml

@@ -143,6 +143,8 @@ spec:
       containers:
       - name: controller
         image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }}
+        resources:
+{{ toYaml .Values.podCertificateController.resources | indent 10 }}
         args:
         - --in-cluster=true
         - --sharding-pod-namespace=$(POD_NAMESPACE)

charts/substrate/templates/postgres.yaml

@@ -142,9 +142,7 @@ spec:
         - name: socket
           mountPath: /var/run/postgresql
         resources:
-          requests:
-            cpu: 10m
-            memory: 32Mi
+{{ toYaml .Values.postgres.tlsReloader.resources | indent 10 }}
       {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }}
       containers:
       - name: postgres

charts/substrate/templates/rustfs.yaml

@@ -67,6 +67,8 @@ spec:
       containers:
       - name: rustfs
         image: {{ include "substrate.thirdPartyImage" (list .Values.images.rustfs .) }}
+        resources:
+{{ toYaml .Values.rustfs.resources | indent 10 }}
         imagePullPolicy: {{ include "substrate.imagePullPolicy" . }}
         ports:
         - containerPort: 9000
@@ -108,6 +110,8 @@ spec:
       containers:
       - name: create-bucket
         image: {{ include "substrate.thirdPartyImage" (list .Values.images.awsCli .) }}
+        resources:
+{{ toYaml .Values.rustfs.bucketInit.resources | indent 10 }}
         env:
         - name: AWS_ACCESS_KEY_ID
           value: {{ .Values.rustfs.accessKey | quote }}