From 8cfee225aabfc47bde91cc71b8f0896afc0e0ea4 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sat, 10 Oct 2026 21:11:27 -0400 Subject: [PATCH 1/2] Add Loki to the kind overlay as the log store The logs pipeline ended at the debug exporter, so lifecycle events died with kubelet's 10 MiB stdout rotation. A single-binary Loki in otel-system now receives them through an otlphttp exporter next to debug. --- docs/dev/best-practices/otel-collector.md | 12 +- manifests/ate-install/kind/kustomization.yaml | 1 + manifests/ate-install/kind/loki.yaml | 134 ++++++++++++++++++ .../ate-install/kind/otel-collector.yaml | 15 +- 4 files changed, 150 insertions(+), 12 deletions(-) create mode 100644 manifests/ate-install/kind/loki.yaml diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index 864ded4538..41aa642125 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -504,12 +504,12 @@ failure. ## Local Development (kind) -`hack/install-ate-kind.sh` installs a collector plus a Jaeger all-in-one into -the `otel-system` namespace — see -`manifests/ate-install/kind/otel-collector.yaml`. It is a single-replica -Deployment, which is appropriate for a one-node kind cluster. The kind -overlay patches every component's `OTEL_EXPORTER_OTLP_ENDPOINT` to point at -it. +`hack/install-ate-kind.sh` installs a collector plus a Jaeger all-in-one, a +Prometheus and a Loki into the `otel-system` namespace — see +`manifests/ate-install/kind/otel-collector.yaml`, `prometheus.yaml` and +`loki.yaml`. The collector is a single-replica Deployment, which is +appropriate for a one-node kind cluster. The kind overlay patches every +component's `OTEL_EXPORTER_OTLP_ENDPOINT` to point at it. View traces: diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index c757f2362f..b6ac8bf9f3 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -37,3 +37,4 @@ resources: - rustfs.yaml - ./otel-collector.yaml - ./prometheus.yaml + - ./loki.yaml diff --git a/manifests/ate-install/kind/loki.yaml b/manifests/ate-install/kind/loki.yaml new file mode 100644 index 0000000000..fbb0c7bdae --- /dev/null +++ b/manifests/ate-install/kind/loki.yaml @@ -0,0 +1,134 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-config + namespace: otel-system +data: + loki.yaml: | + # Single-binary Loki on local disk: the log store of the kind cluster. + auth_enabled: false + server: + http_listen_port: 3100 + grpc_listen_port: 9096 + common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + schema_config: + configs: + - from: "2024-04-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + # The OTLP endpoint stores the ate.* attributes as structured metadata. + limits_config: + allow_structured_metadata: true + analytics: + reporting_enabled: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + namespace: otel-system + labels: + app: loki +spec: + replicas: 1 + selector: + matchLabels: + app: loki + template: + metadata: + labels: + app: loki + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: loki + image: grafana/loki:3.7.8@sha256:1107dd5274e0ada47e42472b7a7e71f3b2a2fe878878108f3e2f9e51528f0193 + args: + - -config.file=/etc/loki/loki.yaml + ports: + - name: http + containerPort: 3100 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /etc/loki + readOnly: true + - name: storage + mountPath: /loki + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: loki-config + - name: storage + emptyDir: {} + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: loki + namespace: otel-system +spec: + type: ClusterIP + selector: + app: loki + ports: + - name: http + port: 3100 + targetPort: http diff --git a/manifests/ate-install/kind/otel-collector.yaml b/manifests/ate-install/kind/otel-collector.yaml index e482ab415e..e7e390a953 100644 --- a/manifests/ate-install/kind/otel-collector.yaml +++ b/manifests/ate-install/kind/otel-collector.yaml @@ -95,6 +95,9 @@ data: insecure: true prometheus: endpoint: 0.0.0.0:8889 + # The log store of the kind cluster (loki.yaml); /v1/logs is appended. + otlphttp/loki: + endpoint: http://loki.otel-system.svc.cluster.local:3100/otlp debug: verbosity: detailed service: @@ -108,15 +111,15 @@ data: processors: [batch] exporters: [prometheus, debug] - # Every ate event, plus any workload that sends OTLP logs. debug is the - # only exporter: kind has no log store, and the e2e read-back goes - # through the count connector below. Never put a sampler or a filter - # here. A consumer takes the last event for an actor, thus a dropped - # record reports a stale state with no sign that anything is missing. + # Every ate event, plus any workload that sends OTLP logs, to Loki and + # to the collector's own log. The e2e read-back goes through the count + # connector below. Never put a sampler or a filter here. A consumer + # takes the last event for an actor, thus a dropped record reports a + # stale state with no sign that anything is missing. logs: receivers: [otlp] processors: [batch] - exporters: [debug] + exporters: [otlphttp/loki, debug] # The count pipelines read the same receiver as the pipelines above, # and they only count. The connector declares MutatesData false, thus From a82bc3295e9b78e6a4bea1b646db2b472ee1da09 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sat, 10 Oct 2026 21:11:28 -0400 Subject: [PATCH 2/2] docs: Querying Loki on kind and aggregating per-replica observables Local-logs section with the port-forward and a query by actor name. ate.workerpool.workers is reported by every ateapi replica, so use max by the pool labels, not sum. --- docs/observability.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/observability.md b/docs/observability.md index 15f872c60e..a68462d332 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -268,6 +268,29 @@ Control plane and node component logs mask proto fields marked `debug_redact`, s --- +### Local Logs with Loki (Kind Cluster) + +For local development inside a `kind` cluster, Agent Substrate provisions a single-binary [Loki](https://grafana.com/docs/loki/latest/) in the `otel-system` namespace, and the collector's logs pipeline sends every record to it next to the `debug` exporter, so the lifecycle events outlive kubelet's 10 MiB stdout rotation. + +Loki indexes `service_name`, `service_instance_id`, `k8s_namespace_name` and `k8s_pod_name` as stream labels and keeps the other attributes, the `ate.*` keys among them, as structured metadata with dots replaced by underscores. Actor identity never becomes an index label. + +To read the lifecycle of one actor: + +1. **Expose the Loki API** via port forwarding: + ```bash + kubectl port-forward -n otel-system svc/loki 3100:3100 + ``` + +2. **Query by actor name**, oldest first. The range defaults to the last hour. + ```bash + curl -sG http://localhost:3100/loki/api/v1/query_range \ + --data-urlencode 'query={service_name="ateapi"} | ate_actor_name="counter-1"' \ + --data-urlencode 'direction=forward' | jq -c '.data.result[] | .stream | {ate_actor_operation_name, ate_actor_state, ate_actor_uid}' + ``` + One line per state change comes back, each with the `trace_id` of the operation that caused it. Add `| ate_actor_state="crashed"` to keep one state, or query by `ate_template_name` to follow every actor of a template. + +> **Note:** Storage is ephemeral (`emptyDir`), so logs are lost when the Loki pod restarts. + ## 2. Metrics Agent Substrate emits foundational OpenTelemetry system and server metrics to monitor the overall health and performance of the control plane services. Every metric below is emitted by a service binary over OTLP and is **independent of the deployment** — a Kind dev cluster gets the same instruments as production; only the backend differs (see [Where Telemetry Goes](#4-where-telemetry-goes)). @@ -322,6 +345,8 @@ For the `ate.filecache.*` instruments, one series per cache under `ate.filecache **No keys** means the operation has no pool. The actor-centric instruments omit the pair, so a crash before the actor reached a worker, or the `no_capacity` outcome, names no pool. No instrument sends the pair with both keys empty. +**Per-replica observables.** Every `ate-api-server` replica reports `ate.workerpool.workers` for the whole fleet, and the install runs two, so a `sum()` across them doubles each count. Aggregate with `max by (ate_workerpool_namespace, ate_workerpool_name, ate_worker_state, ate_sandbox_class) (ate_workerpool_workers)`, or drop the `instance` and `pod` labels first. The same holds for any observable a replicated component reads from a shared source; instruments a replica records for its own requests sum as usual. + The three snapshot labels are orthogonal and mean the same thing on every histogram that carries them: * `ate.snapshot.kind`: which snapshot the operation reads or writes. `local` (the copy on the node: an asynchronous suspend writes it, and a resume reads it while the node still has it, whether or not its upload to durable storage has finished. A synchronous suspend is `latest`, but it also keeps a copy on the node, so the resume after it can be `local`), `latest` (a durable snapshot, including a template's golden snapshot), or `boot` (from scratch, so it never appears on the atelet histograms). * `ate.snapshot.fidelity`: what state it holds. `volumes`, `rootfs`, or `memory`.