From db737ed27cbf0a5db5d172a4643cb47f1e29c005 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 17 Sep 2026 16:10:18 -0700 Subject: [PATCH 1/4] evetest infra: reach kubectl from any test Listing an app's VMIRS objects is how a test sees which generations of a workload exist, and it was reachable only from tests/apps. Give EdgeDevice a kubectl runner and a VMIRS lister, so any suite can ask, and point the app-workload helpers and the tie-breaker cluster test at them instead of each shelling out to "eve exec kube kubectl" on its own. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 (1M context) --- evetest/edgedevice.go | 79 ++++++++++++++++ evetest/tests/apps/appstate_helpers_test.go | 9 +- .../tests/apps/appworkload_helpers_test.go | 92 ++++--------------- evetest/tests/apps/purge_helpers_test.go | 4 +- evetest/tests/apps/testsuite_test.go | 5 +- evetest/tests/cluster/tiebreaker_test.go | 12 +-- 6 files changed, 110 insertions(+), 91 deletions(-) diff --git a/evetest/edgedevice.go b/evetest/edgedevice.go index 122a36936c6..2afc7b0ebf4 100644 --- a/evetest/edgedevice.go +++ b/evetest/edgedevice.go @@ -2769,6 +2769,85 @@ func (d *EdgeDevice) WatchNTPSources() ( return ch, d.trackWatcherUnsub(unsub) } +// EVEKubeAppNamespace is the Kubernetes namespace EVE runs app workloads in; +// both VMIRS objects and their PVCs live there. See +// pkg/pillar/kubeapi.EVEKubeNameSpace. +const EVEKubeAppNamespace = "eve-kube-app" + +// appDomainNameLabel holds the owning app's DomainName, +// ".." (the eveLabelKey constant in +// hypervisor/kubevirt.go). EVE puts this label in the VMIRS +// spec.selector.matchLabels and in the VMI template, but not in the VMIRS +// metadata.labels, so attribution reads the selector - the same way pillar +// attributes a VMIRS in sweepStaleGenerations. +const appDomainNameLabel = "App-Domain-Name" + +// RunKubectl runs kubectl on the device with the given arguments and returns +// its trimmed standard output. The call goes through "eve exec kube" because +// kubectl exists only in the kube container, so the device has to be an EVE-K +// node. +// +// The returned error carries kubectl's own standard error, which says why the +// command was refused. A caller polling a cluster that is still coming up +// should treat any error as a retryable "not yet": kubectl fails this way +// while k3s is starting, and that is indistinguishable here from a real +// refusal. +func (d *EdgeDevice) RunKubectl(args string, timeout time.Duration) (string, error) { + stdout, stderr, err := d.RunShellScript( + "eve exec kube kubectl "+args, timeout, 0) + if err != nil { + return "", fmt.Errorf("kubectl %s: %w (stderr: %s)", + args, err, strings.TrimSpace(stderr)) + } + return strings.TrimSpace(stdout), nil +} + +// ListAppVMIRS returns the sorted names of every VMIRS belonging to appUUID, +// whichever purge generation each one is. A VMIRS name embeds the app's purge +// counter (base.GetAppKubeNameWithPurge), which makes this the one vantage +// point from which a stale generation surviving a purge is observable at all: +// pillar's own DomainStatus is keyed by app UUID and so can only ever describe +// one of them. +// +// Attribution is by the appDomainNameLabel selector rather than the name, +// since the label value also carries a version and an appnum that do not +// matter here. An empty list with a nil error means the app has no VMIRS; an +// error means the list could not be read, which a caller inside an Eventually +// should retry rather than fail on. +func (d *EdgeDevice) ListAppVMIRS( + appUUID uuid.UUID, timeout time.Duration) ([]string, error) { + stdout, err := d.RunKubectl( + "-n "+EVEKubeAppNamespace+" get vmirs -o json", timeout) + if err != nil { + return nil, err + } + var list struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Selector struct { + MatchLabels map[string]string `json:"matchLabels"` + } `json:"selector"` + } `json:"spec"` + } `json:"items"` + } + if err := json.Unmarshal([]byte(stdout), &list); err != nil { + return nil, fmt.Errorf("parsing kubectl vmirs output: %w", err) + } + prefix := appUUID.String() + "." + var names []string + for _, item := range list.Items { + if strings.HasPrefix( + item.Spec.Selector.MatchLabels[appDomainNameLabel], prefix) { + names = append(names, item.Metadata.Name) + } + } + sort.Strings(names) + return names, nil +} + // GetClusterInfo returns the last recorded information about the Kubernetes // cluster, or nil if no such info message has been received yet. func (d *EdgeDevice) GetClusterInfo() *eveinfo.ZInfoKubeCluster { diff --git a/evetest/tests/apps/appstate_helpers_test.go b/evetest/tests/apps/appstate_helpers_test.go index 5b2157b60ec..ba466b5e6a0 100644 --- a/evetest/tests/apps/appstate_helpers_test.go +++ b/evetest/tests/apps/appstate_helpers_test.go @@ -63,8 +63,8 @@ func appPurgePhase(dev *evetest.EdgeDevice, appUUID uuid.UUID) ( // appDomainStatus returns domainmgr's published DomainStatus for the app. There // is at most one, because DomainStatus is keyed by app UUID - which is exactly -// why it cannot be used to count workload generations (see listAppVMIRS and -// listKVMDomainDirs in appworkload_helpers_test.go). It is authoritative for the +// why it cannot be used to count workload generations (see +// EdgeDevice.ListAppVMIRS and listKVMDomainDirs in appworkload_helpers_test.go). It is authoritative for the // domain's id, name and attached disks. func appDomainStatus( dev *evetest.EdgeDevice, appUUID uuid.UUID) (types.DomainStatus, bool) { @@ -77,8 +77,9 @@ func appDomainStatus( // before the old generation is actually gone - and it is not republished // anywhere in the EVE API, so it is read from the persisted pubsub state. // -// found is false while the file does not exist, which is the expected state -// before an app's first purge. +// found is false only if the record could not be read: zedmanager allocates it +// when it first handles the app's config, so it exists from well before the +// app's first purge, holding 0. func purgeCounter( dev *evetest.EdgeDevice, appUUID uuid.UUID) (counter uint32, found bool) { var rec types.UuidToNum diff --git a/evetest/tests/apps/appworkload_helpers_test.go b/evetest/tests/apps/appworkload_helpers_test.go index 421e502789e..a4cf59c9e81 100644 --- a/evetest/tests/apps/appworkload_helpers_test.go +++ b/evetest/tests/apps/appworkload_helpers_test.go @@ -4,19 +4,19 @@ // Where and whether the app is actually running, as the hypervisor itself sees // it: VMIRS objects on eve-k, qemu domain state directories on kvm/xen. // -// Rule for this file: readers that enumerate the app's WORKLOAD instances, plus -// the kubectl plumbing they are built on. This is the one place a stale -// generation is observable, because pillar's own DomainStatus is keyed by app -// UUID and so can only ever describe one (appstate_helpers_test.go). +// Rule for this file: readers that enumerate the app's WORKLOAD instances. This +// is the one place a stale generation is observable, because pillar's own +// DomainStatus is keyed by app UUID and so can only ever describe one +// (appstate_helpers_test.go). // // Everything generic - reading a file, listing a directory, testing for a path, -// flushing caches - is an EdgeDevice method in the framework -// (evetest/edgedevice.go); this file only adds what is specific to EVE's -// workload objects. Note the two error conventions that follow from that: -// helpers backed by a framework method surface its error to Gomega, because an -// error there is a transport failure, while a failed kubectl call is reported as -// found=false, because "k3s is not up yet" is an expected transient state a -// caller inside Eventually should retry on rather than fail. +// flushing caches, reaching the cluster's kubectl - is an EdgeDevice method in +// the framework (evetest/edgedevice.go); this file only adds what is specific +// to EVE's workload objects. Note the two error conventions that follow from +// that: helpers backed by a framework method surface its error to Gomega, +// because an error there is a transport failure, while a failed kubectl call is +// reported as found=false, because "k3s is not up yet" is an expected transient +// state a caller inside Eventually should retry on rather than fail. // // Split trigger: if a xen-specific reader is ever needed, break this into // kube- and local-domain helper files. Two backends sharing one file is @@ -29,7 +29,6 @@ import ( "encoding/json" "fmt" "path" - "sort" "strconv" "strings" "time" @@ -45,20 +44,6 @@ const ( // sshCmdTimeout bounds a single kubectl invocation run over SSH. sshCmdTimeout = 20 * time.Second - // eveKubeAppNamespace is the Kubernetes namespace EVE runs app workloads in; - // both VMIRS objects and their PVCs live there. See - // pkg/pillar/kubeapi.EVEKubeNameSpace. - eveKubeAppNamespace = "eve-kube-app" - - // appDomainNameLabel holds the owning app's DomainName, - // "..". See the eveLabelKey constant in - // hypervisor/kubevirt.go. - // - // EVE puts this label in the VMIRS spec.selector.matchLabels and in the VMI - // template, but not in the VMIRS metadata.labels. Read the selector. Pillar - // attributes a VMIRS the same way, in sweepStaleGenerations. - appDomainNameLabel = "App-Domain-Name" - // kvmDomainStateDir mirrors hypervisor/kvm.go's kvmStateDir: qemu gets one // directory per domain, holding that domain's pidfile. EVE bind-mounts /run // into the pillar container, so this path is readable from dom0 - the same @@ -66,21 +51,13 @@ const ( kvmDomainStateDir = "/run/hypervisor/kvm" ) -// kubeItemList is the minimal shape needed from any `kubectl get -// -o json`: a name per item, the item's own labels, and the selector labels. -// A VMIRS is attributed by its selector, which is the only place EVE puts the -// App-Domain-Name label. A PVC has neither field. +// kubeItemList is the minimal shape needed from `kubectl get -o +// json`: a name per item. type kubeItemList struct { Items []struct { Metadata struct { - Name string `json:"name"` - Labels map[string]string `json:"labels"` + Name string `json:"name"` } `json:"metadata"` - Spec struct { - Selector struct { - MatchLabels map[string]string `json:"matchLabels"` - } `json:"selector"` - } `json:"spec"` } `json:"items"` } @@ -96,21 +73,12 @@ type kubeItemList struct { // publishes carries only the single name of the desired generation. Until that // gap is closed, this is the only vantage point from which a stale generation // surviving a purge is observable at all. -// -// Promotion trigger: when a second suite needs kubectl access, move this to an -// EdgeDevice method in evetest/edgedevice.go. Do not copy it. It is kept local -// for now because tests/cluster has no kubectl calls at all, so the shape is -// unsettled after two consumers, and because the framework has no non-fatal -// read family to fit it into yet. func kubectlListItems( dev *evetest.EdgeDevice, resource string) (list kubeItemList, found bool) { - stdout, stderr, err := dev.RunShellScript( - "eve exec kube kubectl -n "+eveKubeAppNamespace+" get "+resource+" -o json", - sshCmdTimeout, 0) + stdout, err := dev.RunKubectl( + "-n "+evetest.EVEKubeAppNamespace+" get "+resource+" -o json", sshCmdTimeout) if err != nil { - evetest.Logger().Warnf( - "kubectlListItems: kubectl get %s failed: %v (stderr: %s)", - resource, err, stderr) + evetest.Logger().Warnf("kubectlListItems: %v", err) return list, false } if err := json.Unmarshal([]byte(stdout), &list); err != nil { @@ -121,35 +89,11 @@ func kubectlListItems( return list, true } -// listAppVMIRS returns the names of every VMIRS (any generation) that belongs to -// appUUID. It matches the prefix "." on the App-Domain-Name selector -// label, because the label value also carries a version and an appnum that do -// not matter here. Names are sorted, so a caller can compare the whole set. -// -// found is false if the list could not be read. An empty list then means "no -// VMIRS", and not "the device did not answer". -func listAppVMIRS( - dev *evetest.EdgeDevice, appUUID uuid.UUID) (names []string, found bool) { - list, ok := kubectlListItems(dev, "vmirs") - if !ok { - return nil, false - } - prefix := appUUID.String() + "." - for _, item := range list.Items { - if strings.HasPrefix( - item.Spec.Selector.MatchLabels[appDomainNameLabel], prefix) { - names = append(names, item.Metadata.Name) - } - } - sort.Strings(names) - return names, true -} - // listKVMDomainDirs returns the qemu per-domain state directories belonging to // appUUID, found by prefix on the domain name ("..", see // types.DomainConfig.GetTaskName). // -// Note the asymmetry with listAppVMIRS: a kvm domain name carries no purge +// Note the asymmetry with EdgeDevice.ListAppVMIRS: a kvm domain name carries no purge // counter, so two generations of the same app at the same version would share one // directory name and be indistinguishable here. That is also why a surviving // generation cannot take this shape on kvm at all. What this does catch is a diff --git a/evetest/tests/apps/purge_helpers_test.go b/evetest/tests/apps/purge_helpers_test.go index 62fa1048023..9ebc83e19bf 100644 --- a/evetest/tests/apps/purge_helpers_test.go +++ b/evetest/tests/apps/purge_helpers_test.go @@ -213,8 +213,8 @@ func assertExactlyOneVMIRSAtGeneration( // equivalent. wantName := base.GetAppKubeName(appDisplayName, appUUID) + "-" + strconv.FormatUint(uint64(newCounter), 10) - names, found := listAppVMIRS(dev, appUUID) - g.Expect(found).To(BeTrue(), + names, err := dev.ListAppVMIRS(appUUID, sshCmdTimeout) + g.Expect(err).ToNot(HaveOccurred(), "could not list VMIRS objects; k3s may still be starting") g.Expect(names).To(HaveLen(1), "expected exactly one VMIRS for the app, found %v", names) diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go index dd8c5af1b64..c0654407a4c 100644 --- a/evetest/tests/apps/testsuite_test.go +++ b/evetest/tests/apps/testsuite_test.go @@ -17,8 +17,9 @@ // appstate_helpers_test.go pillar's own view of the app - pubsub and // persisted state keyed by app UUID // appworkload_helpers_test.go where the app is running as the hypervisor sees -// it - VMIRS objects (and the kubectl plumbing for -// them), qemu domain state directories +// it - VMIRS objects, qemu domain state +// directories, and the kubectl reads the volume +// helpers share // appvolumes_helpers_test.go the app's disk in all three forms - // VolumeStatus, PVC, file under /persist - and the // storage invariants diff --git a/evetest/tests/cluster/tiebreaker_test.go b/evetest/tests/cluster/tiebreaker_test.go index 9387c49e6e3..4b688f7a19e 100644 --- a/evetest/tests/cluster/tiebreaker_test.go +++ b/evetest/tests/cluster/tiebreaker_test.go @@ -55,16 +55,10 @@ const ( tieBreakerReplicas = "2" ) -// runKubectl goes through "eve exec kube" because kubectl exists only in the -// kube container, not in the host shell. +// runKubectl pins every query below to the tie-breaker phase's own kubectl +// budget, so the call sites do not each have to repeat it. func runKubectl(device *evetest.EdgeDevice, args string) (string, error) { - stdout, stderr, err := device.RunShellScript( - "eve exec kube kubectl "+args, tieBreakerKubectlTimeout, 0) - if err != nil { - return "", fmt.Errorf("kubectl %s: %w (stderr: %s)", - args, err, strings.TrimSpace(stderr)) - } - return strings.TrimSpace(stdout), nil + return device.RunKubectl(args, tieBreakerKubectlTimeout) } func expectKubectl(t *WithT, device *evetest.EdgeDevice, From 0c1ad16c58ee1e172a8b93e8d4797d4b9dfaebd7 Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Sat, 19 Sep 2026 09:23:38 -0700 Subject: [PATCH 2/4] evetest infra: let a cluster purge exclude a node EdgeCluster.PurgeApplication locates the node hosting the application and waits there for the purge to be observed. A caller that has powered a node off cannot let that node be the answer: the cluster info it published before going down still names it as the host, so the wait sits on a device that will never report again and fails on the timeout. Accept the same excludeDevNames the host lookup already takes, and pass it through. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 (1M context) --- evetest/edgecluster.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/evetest/edgecluster.go b/evetest/edgecluster.go index c1308fbbb9d..2a09bc5cb32 100644 --- a/evetest/edgecluster.go +++ b/evetest/edgecluster.go @@ -395,10 +395,14 @@ func (ec *EdgeCluster) RebootApplication(appUUID uuid.UUID, waitUntilRebooted bo // The purge counter is incremented on all devices, but the wait (if requested) // is performed only on the device hosting the application. See // EdgeDevice.PurgeApplication for volumeGen. +// +// Name in excludeDevNames any node that must not be taken for the host, such as +// one the caller has powered off: the cluster info it published before going +// down still names it as the host, and waiting there never completes. func (ec *EdgeCluster) PurgeApplication(appUUID uuid.UUID, volumeGen VolumeGenerationPolicy, - waitUntilPurged bool, timeout time.Duration) { + waitUntilPurged bool, timeout time.Duration, excludeDevNames ...string) { ec.checkDevices("PurgeApplication") - hostDev := ec.FindDeviceHostingApp(appUUID, timeout) + hostDev := ec.FindDeviceHostingApp(appUUID, timeout, excludeDevNames...) ec.forEachDeviceExcept(hostDev, func(dev *EdgeDevice) { dev.PurgeApplication(appUUID, volumeGen, false, 0) }) From b7b4ed622cdbe7bb67ffa4d1fe2417b1924111fd Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 17 Sep 2026 16:10:40 -0700 Subject: [PATCH 3/4] evetest test: purge with the designated node down A purge must not gate its teardown on the app's designated node, nor on wherever a replica happens to be scheduled: neither signal is both durable and liveness-aware, so either one deadlocks a purge issued while the designated node is down. Add a three-node cluster test for that case. The app is deployed with a preferred designated node, that node is powered off so KubeVirt reschedules the replica elsewhere, and the purge is issued while it is still down; the surviving workload must be exactly one VMIRS, named for the new generation. The outage threshold that lets a peer act for a downed designated node is pinned past the whole test, so what the purge exercises is unambiguously the path that needs no such stand-in. The purge is observed on the node hosting the app rather than through the controller. While the designated node is down no node uploads this app's state at all, so PURGING and HALTING happen where nothing reports them; waiting on controller-visible state would time out on a purge that ran correctly. Waiting for the new generation's VMIRS covers both that the purge finished and the end state asserted here. Rescheduling is given twenty minutes rather than ten. Detaching the old workload scales the previous VMIRS down through KubeVirt's validating webhook, so a reschedule cannot finish until virt-api is serving again on a surviving node; observed between 7m47s and over ten minutes. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 (1M context) --- .../cluster/purge_during_failover_test.go | 427 ++++++++++++++++++ evetest/tests/cluster/testsuite_test.go | 10 +- 2 files changed, 435 insertions(+), 2 deletions(-) create mode 100644 evetest/tests/cluster/purge_during_failover_test.go diff --git a/evetest/tests/cluster/purge_during_failover_test.go b/evetest/tests/cluster/purge_during_failover_test.go new file mode 100644 index 00000000000..88e2708744c --- /dev/null +++ b/evetest/tests/cluster/purge_during_failover_test.go @@ -0,0 +1,427 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cluster_test + +import ( + "fmt" + "strconv" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// Timeouts for this test, named for it so the package can hold more than one +// set. Eventually returns as soon as its condition holds, so a generous budget +// costs nothing when the system behaves and only delays a genuine failure; +// each is set from an observed duration with headroom, never tuned to +// just-barely-pass. +const ( + // failoverClusterFormationTimeout bounds a three-node cluster forming, which is + // slower than one node joining itself. + failoverClusterFormationTimeout = 30 * time.Minute + + // failoverAppReadyTimeout bounds WaitUntilAppIsRunning. Excludes image download, + // which the framework accounts for separately. + failoverAppReadyTimeout = 10 * time.Minute + + // failoverRescheduleTimeout bounds KubeVirt rescheduling a replica after + // its node is powered off. Dominated by node-not-ready detection and by + // KubeVirt's own control plane recovering: detaching the old workload + // scales the previous VMIRS down, which goes through the + // virtualmachinereplicaset validating webhook, so the reschedule cannot + // finish until virt-api is serving again on a surviving node. Observed + // between 7m47s and over 10 minutes, the long tail being runs where + // virt-api had to come back first. + failoverRescheduleTimeout = 20 * time.Minute + + // failoverPurgeIssueTimeout bounds resolving which node hosts the app so + // the purge can be addressed to it. The purge counter is then pushed to + // every device without waiting; failoverPurgeObservedTimeout covers the + // purge itself. + failoverPurgeIssueTimeout = 5 * time.Minute + + // failoverPurgeObservedTimeout bounds the whole purge as seen from the + // hosting node: the old generation torn down through to the new + // generation's VMIRS existing. + failoverPurgeObservedTimeout = 15 * time.Minute + + // failoverPollInterval is how often the end-state assertion re-checks. + failoverPollInterval = 5 * time.Second + + // failoverVMIRSListTimeout bounds one kubectl invocation run over SSH. + failoverVMIRSListTimeout = 20 * time.Second +) + +const ( + // failoverThresholdKey configures how long an app's designated node has + // to have been unhealthy before another cluster node may act on that app + // in its place. Referenced by its literal string: pillar names it in a Go + // constant only where that takeover is implemented, which this test does + // not require to be present. + failoverThresholdKey = types.GlobalSettingKey("cluster.dnid.backupnode.threshold") + + // failoverThresholdPin is what this test pins that setting to: several + // times the longest it can hold the node down (failoverRescheduleTimeout + // plus failoverPurgeObservedTimeout), and inside the range the setting + // accepts. + failoverThresholdPin = 2 * time.Hour +) + +// failoverDeviceRequirements is clusterDeviceRequirements with the device +// always re-created from scratch. What this test asserts is precisely which +// generations of a workload exist, so a warm device carrying a previous test's +// purge counters would make a false pass indistinguishable from a true one. +func failoverDeviceRequirements(devName string, withTPM bool, + filesystem evetest.Filesystem) evetest.RequireEdgeDevice { + req := clusterDeviceRequirements(devName, withTPM, filesystem, false) + req.DeviceReusePolicy = evetest.CreateFromScratchWithLiveImage + return req +} + +// vmShimApplication returns the app fixture: the standard evetest-ubuntu-ctr +// container run with VirtualizationMode=HVM. HVM, rather than the +// container-native NOHYPER default, is what makes domainmgr's kube path create +// a VMIRS (hypervisor/kubevirt.go CreateReplicaVMIConfig) instead of a plain +// pod, so this "shim VM" is the cheapest fixture that exercises the VMIRS +// lifecycle a purge has to drive. +// +// The forwarded SSH port is not used here - this test makes no guest-level +// assertion. It is part of the fixture so that this app is the same one +// tests/apps runs its purge tests against, and a difference in purge behavior +// cannot be put down to a difference in the app. +func vmShimApplication( + displayName string, niUUID uuid.UUID) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: displayName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: "lfedge/evetest-ubuntu-ctr", + Tag: "1.0", + }, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: 1, + MemoryBytes: 500 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: 2222, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + } +} + +// appPurgeCounter reads the persisted purge counter zedmanager keeps for +// appUUID (pkg/pillar/types.UuidToNum, NumType "purgeCmdCounter"). It is not +// republished anywhere in the EVE API, so it is read from the persisted pubsub +// state. +// +// found is false only if the record could not be read: zedmanager allocates it +// when it first handles the app's config, so it exists from well before the +// app's first purge, holding 0. +func appPurgeCounter( + dev *evetest.EdgeDevice, appUUID uuid.UUID) (counter uint32, found bool) { + var rec types.UuidToNum + if err := evetest.ReadPublication( + dev, "zedmanager", true, appUUID.String(), &rec); err != nil { + // Absent before the app's first purge, which is expected; a transient + // read failure lands here too and the caller's retry absorbs it. + return 0, false + } + return uint32(rec.Number), true +} + +// assertExactlyOneVMIRSAtGeneration is this test's end-state detector: after a +// purge to newCounter there must be exactly one VMIRS for the app, and it must +// be named for the NEW generation - not the old one (a stalled purge leaves the +// old generation's VMIRS alone) and not both (the old generation's VMIRS +// surviving alongside a newly created one). +func assertExactlyOneVMIRSAtGeneration( + g Gomega, dev *evetest.EdgeDevice, appUUID uuid.UUID, appDisplayName string, + newCounter uint32) { + // base.GetAppKubeNameWithPurge would be the exact match for this (name + "-" + // + purge counter), but it is newer than the pillar module version currently + // pinned by evetest's go.mod, so the suffix is appended here instead - see + // base.GetAppKubeNameWithPurge's own implementation for why this is exactly + // equivalent. + wantName := base.GetAppKubeName(appDisplayName, appUUID) + "-" + + strconv.FormatUint(uint64(newCounter), 10) + names, err := dev.ListAppVMIRS(appUUID, failoverVMIRSListTimeout) + g.Expect(err).ToNot(HaveOccurred(), + "could not list VMIRS objects; k3s may still be starting") + g.Expect(names).To(HaveLen(1), + "expected exactly one VMIRS for the app, found %v", names) + if len(names) == 1 { + g.Expect(names[0]).To(Equal(wantName), + "the surviving VMIRS must be the NEW generation %q, not a stale one", wantName) + } +} + +// TestVMAppPurgeDuringFailover exercises a purge issued after the app's +// designated node has failed over: the app's designated node is powered +// off, KubeVirt reschedules the replica onto a different node, and a purge +// is then issued while the designated node is still down. The purge must +// not wait on the dead node: gating the teardown on the app's designated +// node, or on where a replica currently happens to be scheduled, would +// deadlock exactly this case, because neither signal is both durable and +// liveness-aware on its own. +// +// Network model +// ------------- +// - netmodels.SeparateClusterPort -- six ports (two per device): eth0 +// ports share a management+app SDN bridge with DHCP and controller +// reachability; eth1 ports share a separate cluster-only bridge used +// for inter-node K3s traffic. +// +// Device configuration +// -------------------- +// - Three failoverDeviceRequirements devices - clusterDeviceRequirements +// always re-created from scratch - following TestThreeNodesCluster's +// topology. +// - ClusterConfig (REPLICATED_STORAGE) with three ClusterNode entries on +// 10.244.244.0/24; node 1 is the bootstrap node. +// - One Local NI "local-ni" (10.11.14.0/24) and one shim-VM app +// (vmShimApplication) with DesignatedNodeName=devName[0] (node 1) and +// Affinity=PREFERRED. +// - failoverThresholdKey pinned past the test's own duration, so no peer +// ever becomes eligible to act for the powered-off designated node. The +// purge asserted below is the one that needs no such stand-in. +// +// Test parameters +// --------------- +// - TPM via evetest.TPMParameter(). +// - FILESYSTEM (ext4|zfs, defaults to ext4) via evetest.FilesystemParameter(). +// +// Phases +// ------ +// 1. setup-done -> nodes-are-ready: bring up the three-node cluster. +// 2. app-is-deployed: deploy the app; assert it is in fact running on its +// preferred (DNID) node, node 1, while node 1 is healthy. +// 3. dnid-node-powered-off: EdgeDevice.PowerOff() on node 1. +// 4. failed-over: EdgeCluster.FindDeviceHostingApp with node 1 excluded waits +// for KubeVirt to reschedule the replica onto node 2 or node 3. The +// exclusion matters: without it the powered-off node's own stale cluster +// info still names it as the host and would be returned immediately. +// 5. purge-issued: EdgeCluster.PurgeApplication bumps the purge counter on +// every device, including the powered-off node 1 - EdgeDevice.ApplyConfig's +// push does not require device reachability - and returns without waiting. +// The controller cannot be asked whether the purge ran: with the +// designated node down no node uploads this app's state at all, so the +// PURGING and HALTING it would be waited on for are never reported. See +// the comment at the call. +// 6. purge-end-state-asserted: exactly one VMIRS, named for the NEW +// generation, read from the node that now hosts the app - the purge having +// finished and the end state, in one wait. No volume or +// guest-level assertion is made here (VolumeStatus is a per-node ephemeral +// publication and its +// clustered/replicated-storage semantics across a node failover have +// not been established for this suite; the app's forwarded SSH port on +// the new host has not been either). +// +// Node 1 is powered back on from a defer armed at step 3, so cluster teardown +// never has to reason about an already-off device -- including when an +// assertion above aborts the test body. +// +// Suite placement +// --------------- +// - TestNodeClusterSuite, after the other three-node subtests: it powers a +// node off and needs a cluster of its own, so it is the most expensive one +// to set up. +func TestVMAppPurgeDuringFailover(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.TPMParameter(), + evetest.FilesystemParameter(), + ) + withTPM := evetest.GetTPMParameterValue() + filesystem := evetest.GetFilesystemParameterValue() + + var requiredDevices [3]evetest.Requirement + var devName [3]string + for i := 0; i < 3; i++ { + devName[i] = fmt.Sprintf("edge-dev%d", i+1) + requiredDevices[i] = failoverDeviceRequirements(devName[i], withTPM, filesystem) + } + requiredNetModel := evetest.RequireNetworkModel{ + NetworkModel: netmodels.SeparateClusterPort(devName[:]...), + } + var requirements []evetest.Requirement + requirements = append(requirements, requiredDevices[:]...) + requirements = append(requirements, requiredNetModel) + evetest.Setup(requirements...) + evetest.Checkpoint("setup-done") + + var nodes [3]evetest.ClusterNode + for i := 0; i < 3; i++ { + clusterIP := evetest.IPAddressWithPrefix(fmt.Sprintf("10.244.244.%d/24", i+2)) + nodes[i] = evetest.ClusterNode{ + DevName: devName[i], + ClusterIP: clusterIP, + ClusterInterface: "ethernet1", + BootstrapNode: i == 0, + } + } + clusterConfig := evetest.NewEdgeClusterConfig( + eveconfig.ClusterType_CLUSTER_TYPE_REPLICATED_STORAGE, + nodes[:]..., + ) + + // The default threshold is ten minutes, which is not margin enough to say + // that the purge below ran without a peer standing in for the downed node: + // the failover wait alone is allowed ten minutes, so a slow failover would + // leave the purge running just as a peer became eligible. Pinning the + // threshold past the whole test removes the overlap. A device that does + // not implement the setting records a parse error for the item and applies + // the rest of the config, and has no such takeover to begin with. + cfgProps := types.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(failoverThresholdKey, + uint32(failoverThresholdPin.Seconds())) + clusterConfig.SetConfigProperties(cfgProps) + + dhcpNet := clusterConfig.AddNetwork( + evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + noIPNet := clusterConfig.AddNetwork(evetest.NoIPNetworkConfig{}) + clusterConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + clusterConfig.AddNetworkAdapter( + evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet1", + PhysicalLabel: "eth1", + InterfaceName: "eth1", + NetworkUUID: noIPNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageShared, + }) + + cluster := evetest.NewEdgeCluster("purge-failover-cluster") + cluster.ApplyConfig(clusterConfig, true, true) + evetest.Checkpoint("initial-config-applied") + + cluster.WaitUntilNodesAreReady(failoverClusterFormationTimeout) + evetest.Checkpoint("nodes-are-ready") + + niUUID := clusterConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: "local-ni", + Port: "ethernet0", + Subnet: evetest.IPSubnet("10.11.14.0/24"), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.14.2"), + End: evetest.IPAddress("10.11.14.254"), + }, + Gateway: evetest.IPAddress("10.11.14.1"), + EnableFlowlog: true, + MTU: 1500, + ForwardLLDP: false, + }) + const appDisplayName = "failover-purge-app" + appUUID := clusterConfig.AddApplication(evetest.ClusterApplicationInstanceConfig{ + ApplicationInstanceConfig: vmShimApplication(appDisplayName, niUUID), + DesignatedNodeName: devName[0], + Affinity: eveconfig.AffinityType_AFFINITY_TYPE_PREFERRED, + }) + cluster.ApplyConfig(clusterConfig, true, true) + log := evetest.Logger() + log.Infof("Submitted config with application UUID=%v, DNID node=%q", appUUID, devName[0]) + evetest.Checkpoint("app-config-is-submitted") + + cluster.WaitUntilAppIsRunning(appUUID, failoverAppReadyTimeout) + evetest.Checkpoint("app-is-deployed") + + initialHost := cluster.FindDeviceHostingApp(appUUID, time.Minute) + t.Expect(initialHost.Name()).To(Equal(devName[0]), + "app should have been scheduled onto its preferred (DNID) node while it is healthy") + + dnidDevice := evetest.GetEdgeDevice(devName[0]) + baselineCounter, found := appPurgeCounter(dnidDevice, appUUID) + t.Expect(found).To(BeTrue(), + "zedmanager allocates the purge-counter record when it first handles the "+ + "app config, so it exists long before any purge; not finding it means "+ + "the read failed, and the baseline would silently be 0") + t.Expect(baselineCounter).To(BeZero(), + "the app has not been purged yet, so its counter must still be 0; "+ + "anything else means the baseline is not the one the assertions assume") + + // Whether the pin took effect is only visible in what each device reports + // back for the key: a value means it holds the pin, an error means it does + // not know the setting. + for _, name := range devName { + item := evetest.GetEdgeDevice(name).GetDeviceInfo().GetConfigItemStatus(). + GetConfigItems()[string(failoverThresholdKey)] + log.Infof("Device %q reports %s=%q (error: %q)", name, + failoverThresholdKey, item.GetValue(), item.GetError()) + } + + log.Infof("Powering off DNID node %q to force a failover", devName[0]) + dnidDevice.PowerOff() + // Deferred rather than done at the end of the test: an assertion failure + // below aborts the test body, and cluster teardown then has to reason about + // a member that is still powered off. + defer func() { + log.Infof("Powering DNID node %q back on", devName[0]) + dnidDevice.PowerOn(true) + }() + evetest.Checkpoint("dnid-node-powered-off") + + failoverHost := cluster.FindDeviceHostingApp( + appUUID, failoverRescheduleTimeout, devName[0]) + log.Infof("App failed over to device %q", failoverHost.Name()) + evetest.Checkpoint("failed-over") + + // BumpVolumeGeneration: the purge under test is the one that increments + // the generation on the volume the cluster already replicates, not the + // fresh-UUID path. devName[0] is powered off, so it is excluded from the + // search for the node the purge is addressed to. + // + // Not waitUntilPurged: that waits on the app's state as the CONTROLLER + // sees it, and no node uploads this app's state while its designated node + // is down. zedmanager elects one reporter - the node the pod is scheduled + // on, else the designated node - and a purge renames the pod it looks for + // (GetAppKubeNameWithPurge embeds the purge counter), so every node finds + // no pod and defers to a node that is powered off. PURGING and HALTING + // happen where nothing uploads them. + cluster.PurgeApplication(appUUID, evetest.BumpVolumeGeneration, false, + failoverPurgeIssueTimeout, devName[0]) + evetest.Checkpoint("purge-issued") + + // The new generation's VMIRS existing is both the purge having finished + // and the end state asserted here, so one wait covers both. Read from the + // hosting node, which stays truthful while no node is reporting. + wantCounter := baselineCounter + 1 + t.Eventually(func(g Gomega) { + assertExactlyOneVMIRSAtGeneration(g, failoverHost, appUUID, appDisplayName, wantCounter) + }, failoverPurgeObservedTimeout, failoverPollInterval).Should(Succeed()) + evetest.Checkpoint("purge-end-state-asserted") +} diff --git a/evetest/tests/cluster/testsuite_test.go b/evetest/tests/cluster/testsuite_test.go index 8e1868c4ab8..503a1964898 100644 --- a/evetest/tests/cluster/testsuite_test.go +++ b/evetest/tests/cluster/testsuite_test.go @@ -14,9 +14,12 @@ import ( // the subtests for efficiency. All subtests pin the device to the Kubevirt // hypervisor (aka eve-k). // -// The single-node subtests run before the three-node one, and the happy-path +// The single-node subtests run before the three-node ones, and the happy-path // purge runs before the fault-injecting VMIRS test, so a failure in the -// ordinary app lifecycle is not masked by chaos. +// ordinary app lifecycle is not masked by chaos. TestVMAppPurgeDuringFailover +// is placed after the other three-node subtests because it is the one that +// builds its cluster from scratch rather than resetting the device config, +// which makes it the most expensive to set up. // // Test parameters // --------------- @@ -56,5 +59,8 @@ func TestNodeClusterSuite(test *testing.T) { evetest.TestCase{ Test: TestClusterToSingleConversion, }, + evetest.TestCase{ + Test: TestVMAppPurgeDuringFailover, + }, ) } From dec41c08361f2258a98c6169c4c8b4db8ec6d62c Mon Sep 17 00:00:00 2001 From: eriknordmark Date: Thu, 17 Sep 2026 16:10:50 -0700 Subject: [PATCH 4/4] evetest test: work around a Longhorn CSI stall A PVC can stay Pending indefinitely while its ProvisioningFailed events alternate "not found" and "already exists": the provisioner created the backend volume, lost that fact from its own cache, and retries forever against a name it no longer recognizes. Deleting the csi-provisioner pod forces a fresh leader election and the next retry succeeds. Watch for that signature while the failover purge test waits for its app, and restart the pod once if it holds for two minutes. Scoped to that one test, which is where it has been observed and which owns the cluster it would restart the pod on. Delete this file once Longhorn recovers on its own. Signed-off-by: eriknordmark Co-Authored-By: Claude Opus 5 (1M context) --- .../longhorn_provisioner_workaround_test.go | 214 ++++++++++++++++++ .../cluster/purge_during_failover_test.go | 7 +- 2 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 evetest/tests/cluster/longhorn_provisioner_workaround_test.go diff --git a/evetest/tests/cluster/longhorn_provisioner_workaround_test.go b/evetest/tests/cluster/longhorn_provisioner_workaround_test.go new file mode 100644 index 00000000000..beb3ed5e405 --- /dev/null +++ b/evetest/tests/cluster/longhorn_provisioner_workaround_test.go @@ -0,0 +1,214 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// ============================================================================ +// REMOVE ME +// ============================================================================ +// +// This file works around an infra bug in Longhorn's CSI provisioner. It is +// not related to anything this suite tests. Delete this file and the one +// call to waitForAppRunningMitigatingPVCStall (grep for it, in +// purge_during_failover_test.go) once Longhorn no longer needs a pod restart +// to recover from a stuck PVC. +// +// Observed symptom: a PVC stays Pending indefinitely. Its ProvisioningFailed +// events alternate "volume ... not found" (404) and "volume ... already +// exists" (500) - the provisioner created the Longhorn backend volume once, +// lost track of that success in its own cache, and keeps retrying against a +// name it no longer recognizes. +// +// Confirmed live, on the current Longhorn version, with no EVE or pillar +// change involved: deleting the csi-provisioner pod forces a fresh leader +// election and cache, and the very next retry succeeds +// (ProvisioningSucceeded within minutes). +// +// Only wired into TestVMAppPurgeDuringFailover: that is the one test this has +// actually been observed to fail, and restarting a cluster-wide Longhorn pod +// is not something to do reflexively from every test that creates a volume. +// +// Scope note: this deletes a cluster-wide Longhorn pod. Safe here because +// TestVMAppPurgeDuringFailover forms its own freshly-created cluster +// (failoverDeviceRequirements), so nothing else is using it. Do not call this +// against a shared or long-lived cluster. +package cluster_test + +import ( + "encoding/json" + "strings" + "time" + + "github.com/lf-edge/eve/evetest" +) + +// pvcStallSignature is the substring both ends of the failure oscillation +// share. "not found" alone is not distinctive enough to key on - a PVC can +// legitimately report "not found" for a moment during normal provisioning. +const pvcStallSignature = "already exists" + +// pvcStallCheckInterval is how often waitForAppRunningMitigatingPVCStall +// polls for the stuck-PVC signature while waitFn is blocked. +const pvcStallCheckInterval = 30 * time.Second + +// pvcStallKubectlTimeout bounds a single kubectl invocation made from here. +const pvcStallKubectlTimeout = 20 * time.Second + +// pvcStallThreshold is how long a PVC must show the signature, continuously, +// before this mitigation acts. A PVC that clears - becomes Bound, or simply +// stops matching the signature - before this elapses is a normal, if slow, +// provisioning retry; passing through and leaving it alone is the point. +const pvcStallThreshold = 2 * time.Minute + +// waitForAppRunningMitigatingPVCStall wraps waitFn - normally +// cluster.WaitUntilAppIsRunning - with a background watcher that restarts +// Longhorn's csi-provisioner if a PVC shows the stuck-PVC signature for at +// least pvcStallThreshold while waitFn is blocked. +// +// waitFn still Fatalf's on its own timeout exactly as it would unwrapped; +// this only gives the provisioner a chance to recover before that timeout. +// kubeDev is any device that can reach the cluster's kubectl - for a cluster +// test, any member node. +func waitForAppRunningMitigatingPVCStall(kubeDev *evetest.EdgeDevice, waitFn func()) { + stop := make(chan struct{}) + done := make(chan struct{}) + // Deferred so the watcher also stops when waitFn Fatalf's (Goexit). + defer func() { + close(stop) + <-done + }() + go func() { + defer close(done) + ticker := time.NewTicker(pvcStallCheckInterval) + defer ticker.Stop() + firstSeenStalled := map[string]time.Time{} + kicked := false + for { + select { + case <-stop: + return + case <-ticker.C: + if kicked { + // One restart per wait is enough. Retrying it on every + // tick would fight a genuinely slow (but healthy) + // provision with unnecessary leader-election churn. + continue + } + now := time.Now() + stalled := stalledPVCNames(kubeDev) + // A PVC no longer in the stalled set recovered on its own - + // forget it, so a later, unrelated stall starts its own + // clock rather than inheriting an old one. + for name := range firstSeenStalled { + if !stalled[name] { + delete(firstSeenStalled, name) + } + } + for name := range stalled { + if _, tracked := firstSeenStalled[name]; !tracked { + firstSeenStalled[name] = now + } + } + for name, since := range firstSeenStalled { + if now.Sub(since) < pvcStallThreshold { + continue + } + evetest.Logger().Warnf( + "waitForAppRunningMitigatingPVCStall: PVC %q stuck for over %s, "+ + "restarting csi-provisioner - see the REMOVE ME note in "+ + "longhorn_provisioner_workaround_test.go", name, pvcStallThreshold) + restartCSIProvisioner(kubeDev) + kicked = true + break + } + } + } + }() + waitFn() +} + +// restartCSIProvisioner deletes Longhorn's csi-provisioner pod(s), forcing a +// fresh leader election and cache. +func restartCSIProvisioner(dev *evetest.EdgeDevice) { + if _, err := dev.RunKubectl( + "-n longhorn-system delete pod -l app=csi-provisioner", + pvcStallKubectlTimeout); err != nil { + evetest.Logger().Warnf("restartCSIProvisioner: %v", err) + } +} + +// kubectlGetJSON runs `kubectl -n get -o json` and +// decodes it into out. Reported as found=false on any failure: this only ever +// feeds the stall clock below, which a transient kubectl error must never +// advance. +func kubectlGetJSON( + dev *evetest.EdgeDevice, namespace, resource string, out any) (found bool) { + stdout, err := dev.RunKubectl( + "-n "+namespace+" get "+resource+" -o json", pvcStallKubectlTimeout) + if err != nil { + evetest.Logger().Warnf("kubectlGetJSON: %v", err) + return false + } + if err := json.Unmarshal([]byte(stdout), out); err != nil { + evetest.Logger().Warnf( + "kubectlGetJSON: failed to parse kubectl %s output: %v", resource, err) + return false + } + return true +} + +// stalledPVCNames returns the names of PVCs that are currently Pending and +// have a ProvisioningFailed event carrying pvcStallSignature. Checking events +// against PVCs still Pending, rather than events alone, means a PVC that has +// since become Bound is never counted, even though its old events persist +// for a while - that is exactly the "passes through once bound" behavior +// this mitigation is meant to have. +// +// Empty on any read failure - a transient kubectl error must never +// contribute to the stall clock. +func stalledPVCNames(dev *evetest.EdgeDevice) map[string]bool { + stalled := map[string]bool{} + + var pvcs struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Phase string `json:"phase"` + } `json:"status"` + } `json:"items"` + } + if !kubectlGetJSON(dev, evetest.EVEKubeAppNamespace, "pvc", &pvcs) { + return stalled + } + pending := make(map[string]bool) + for _, item := range pvcs.Items { + if item.Status.Phase == "Pending" { + pending[item.Metadata.Name] = true + } + } + if len(pending) == 0 { + return stalled + } + + var events struct { + Items []struct { + Reason string `json:"reason"` + Message string `json:"message"` + InvolvedObject struct { + Name string `json:"name"` + } `json:"involvedObject"` + } `json:"items"` + } + if !kubectlGetJSON(dev, evetest.EVEKubeAppNamespace, "events", &events) { + return stalled + } + for _, ev := range events.Items { + if ev.Reason != "ProvisioningFailed" || !pending[ev.InvolvedObject.Name] { + continue + } + if strings.Contains(ev.Message, pvcStallSignature) { + stalled[ev.InvolvedObject.Name] = true + } + } + return stalled +} diff --git a/evetest/tests/cluster/purge_during_failover_test.go b/evetest/tests/cluster/purge_during_failover_test.go index 88e2708744c..c2cab51fa1e 100644 --- a/evetest/tests/cluster/purge_during_failover_test.go +++ b/evetest/tests/cluster/purge_during_failover_test.go @@ -357,7 +357,12 @@ func TestVMAppPurgeDuringFailover(test *testing.T) { log.Infof("Submitted config with application UUID=%v, DNID node=%q", appUUID, devName[0]) evetest.Checkpoint("app-config-is-submitted") - cluster.WaitUntilAppIsRunning(appUUID, failoverAppReadyTimeout) + // Any cluster member can reach kubectl; devName[0] is as good as any for + // the mitigation's own queries while the wait below is blocked. + kubeDev := evetest.GetEdgeDevice(devName[0]) + waitForAppRunningMitigatingPVCStall(kubeDev, func() { + cluster.WaitUntilAppIsRunning(appUUID, failoverAppReadyTimeout) + }) evetest.Checkpoint("app-is-deployed") initialHost := cluster.FindDeviceHostingApp(appUUID, time.Minute)