evetest: purge with the designated node down

evetest/edgecluster.go

@@ -395,10 +395,14 @@ func (ec *EdgeCluster) RebootApplication(appUUID uuid.UUID, waitUntilRebooted bo
 // The purge counter is incremented on all devices, but the wait (if requested)
 // is performed only on the device hosting the application. See
 // EdgeDevice.PurgeApplication for volumeGen.
+//
+// Name in excludeDevNames any node that must not be taken for the host, such as
+// one the caller has powered off: the cluster info it published before going
+// down still names it as the host, and waiting there never completes.
 func (ec *EdgeCluster) PurgeApplication(appUUID uuid.UUID, volumeGen VolumeGenerationPolicy,
-	waitUntilPurged bool, timeout time.Duration) {
+	waitUntilPurged bool, timeout time.Duration, excludeDevNames ...string) {
 	ec.checkDevices("PurgeApplication")
-	hostDev := ec.FindDeviceHostingApp(appUUID, timeout)
+	hostDev := ec.FindDeviceHostingApp(appUUID, timeout, excludeDevNames...)
 	ec.forEachDeviceExcept(hostDev, func(dev *EdgeDevice) {
 		dev.PurgeApplication(appUUID, volumeGen, false, 0)
 	})

evetest/edgedevice.go

@@ -2769,6 +2769,85 @@ func (d *EdgeDevice) WatchNTPSources() (
 	return ch, d.trackWatcherUnsub(unsub)
 }
 
+// EVEKubeAppNamespace is the Kubernetes namespace EVE runs app workloads in;
+// both VMIRS objects and their PVCs live there. See
+// pkg/pillar/kubeapi.EVEKubeNameSpace.
+const EVEKubeAppNamespace = "eve-kube-app"
+
+// appDomainNameLabel holds the owning app's DomainName,
+// "<uuid>.<version>.<appnum>" (the eveLabelKey constant in
+// hypervisor/kubevirt.go). EVE puts this label in the VMIRS
+// spec.selector.matchLabels and in the VMI template, but not in the VMIRS
+// metadata.labels, so attribution reads the selector - the same way pillar
+// attributes a VMIRS in sweepStaleGenerations.
+const appDomainNameLabel = "App-Domain-Name"
+
+// RunKubectl runs kubectl on the device with the given arguments and returns
+// its trimmed standard output. The call goes through "eve exec kube" because
+// kubectl exists only in the kube container, so the device has to be an EVE-K
+// node.
+//
+// The returned error carries kubectl's own standard error, which says why the
+// command was refused. A caller polling a cluster that is still coming up
+// should treat any error as a retryable "not yet": kubectl fails this way
+// while k3s is starting, and that is indistinguishable here from a real
+// refusal.
+func (d *EdgeDevice) RunKubectl(args string, timeout time.Duration) (string, error) {
+	stdout, stderr, err := d.RunShellScript(
+		"eve exec kube kubectl "+args, timeout, 0)
+	if err != nil {
+		return "", fmt.Errorf("kubectl %s: %w (stderr: %s)",
+			args, err, strings.TrimSpace(stderr))
+	}
+	return strings.TrimSpace(stdout), nil
+}
+
+// ListAppVMIRS returns the sorted names of every VMIRS belonging to appUUID,
+// whichever purge generation each one is. A VMIRS name embeds the app's purge
+// counter (base.GetAppKubeNameWithPurge), which makes this the one vantage
+// point from which a stale generation surviving a purge is observable at all:
+// pillar's own DomainStatus is keyed by app UUID and so can only ever describe
+// one of them.
+//
+// Attribution is by the appDomainNameLabel selector rather than the name,
+// since the label value also carries a version and an appnum that do not
+// matter here. An empty list with a nil error means the app has no VMIRS; an
+// error means the list could not be read, which a caller inside an Eventually
+// should retry rather than fail on.
+func (d *EdgeDevice) ListAppVMIRS(
+	appUUID uuid.UUID, timeout time.Duration) ([]string, error) {
+	stdout, err := d.RunKubectl(
+		"-n "+EVEKubeAppNamespace+" get vmirs -o json", timeout)
+	if err != nil {
+		return nil, err
+	}
+	var list struct {
+		Items []struct {
+			Metadata struct {
+				Name string `json:"name"`
+			} `json:"metadata"`
+			Spec struct {
+				Selector struct {
+					MatchLabels map[string]string `json:"matchLabels"`
+				} `json:"selector"`
+			} `json:"spec"`
+		} `json:"items"`
+	}
+	if err := json.Unmarshal([]byte(stdout), &list); err != nil {
+		return nil, fmt.Errorf("parsing kubectl vmirs output: %w", err)
+	}
+	prefix := appUUID.String() + "."
+	var names []string
+	for _, item := range list.Items {
+		if strings.HasPrefix(
+			item.Spec.Selector.MatchLabels[appDomainNameLabel], prefix) {
+			names = append(names, item.Metadata.Name)
+		}
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
 // GetClusterInfo returns the last recorded information about the Kubernetes
 // cluster, or nil if no such info message has been received yet.
 func (d *EdgeDevice) GetClusterInfo() *eveinfo.ZInfoKubeCluster {

evetest/tests/apps/appstate_helpers_test.go

@@ -63,8 +63,8 @@ func appPurgePhase(dev *evetest.EdgeDevice, appUUID uuid.UUID) (
 
 // appDomainStatus returns domainmgr's published DomainStatus for the app. There
 // is at most one, because DomainStatus is keyed by app UUID - which is exactly
-// why it cannot be used to count workload generations (see listAppVMIRS and
-// listKVMDomainDirs in appworkload_helpers_test.go). It is authoritative for the
+// why it cannot be used to count workload generations (see
+// EdgeDevice.ListAppVMIRS and listKVMDomainDirs in appworkload_helpers_test.go). It is authoritative for the
 // domain's id, name and attached disks.
 func appDomainStatus(
 	dev *evetest.EdgeDevice, appUUID uuid.UUID) (types.DomainStatus, bool) {
@@ -77,8 +77,9 @@ func appDomainStatus(
 // before the old generation is actually gone - and it is not republished
 // anywhere in the EVE API, so it is read from the persisted pubsub state.
 //
-// found is false while the file does not exist, which is the expected state
-// before an app's first purge.
+// found is false only if the record could not be read: zedmanager allocates it
+// when it first handles the app's config, so it exists from well before the
+// app's first purge, holding 0.
 func purgeCounter(
 	dev *evetest.EdgeDevice, appUUID uuid.UUID) (counter uint32, found bool) {
 	var rec types.UuidToNum

evetest/tests/apps/appworkload_helpers_test.go

@@ -4,19 +4,19 @@
 // Where and whether the app is actually running, as the hypervisor itself sees
 // it: VMIRS objects on eve-k, qemu domain state directories on kvm/xen.
 //
-// Rule for this file: readers that enumerate the app's WORKLOAD instances, plus
-// the kubectl plumbing they are built on. This is the one place a stale
-// generation is observable, because pillar's own DomainStatus is keyed by app
-// UUID and so can only ever describe one (appstate_helpers_test.go).
+// Rule for this file: readers that enumerate the app's WORKLOAD instances. This
+// is the one place a stale generation is observable, because pillar's own
+// DomainStatus is keyed by app UUID and so can only ever describe one
+// (appstate_helpers_test.go).
 //
 // Everything generic - reading a file, listing a directory, testing for a path,
-// flushing caches - is an EdgeDevice method in the framework
-// (evetest/edgedevice.go); this file only adds what is specific to EVE's
-// workload objects. Note the two error conventions that follow from that:
-// helpers backed by a framework method surface its error to Gomega, because an
-// error there is a transport failure, while a failed kubectl call is reported as
-// found=false, because "k3s is not up yet" is an expected transient state a
-// caller inside Eventually should retry on rather than fail.
+// flushing caches, reaching the cluster's kubectl - is an EdgeDevice method in
+// the framework (evetest/edgedevice.go); this file only adds what is specific
+// to EVE's workload objects. Note the two error conventions that follow from
+// that: helpers backed by a framework method surface its error to Gomega,
+// because an error there is a transport failure, while a failed kubectl call is
+// reported as found=false, because "k3s is not up yet" is an expected transient
+// state a caller inside Eventually should retry on rather than fail.
 //
 // Split trigger: if a xen-specific reader is ever needed, break this into
 // kube- and local-domain helper files. Two backends sharing one file is
@@ -29,7 +29,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"path"
-	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -45,42 +44,20 @@ const (
 	// sshCmdTimeout bounds a single kubectl invocation run over SSH.
 	sshCmdTimeout = 20 * time.Second
 
-	// eveKubeAppNamespace is the Kubernetes namespace EVE runs app workloads in;
-	// both VMIRS objects and their PVCs live there. See
-	// pkg/pillar/kubeapi.EVEKubeNameSpace.
-	eveKubeAppNamespace = "eve-kube-app"
-
-	// appDomainNameLabel holds the owning app's DomainName,
-	// "<uuid>.<version>.<appnum>". See the eveLabelKey constant in
-	// hypervisor/kubevirt.go.
-	//
-	// EVE puts this label in the VMIRS spec.selector.matchLabels and in the VMI
-	// template, but not in the VMIRS metadata.labels. Read the selector. Pillar
-	// attributes a VMIRS the same way, in sweepStaleGenerations.
-	appDomainNameLabel = "App-Domain-Name"
-
 	// kvmDomainStateDir mirrors hypervisor/kvm.go's kvmStateDir: qemu gets one
 	// directory per domain, holding that domain's pidfile. EVE bind-mounts /run
 	// into the pillar container, so this path is readable from dom0 - the same
 	// assumption evetest.ReadAllPublications already makes for /run/<agent>.
 	kvmDomainStateDir = "/run/hypervisor/kvm"
 )
 
-// kubeItemList is the minimal shape needed from any `kubectl get <resource>
-// -o json`: a name per item, the item's own labels, and the selector labels.
-// A VMIRS is attributed by its selector, which is the only place EVE puts the
-// App-Domain-Name label. A PVC has neither field.
+// kubeItemList is the minimal shape needed from `kubectl get <resource> -o
+// json`: a name per item.
 type kubeItemList struct {
 	Items []struct {
 		Metadata struct {
-			Name   string            `json:"name"`
-			Labels map[string]string `json:"labels"`
+			Name string `json:"name"`
 		} `json:"metadata"`
-		Spec struct {
-			Selector struct {
-				MatchLabels map[string]string `json:"matchLabels"`
-			} `json:"selector"`
-		} `json:"spec"`
 	} `json:"items"`
 }
 
@@ -96,21 +73,12 @@ type kubeItemList struct {
 // publishes carries only the single name of the desired generation. Until that
 // gap is closed, this is the only vantage point from which a stale generation
 // surviving a purge is observable at all.
-//
-// Promotion trigger: when a second suite needs kubectl access, move this to an
-// EdgeDevice method in evetest/edgedevice.go. Do not copy it. It is kept local
-// for now because tests/cluster has no kubectl calls at all, so the shape is
-// unsettled after two consumers, and because the framework has no non-fatal
-// read family to fit it into yet.
 func kubectlListItems(
 	dev *evetest.EdgeDevice, resource string) (list kubeItemList, found bool) {
-	stdout, stderr, err := dev.RunShellScript(
-		"eve exec kube kubectl -n "+eveKubeAppNamespace+" get "+resource+" -o json",
-		sshCmdTimeout, 0)
+	stdout, err := dev.RunKubectl(
+		"-n "+evetest.EVEKubeAppNamespace+" get "+resource+" -o json", sshCmdTimeout)
 	if err != nil {
-		evetest.Logger().Warnf(
-			"kubectlListItems: kubectl get %s failed: %v (stderr: %s)",
-			resource, err, stderr)
+		evetest.Logger().Warnf("kubectlListItems: %v", err)
 		return list, false
 	}
 	if err := json.Unmarshal([]byte(stdout), &list); err != nil {
@@ -121,35 +89,11 @@ func kubectlListItems(
 	return list, true
 }
 
-// listAppVMIRS returns the names of every VMIRS (any generation) that belongs to
-// appUUID. It matches the prefix "<uuid>." on the App-Domain-Name selector
-// label, because the label value also carries a version and an appnum that do
-// not matter here. Names are sorted, so a caller can compare the whole set.
-//
-// found is false if the list could not be read. An empty list then means "no
-// VMIRS", and not "the device did not answer".
-func listAppVMIRS(
-	dev *evetest.EdgeDevice, appUUID uuid.UUID) (names []string, found bool) {
-	list, ok := kubectlListItems(dev, "vmirs")
-	if !ok {
-		return nil, false
-	}
-	prefix := appUUID.String() + "."
-	for _, item := range list.Items {
-		if strings.HasPrefix(
-			item.Spec.Selector.MatchLabels[appDomainNameLabel], prefix) {
-			names = append(names, item.Metadata.Name)
-		}
-	}
-	sort.Strings(names)
-	return names, true
-}
-
 // listKVMDomainDirs returns the qemu per-domain state directories belonging to
 // appUUID, found by prefix on the domain name ("<uuid>.<version>.<appnum>", see
 // types.DomainConfig.GetTaskName).
 //
-// Note the asymmetry with listAppVMIRS: a kvm domain name carries no purge
+// Note the asymmetry with EdgeDevice.ListAppVMIRS: a kvm domain name carries no purge
 // counter, so two generations of the same app at the same version would share one
 // directory name and be indistinguishable here. That is also why a surviving
 // generation cannot take this shape on kvm at all. What this does catch is a

evetest/tests/apps/purge_helpers_test.go

@@ -213,8 +213,8 @@ func assertExactlyOneVMIRSAtGeneration(
 	// equivalent.
 	wantName := base.GetAppKubeName(appDisplayName, appUUID) + "-" +
 		strconv.FormatUint(uint64(newCounter), 10)
-	names, found := listAppVMIRS(dev, appUUID)
-	g.Expect(found).To(BeTrue(),
+	names, err := dev.ListAppVMIRS(appUUID, sshCmdTimeout)
+	g.Expect(err).ToNot(HaveOccurred(),
 		"could not list VMIRS objects; k3s may still be starting")
 	g.Expect(names).To(HaveLen(1),
 		"expected exactly one VMIRS for the app, found %v", names)

evetest/tests/apps/testsuite_test.go

@@ -17,8 +17,9 @@
 //	appstate_helpers_test.go     pillar's own view of the app - pubsub and
 //	                             persisted state keyed by app UUID
 //	appworkload_helpers_test.go  where the app is running as the hypervisor sees
-//	                             it - VMIRS objects (and the kubectl plumbing for
-//	                             them), qemu domain state directories
+//	                             it - VMIRS objects, qemu domain state
+//	                             directories, and the kubectl reads the volume
+//	                             helpers share
 //	appvolumes_helpers_test.go   the app's disk in all three forms -
 //	                             VolumeStatus, PVC, file under /persist - and the
 //	                             storage invariants