From 405ec463049576fe02a9fd0928d3270ef0150b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Ollivier?= Date: Tue, 3 Nov 2026 08:59:41 +0100 Subject: [PATCH] Add air-gapped runs and an offline CI job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an offline bundle and cache for every host-side download, so a run can resolve Helm, Kubescape, the Kyverno CLI and policies, the Litmus operator and fault manifests, and the Kyverno policies without network access. - Add Offline and Airgap helpers, airgap_prepare/airgap_status, recursive CNF manifest scanning, and cached image collection. - Fail missing downloads with the URL and bundle path, and skip registry-only checks with a reason instead of waiting on a timeout. - Patch cached Litmus manifests to use local images and include the operator's CHAOS_RUNNER_IMAGE in the collected image list. - Add the airgap workflow: GitHub-hosted runner, kind cluster, blocked egress, preloaded images, and an offline cert run. - Document the bundle and the two offline environment variables, and fix the Renovate image regex for the ClusterTools constant. Verified with crystal build, the airgap/offline specs, and a local airgap_prepare smoke test. Signed-off-by: CΓ©dric Ollivier --- .github/workflows/airgap.yml | 332 ++++++++++++++++++ docs/USAGE.md | 45 +++ renovate.json | 4 +- spec/tasks/airgap_spec.cr | 264 ++++++++++++++ spec/utils/offline_spec.cr | 142 ++++++++ src/modules/cluster_tools/cluster_tools.cr | 5 + .../cluster_tools/templates/manifest.yml.ecr | 3 +- src/tasks/airgap.cr | 288 +++++++++++++++ src/tasks/setup/litmus_setup.cr | 12 +- src/tasks/utils/cli_help.cr | 1 + src/tasks/utils/kyverno.cr | 14 + src/tasks/utils/litmus_manager.cr | 46 ++- src/tasks/utils/offline.cr | 92 +++++ src/tasks/utils/release_upgrade.cr | 22 ++ src/tasks/utils/utils.cr | 43 ++- src/tasks/workload/compatibility.cr | 49 ++- src/tasks/workload/microservice.cr | 5 + src/tasks/workload/security.cr | 20 ++ 18 files changed, 1369 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/airgap.yml create mode 100644 spec/tasks/airgap_spec.cr create mode 100644 spec/utils/offline_spec.cr create mode 100644 src/tasks/airgap.cr create mode 100644 src/tasks/utils/offline.cr diff --git a/.github/workflows/airgap.yml b/.github/workflows/airgap.yml new file mode 100644 index 000000000..fa04c7776 --- /dev/null +++ b/.github/workflows/airgap.yml @@ -0,0 +1,332 @@ +name: airgap +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: airgap-${{ github.ref }} + cancel-in-progress: true + +jobs: + airgap: + name: Air-gapped run + # The iptables rules below are host-wide. On a shared/self-hosted runner + # they can cut off another kind cluster that uses the same Docker network + # and pod CIDR, so this job needs the isolated VM of a GitHub-hosted runner. + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + env: + KIND_CLUSTER_NAME: airgap-${{ github.run_id }} + steps: + - name: Set runner paths + run: | + echo "KUBECONFIG=$RUNNER_TEMP/airgap-kubeconfig" >> "$GITHUB_ENV" + echo "CNTI_TESTSUITE_OFFLINE_DIR=$RUNNER_TEMP/airgap-bundle" >> "$GITHUB_ENV" + - name: Checkout Code + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - name: Install the CNTi toolchain + uses: ./.github/actions/install-cnti-toolchain + - name: Install kind + uses: ./.github/actions/install-kind + - name: Build CNTi Test Suite + run: | + shards install + crystal build src/cnti-testsuite.cr + - name: Prepare the CNF chart for a local install + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/airgap-cnf" + helm pull coredns --repo https://cncf.gitlab.io/stable --version 1.13.8 \ + --untar --untardir "$RUNNER_TEMP/airgap-cnf" + cat > "$RUNNER_TEMP/airgap-cnf/cnti-testsuite.yaml" <<'EOF' + --- + config_version: v2 + common: + container_names: + - name: coredns + rollback_from_tag: 1.8.0 + rolling_update_test_tag: 1.8.0 + rolling_downgrade_test_tag: 1.6.7 + rolling_version_change_test_tag: 1.8.0 + deployments: + helm_dirs: + - name: coredns + helm_directory: coredns + EOF + - name: Prepare the offline bundle + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + ./cnti-testsuite airgap_prepare \ + --cnf-config "$RUNNER_TEMP/airgap-cnf/cnti-testsuite.yaml" + # The coredns chart's image is a template, so airgap_prepare cannot + # read it; render the chart and add what the compatibility tests use. + helm template coredns "$RUNNER_TEMP/airgap-cnf/coredns" \ + | awk '$1 == "image:" { gsub(/"/, "", $2); print $2 }' \ + >> "$CNTI_TESTSUITE_OFFLINE_DIR/images.txt" + printf 'coredns/coredns:1.6.7\ncoredns/coredns:1.8.0\nbusybox:1.36\n' \ + >> "$CNTI_TESTSUITE_OFFLINE_DIR/images.txt" + sort -u -o "$CNTI_TESTSUITE_OFFLINE_DIR/images.txt" \ + "$CNTI_TESTSUITE_OFFLINE_DIR/images.txt" + cat "$CNTI_TESTSUITE_OFFLINE_DIR/images.txt" + ./cnti-testsuite airgap_status + - name: Write the kind cluster config + run: | + cat > "$RUNNER_TEMP/airgap-cluster.yml" <<'EOF' + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + etcd: + local: + dataDir: /tmp/etcd + - role: worker + - role: worker + EOF + - name: Create the kind cluster + uses: ./.github/actions/create-kind-cluster + with: + name: ${{ env.KIND_CLUSTER_NAME }} + config: ${{ runner.temp }}/airgap-cluster.yml + kubeconfig: ${{ runner.temp }}/airgap-kubeconfig + - name: Check for Docker Hub credentials + id: dockerhub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + if [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_PASSWORD" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + - name: Login to Docker Hub + if: steps.dockerhub.outputs.available == 'true' + uses: docker/login-action@v4.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Load the bundle images into kind + run: | + set -euo pipefail + while IFS= read -r image; do + [ -z "$image" ] && continue + echo "Loading $image" + docker pull "$image" + kind load docker-image "$image" --name "$KIND_CLUSTER_NAME" + done < "$CNTI_TESTSUITE_OFFLINE_DIR/images.txt" + - name: Logout of Docker Hub + if: always() && steps.dockerhub.outputs.available == 'true' + run: docker logout + - name: Cut the kind network off from the internet + run: | + set -euo pipefail + # Docker can list the IPv6 subnet first: iptables rejects it with + # "invalid mask '64'", and the IPv4 side would stay open. Block + # every address family the network has with its own table. + KIND_SUBNETS=$(docker network inspect kind \ + -f '{{range .IPAM.Config}}{{println .Subnet}}{{end}}' | tr '\n' ' ') + KIND_SUBNETS=${KIND_SUBNETS% } + if [ -z "$KIND_SUBNETS" ]; then + echo "::error::the kind network has no subnet to block" + exit 1 + fi + echo "KIND_SUBNETS=$KIND_SUBNETS" >> "$GITHUB_ENV" + # Reject first, then prepend the cluster destinations that stay + # reachable. Each rule has one -d: iptables rejects several. + IPV4_KIND_SUBNETS=() + for subnet in $KIND_SUBNETS; do + case "$subnet" in + *:*) + tables=ip6tables + ;; + *) + tables=iptables + IPV4_KIND_SUBNETS+=("$subnet") + ;; + esac + echo "Blocking new connections from $subnet outside the kind network" + sudo "$tables" -I DOCKER-USER -s "$subnet" \ + -m conntrack --ctstate NEW -j REJECT + done + # Pod traffic leaves the node SNATed to the kind network; the IPv4 + # pod range is covered too for a CNI that does not SNAT. + if [ "${#IPV4_KIND_SUBNETS[@]}" -gt 0 ]; then + sudo iptables -I DOCKER-USER -s 10.244.0.0/16 \ + -m conntrack --ctstate NEW -j REJECT + fi + for destination in 10.96.0.0/16 10.244.0.0/16; do + sudo iptables -I DOCKER-USER -d "$destination" -j ACCEPT + done + for subnet in $KIND_SUBNETS; do + case "$subnet" in + *:*) sudo ip6tables -I DOCKER-USER -d "$subnet" -j ACCEPT ;; + *) sudo iptables -I DOCKER-USER -d "$subnet" -j ACCEPT ;; + esac + done + - name: Verify the cluster cannot reach the internet + run: | + set -euo pipefail + kubectl run airgap-check --image=busybox:1.36 --restart=Never \ + --command -- sleep 300 + kubectl wait --for=condition=Ready pod/airgap-check --timeout=120s + if kubectl exec airgap-check -- sh -c 'nc -w 5 1.1.1.1 443 /dev/null 2>&1; then + echo "::error::setpriv is required to run the suite with the airgap group" + exit 1 + fi + HOST_TABLES=(iptables) + if command -v ip6tables >/dev/null 2>&1; then + HOST_TABLES+=(ip6tables) + fi + HOST_RULES_ADDED=0 + remove_host_egress_block() { + if [ "$HOST_RULES_ADDED" = 1 ]; then + for tables in "${HOST_TABLES[@]}"; do + sudo "$tables" -D OUTPUT -m owner --gid-owner "$SUITE_GID" -o lo -j ACCEPT || true + sudo "$tables" -D OUTPUT -m owner --gid-owner "$SUITE_GID" \ + -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT || true + sudo "$tables" -D OUTPUT -m owner --gid-owner "$SUITE_GID" -j REJECT || true + done + fi + } + trap remove_host_egress_block EXIT + HOST_RULES_ADDED=1 + for tables in "${HOST_TABLES[@]}"; do + sudo "$tables" -I OUTPUT 1 -m owner --gid-owner "$SUITE_GID" -j REJECT + sudo "$tables" -I OUTPUT 1 -m owner --gid-owner "$SUITE_GID" \ + -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + sudo "$tables" -I OUTPUT 1 -m owner --gid-owner "$SUITE_GID" -o lo -j ACCEPT + done + run_offline() { + sudo --preserve-env=PATH,HOME,KUBECONFIG,CNTI_TESTSUITE_OFFLINE,CNTI_TESTSUITE_OFFLINE_DIR \ + setpriv --reuid "$SUITE_UID" --regid "$SUITE_GID" --clear-groups -- "$@" + } + run_offline true + if run_offline timeout 5 bash -c 'cat < /dev/null > /dev/tcp/1.1.1.1/443'; then + echo "::error::the suite process reached 1.1.1.1:443; host-side egress is not blocked" + exit 1 + fi + echo "No route to 1.1.1.1:443 from the suite process" + run_offline ./cnti-testsuite setup + run_offline ./cnti-testsuite cnf_install \ + --cnf-config "$RUNNER_TEMP/airgap-cnf/cnti-testsuite.yaml" + set +e + run_offline ./cnti-testsuite cert 2>&1 | tee airgap-cert.log + cert_status=${PIPESTATUS[0]} + run_offline ./cnti-testsuite workload 2>&1 | tee airgap-workload.log + workload_status=${PIPESTATUS[0]} + set -e + run_offline ./cnti-testsuite cnf_uninstall || true + run_offline ./cnti-testsuite uninstall_all || true + failed=0 + if [ "$cert_status" -ne 0 ]; then + echo "::error::cert exited $cert_status in offline mode" + failed=1 + fi + if ! grep -q '^Cert: PASSED ' airgap-cert.log; then + echo "::error::cert did not report PASSED in offline mode" + failed=1 + fi + # `workload` exits 0 when every test passed and 1 when some failed; + # both are acceptable, as in example_cnfs.yml. Only an errored test + # (exit 2) fails the job. The coredns sample CNF is expected to fail + # some compliance tests. + if [ "$workload_status" -gt 1 ]; then + echo "::error::workload errored (exit $workload_status) in offline mode" + failed=1 + fi + if ! grep -q '^Final workload score: ' airgap-workload.log; then + echo "::error::workload did not reach its verdict in offline mode" + failed=1 + fi + if [ "$failed" -ne 0 ]; then + exit 1 + fi + - name: Upload the suite log and results + if: always() + uses: actions/upload-artifact@v7.0.2 + with: + name: airgap + path: | + airgap-cert.log + airgap-workload.log + cnti/results/ + if-no-files-found: warn + - name: Remove the network block and the kind cluster + if: always() + continue-on-error: true + timeout-minutes: 10 + run: | + SUITE_GID=$(getent group airgap-suite | cut -d: -f3 || true) + if [ -n "$SUITE_GID" ]; then + for tables in iptables ip6tables; do + command -v "$tables" >/dev/null 2>&1 || continue + sudo "$tables" -D OUTPUT -m owner --gid-owner "$SUITE_GID" -o lo -j ACCEPT || true + sudo "$tables" -D OUTPUT -m owner --gid-owner "$SUITE_GID" \ + -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT || true + sudo "$tables" -D OUTPUT -m owner --gid-owner "$SUITE_GID" -j REJECT || true + done + fi + IPV4_KIND_SUBNETS=() + for subnet in ${KIND_SUBNETS:-}; do + case "$subnet" in + *:*) + tables=ip6tables + ;; + *) + tables=iptables + IPV4_KIND_SUBNETS+=("$subnet") + ;; + esac + sudo "$tables" -D DOCKER-USER -s "$subnet" \ + -m conntrack --ctstate NEW -j REJECT || true + done + if [ "${#IPV4_KIND_SUBNETS[@]}" -gt 0 ]; then + sudo iptables -D DOCKER-USER -s 10.244.0.0/16 \ + -m conntrack --ctstate NEW -j REJECT || true + fi + for destination in 10.96.0.0/16 10.244.0.0/16; do + sudo iptables -D DOCKER-USER -d "$destination" -j ACCEPT || true + done + for subnet in ${KIND_SUBNETS:-}; do + case "$subnet" in + *:*) sudo ip6tables -D DOCKER-USER -d "$subnet" -j ACCEPT || true ;; + *) sudo iptables -D DOCKER-USER -d "$subnet" -j ACCEPT || true ;; + esac + done + if command -v kind >/dev/null 2>&1 && \ + kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER_NAME"; then + until kind delete cluster --name "$KIND_CLUSTER_NAME" --verbosity 1; do sleep 1; done + else + echo "kind cluster $KIND_CLUSTER_NAME is not present; nothing to delete" + fi diff --git a/docs/USAGE.md b/docs/USAGE.md index c4efa607e..80311c836 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -854,10 +854,55 @@ CNTI_TESTSUITE_NETWORK_RETRY_ATTEMPTS=5 # attempts per fetch CNTI_TESTSUITE_NETWORK_RETRY_BACKOFF=10 # base seconds between attempts (attempt N waits N x backoff: 10s, 20s, 30s, 40s) ``` +#### Air-gapped runs: + +On a connected host, prepare a bundle with the files the suite would otherwise download (Helm, +Kubescape, the Kyverno CLI and policies, the LitmusChaos operator and the fault manifests): + +``` +./cnti-testsuite airgap_prepare --cnf-config ./cnti-testsuite.yaml +``` + +With the CNF installed on the connected host, `airgap_prepare` reads its manifests. Without an +installation, `--cnf-config` makes it scan that config's directory instead. It writes the bundle to +`~/.cnti-testsuite/airgap` by default and lists the images it +finds in `airgap/images.txt`: the suite's cluster-tools image, the LitmusChaos operator, runner and +fault images, and the images declared by the installed CNF (or by the directory of +`--cnf-config`). When the config sets `rolling_*_test_tag` or `rollback_from_tag` for a container, +it also adds those target images for that container's deployed repository, so a rollout or +rollback does not wait for an image the cluster cannot pull. A chart that templates its image +references may need entries added by hand. The bundle holds host binaries, so prepare it for the +OS and architecture of the disconnected host. +Copy the bundle to that host, load those images into the cluster, and run the suite with: + +``` +export CNTI_TESTSUITE_OFFLINE=true +export CNTI_TESTSUITE_OFFLINE_DIR=/path/to/airgap +./cnti-testsuite setup +./cnti-testsuite cnf_install --cnf-config ./cnti-testsuite.yaml +./cnti-testsuite workload +``` + +In offline mode every download is resolved from the bundle, and a missing file fails the task with +the URL and the bundle path instead of waiting on a network timeout. The CNF must install from +local sources (`helm_dirs` or `manifests` in `cnti-testsuite.yaml`), not from a chart repository. +Tests that read a registry cannot judge a disconnected cluster and report skipped with the reason: +`sbom_available`, `reasonable_image_size`, `helm_chart_published` (for charts served from a Helm +repository), and the registry lookup of a previous release in the rolling-update and rollback +tests when no `*_test_tag`/`rollback_from_tag` is configured. A CNF installed from a Helm +repository or OCI registry is also skipped by the rollout tests offline: they need to list or pull +another chart version. Install the CNF from local sources, or prepare both chart versions locally +and set `previous_chart_version`, to judge those tests offline. + +`./cnti-testsuite airgap_status` shows what the bundle has and what is missing without touching the +cluster. + #### Other environment variables: - **CNTI_TESTSUITE_DIR**: the suite home (default `~/.cnti-testsuite`): where `config.yml` is looked up and where the tools the suite installs for itself are cached under `tools/`. Each cached tool carries a `.version` marker with the version it was installed at; a suite that pins a different version reinstalls it on the next run, so the cache never serves a stale tool. - **CNTI_TESTSUITE_FORCE_INSTALL**: set (to any value) to reinstall the suite-managed local Helm even when an installation is already present. +- **CNTI_TESTSUITE_OFFLINE**: set to `true`, `1`, `yes` or `on` to resolve every download from the air-gap bundle instead of the network; see [Air-gapped runs](#air-gapped-runs). +- **CNTI_TESTSUITE_OFFLINE_DIR**: the air-gap bundle directory (default `~/.cnti-testsuite/airgap`). - **CNTI_TESTSUITE_ENV**: set to `TEST` for test-mode shortcuts (smaller samples, quicker checks). Used by the spec suite; not meant for normal runs. - **CNTI_TESTSUITE_RESULTS_DIR**: redirect the results directory; see [Results file](#results-file). - **CNTI_TESTSUITE_CHAOS_CHARTS_URL**: where the LitmusChaos fault templates are fetched from, for a mirror of the [chaos-charts](https://github.com/litmuschaos/chaos-charts) tree with the same layout (`/faults/kubernetes//fault.yaml`); by default the pinned release on raw.githubusercontent.com. A template that cannot be fetched after the network retries, or an experiment or its RBAC that cannot be applied, ends the test as an error with the reason: the CNF was not judged, so the run has no verdict (exit code 2) and a published badge keeps its last score. The templates are applied with the suite's cluster credentials, so only an `https://` mirror is accepted; any other scheme ends the Litmus tests as errors with that reason. Point it only at a mirror you trust. diff --git a/renovate.json b/renovate.json index ff97f2998..a5f13a36a 100644 --- a/renovate.json +++ b/renovate.json @@ -24,10 +24,10 @@ }, { "customType": "regex", - "description": "Container image with tag and digest in a Crystal string, or a YAML image: line, after a '# renovate:' annotation", + "description": "Container image with tag and digest in a Crystal constant or string, or a YAML image: line, after a '# renovate:' annotation", "managerFilePatterns": ["/\\.cr$/", "/\\.ecr$/", "/\\.ya?ml$/"], "matchStrings": [ - "# renovate: datasource=docker depName=(?[^\\s]+)\\n[^\\S\\n]*(?:image: |\")(?:[^\\s\"@:]+/)?[^\\s\"@:]+:(?[^\\s\"@]+)(?:@(?sha256:[a-f0-9]{64}))?\"?" + "# renovate: datasource=docker depName=(?[^\\s]+)\\n[^\\S\\n]*(?:[A-Za-z0-9_]+\\s*=\\s*)?(?:image: |\")(?:[^\\s\"@:]+/)?[^\\s\"@:]+:(?[^\\s\"@]+)(?:@(?sha256:[a-f0-9]{64}))?\"?" ], "datasourceTemplate": "docker" }, diff --git a/spec/tasks/airgap_spec.cr b/spec/tasks/airgap_spec.cr new file mode 100644 index 000000000..4dc6929aa --- /dev/null +++ b/spec/tasks/airgap_spec.cr @@ -0,0 +1,264 @@ +require "../spec_helper" +require "../../src/tasks/airgap.cr" +require "file_utils" + +describe "Airgap", tags: ["points"] do + it "reads image references from a YAML stream" do + yaml = <<-YAML + apiVersion: apps/v1 + kind: Deployment + spec: + template: + spec: + containers: + - name: app + image: "ghcr.io/acme/app:v1" + - name: sidecar + image: 'docker.io/library/nginx:1.27' + initContainers: + - image: busybox:1.36 + # A trailing YAML comment is not part of the image reference. + - image: ghcr.io/acme/comment:v1 # pinned release + YAML + + Airgap.images_from_yaml(yaml).should eq([ + "ghcr.io/acme/app:v1", + "docker.io/library/nginx:1.27", + "busybox:1.36", + "ghcr.io/acme/comment:v1", + ]) + end + + it "resolves YAML image anchors and aliases" do + yaml = <<-YAML + containers: + - name: app + image: &app ghcr.io/acme/app:v1 + - name: sidecar + image: *app + YAML + + Airgap.images_from_yaml(yaml).should eq(["ghcr.io/acme/app:v1"]) + end + + it "reads container names with their images" do + yaml = <<-YAML + containers: + - name: app + image: ghcr.io/acme/app:v1 + - image: docker.io/library/nginx:1.27 + name: sidecar + YAML + + Airgap.container_images_from_yaml(yaml).should eq([ + {name: "app", image: "ghcr.io/acme/app:v1"}, + {name: "sidecar", image: "docker.io/library/nginx:1.27"}, + ]) + end + + it "adds configured compatibility tags to the deployed repositories" do + config = File.tempname("airgap-config", ".yaml") + cnf_dir = File.tempname("airgap-cnf") + begin + File.write(config, <<-YAML) + --- + config_version: v2 + common: + container_names: + - name: app + rolling_update_test_tag: "2.0" + rolling_downgrade_test_tag: 1.0 + rolling_version_change_test_tag: v2 + rollback_from_tag: 1.0 + deployments: + manifests: + - name: app + manifest_directory: manifests + YAML + Dir.mkdir_p(File.join(cnf_dir, "manifests")) + File.write(File.join(cnf_dir, "manifests", "deployment.yaml"), <<-YAML) + containers: + - name: app + image: registry:5000/acme/app:1.5 + YAML + + Airgap.compatibility_images(config, [cnf_dir]).should eq([ + "registry:5000/acme/app:1.0", + "registry:5000/acme/app:2.0", + "registry:5000/acme/app:v2", + ]) + ensure + FileUtils.rm_rf(config) + FileUtils.rm_rf(cnf_dir) + end + end + + it "reads the Litmus runner image from the operator environment" do + yaml = <<-YAML + env: + - name: CHAOS_RUNNER_IMAGE + value: "litmuschaos.docker.scarf.sh/litmuschaos/chaos-runner:3.31.0" + YAML + + Airgap.images_from_yaml(yaml).should eq([ + "litmuschaos.docker.scarf.sh/litmuschaos/chaos-runner:3.31.0", + ]) + end + + it "uses local images in offline Litmus manifests" do + path = File.tempname("airgap-litmus", ".yaml") + previous = ENV["CNTI_TESTSUITE_OFFLINE"]? + begin + File.write(path, <<-YAML) + containers: + - image: litmuschaos/chaos-runner:v3 + imagePullPolicy: Always + - image: litmuschaos/chaos-operator:v3 + imagePullPolicy: "Always" + YAML + + ENV["CNTI_TESTSUITE_OFFLINE"] = "1" + LitmusManager.use_local_images(path) + + File.read(path).should contain("imagePullPolicy: IfNotPresent") + File.read(path).should_not contain("Always") + ensure + FileUtils.rm_rf(path) + if previous + ENV["CNTI_TESTSUITE_OFFLINE"] = previous + else + ENV.delete("CNTI_TESTSUITE_OFFLINE") + end + end + end + + it "adds a local pull policy when an offline Litmus fault omits it" do + path = File.tempname("airgap-litmus-missing-policy", ".yaml") + previous = ENV["CNTI_TESTSUITE_OFFLINE"]? + begin + File.write(path, <<-YAML) + kind: ChaosExperiment + spec: + definition: + image: "litmuschaos.docker.scarf.sh/litmuschaos/go-runner:3.31.0" + args: + - -c + YAML + + ENV["CNTI_TESTSUITE_OFFLINE"] = "1" + LitmusManager.use_local_images(path) + + content = File.read(path) + content.should contain("image: \"litmuschaos.docker.scarf.sh/litmuschaos/go-runner:3.31.0\"") + content.should contain("imagePullPolicy: IfNotPresent") + content.scan("imagePullPolicy:").size.should eq(1) + ensure + FileUtils.rm_rf(path) + if previous + ENV["CNTI_TESTSUITE_OFFLINE"] = previous + else + ENV.delete("CNTI_TESTSUITE_OFFLINE") + end + end + end + + it "collects suite, operator, fault and CNF images once each" do + operator = File.tempname("airgap-operator", ".yaml") + cnf_dir = File.tempname("airgap-cnf") + bundle = File.tempname("airgap-bundle") + previous_dir = ENV["CNTI_TESTSUITE_OFFLINE_DIR"]? + begin + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = bundle + + File.write(operator, <<-YAML) + containers: + - image: litmuschaos/chaos-operator:v3 + - image: #{ClusterTools::IMAGE} + YAML + + fault = Offline.cache_path(Airgap.fault_url("disk-fill")) + FileUtils.mkdir_p(File.dirname(fault)) + File.write(fault, <<-YAML) + containers: + - image: litmuschaos/chaos-runner:v3 + YAML + + Dir.mkdir_p(File.join(cnf_dir, "manifests")) + File.write(File.join(cnf_dir, "manifests", "deployment.yaml"), <<-YAML) + containers: + - image: ghcr.io/acme/app:v1 + YAML + File.write(File.join(cnf_dir, "manifests", "service.yaml"), <<-YAML) + spec: + ports: + - port: 80 + YAML + + images = Airgap.collect_images(operator, [cnf_dir]) + images.should eq([ + ClusterTools::IMAGE, + "ghcr.io/acme/app:v1", + "litmuschaos/chaos-operator:v3", + "litmuschaos/chaos-runner:v3", + ].sort) + ensure + FileUtils.rm_rf(operator) + FileUtils.rm_rf(cnf_dir) + FileUtils.rm_rf(bundle) + if previous_dir + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = previous_dir + else + ENV.delete("CNTI_TESTSUITE_OFFLINE_DIR") + end + end + end + + it "lists the artifacts a bundle needs, including every Litmus fault" do + artifacts = Airgap.host_artifacts + artifacts.should contain(Setup::HELM_URL) + artifacts.should contain(LitmusManager::LITMUS_OPERATOR) + LitmusManager::FAULTS.each do |fault| + artifacts.should contain(Airgap.fault_url(fault)) + end + end + + it "sends the GitHub token only to GitHub hosts" do + previous = ENV["GITHUB_TOKEN"]? + begin + ENV["GITHUB_TOKEN"] = "test-token" + + Airgap.headers_for("https://github.com/lfn-cnti/testsuite").not_nil!["Authorization"] + .should eq("Bearer test-token") + Airgap.headers_for("https://raw.githubusercontent.com/lfn-cnti/testsuite/main/file") + .should_not be_nil + Airgap.headers_for("http://github.com/lfn-cnti/testsuite").should be_nil + Airgap.headers_for("http://raw.githubusercontent.com/lfn-cnti/testsuite/main/file").should be_nil + Airgap.headers_for("https://artifactory.corp/github.com-remote/chaos-charts") + .should be_nil + Airgap.headers_for("https://github.com.evil.example/file").should be_nil + Airgap.headers_for("https://evil.example/githubusercontent.com/file").should be_nil + ensure + if previous + ENV["GITHUB_TOKEN"] = previous + else + ENV.delete("GITHUB_TOKEN") + end + end + end + + it "uses the configured chaos-charts mirror for fault artifacts" do + previous = ENV[LitmusManager::CHAOS_CHARTS_ENV]? + begin + ENV[LitmusManager::CHAOS_CHARTS_ENV] = "https://mirror.example/chaos-charts/" + + Airgap.fault_url("disk-fill").should eq( + "https://mirror.example/chaos-charts/faults/kubernetes/disk-fill/fault.yaml") + ensure + if previous + ENV[LitmusManager::CHAOS_CHARTS_ENV] = previous + else + ENV.delete(LitmusManager::CHAOS_CHARTS_ENV) + end + end + end +end diff --git a/spec/utils/offline_spec.cr b/spec/utils/offline_spec.cr new file mode 100644 index 000000000..1a677963c --- /dev/null +++ b/spec/utils/offline_spec.cr @@ -0,0 +1,142 @@ +require "../spec_helper" +require "file_utils" + +describe "Offline", tags: ["points"] do + it "recognises the offline switches" do + previous = ENV["CNTI_TESTSUITE_OFFLINE"]? + begin + {"1", "true", "yes", "on", "ON"}.each do |value| + ENV["CNTI_TESTSUITE_OFFLINE"] = value + Offline.enabled?.should be_true + end + {"0", "false", "no", "off", ""}.each do |value| + ENV["CNTI_TESTSUITE_OFFLINE"] = value + Offline.enabled?.should be_false + end + ENV.delete("CNTI_TESTSUITE_OFFLINE") + Offline.enabled?.should be_false + ensure + if previous + ENV["CNTI_TESTSUITE_OFFLINE"] = previous + else + ENV.delete("CNTI_TESTSUITE_OFFLINE") + end + end + end + + it "names a bundle file after the URL so two URLs cannot collide" do + first = Offline.cache_path("https://example.test/releases/app.tar.gz") + second = Offline.cache_path("https://example.test/other/app.tar.gz") + + File.basename(first).should end_with("-app.tar.gz") + File.basename(second).should end_with("-app.tar.gz") + first.should_not eq(second) + Offline.cache_path("https://example.test/releases/app.tar.gz").should eq(first) + end + + it "stores and resolves an artifact from the bundle" do + bundle = File.tempname("offline-bundle") + previous_dir = ENV["CNTI_TESTSUITE_OFFLINE_DIR"]? + previous_mode = ENV["CNTI_TESTSUITE_OFFLINE"]? + begin + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = bundle + ENV["CNTI_TESTSUITE_OFFLINE"] = "true" + + source = File.tempname("offline-source") + File.write(source, "artifact body") + url = "https://example.test/artifact.tar.gz" + + FileUtils.mkdir_p(File.dirname(Offline.cache_path(url))) + File.write(Offline.cache_path(url), "") + Offline.cached?(url).should be_false + + Offline.store(url, source) + Offline.cached?(url).should be_true + File.read(Offline.cache_path(url)).should eq("artifact body") + + output = File.tempname("offline-output") + Offline.fetch_cached(url, output).should be_true + File.read(output).should eq("artifact body") + ensure + if previous_dir + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = previous_dir + else + ENV.delete("CNTI_TESTSUITE_OFFLINE_DIR") + end + if previous_mode + ENV["CNTI_TESTSUITE_OFFLINE"] = previous_mode + else + ENV.delete("CNTI_TESTSUITE_OFFLINE") + end + FileUtils.rm_rf(bundle) + end + end + + it "keeps credentials out of missing-artifact diagnostics" do + bundle = File.tempname("offline-bundle") + previous_dir = ENV["CNTI_TESTSUITE_OFFLINE_DIR"]? + previous_mode = ENV["CNTI_TESTSUITE_OFFLINE"]? + begin + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = bundle + ENV["CNTI_TESTSUITE_OFFLINE"] = "true" + + url = "https://user:secret@mirror.example/charts/fault.yaml" + error = expect_raises(Offline::MissingArtifactError) do + Offline.fetch_cached(url, File.tempname("offline-output")) + end + + message = error.message.to_s + message.should contain("https://mirror.example/charts/fault.yaml") + message.should_not contain("secret") + message.should_not contain("user") + ensure + if previous_dir + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = previous_dir + else + ENV.delete("CNTI_TESTSUITE_OFFLINE_DIR") + end + if previous_mode + ENV["CNTI_TESTSUITE_OFFLINE"] = previous_mode + else + ENV.delete("CNTI_TESTSUITE_OFFLINE") + end + FileUtils.rm_rf(bundle) + end + end + + it "download_file reads the bundle instead of the network" do + bundle = File.tempname("offline-bundle") + previous_dir = ENV["CNTI_TESTSUITE_OFFLINE_DIR"]? + previous_mode = ENV["CNTI_TESTSUITE_OFFLINE"]? + begin + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = bundle + ENV["CNTI_TESTSUITE_OFFLINE"] = "true" + + url = "http://offline.invalid/never-reached.tar.gz" + output = File.tempname("offline-download") + + expect_raises(Offline::MissingArtifactError) do + download_file(url, output) + end + + source = File.tempname("offline-source") + File.write(source, "cached body") + Offline.store(url, source) + + download_file(url, output).should be_true + File.read(output).should eq("cached body") + ensure + if previous_dir + ENV["CNTI_TESTSUITE_OFFLINE_DIR"] = previous_dir + else + ENV.delete("CNTI_TESTSUITE_OFFLINE_DIR") + end + if previous_mode + ENV["CNTI_TESTSUITE_OFFLINE"] = previous_mode + else + ENV.delete("CNTI_TESTSUITE_OFFLINE") + end + FileUtils.rm_rf(bundle) + end + end +end diff --git a/src/modules/cluster_tools/cluster_tools.cr b/src/modules/cluster_tools/cluster_tools.cr index efe343b0a..54f5906b2 100644 --- a/src/modules/cluster_tools/cluster_tools.cr +++ b/src/modules/cluster_tools/cluster_tools.cr @@ -22,6 +22,11 @@ module ClusterTools %(awk -v s="/$name.sock" 'NR > 1 && substr($NF, length($NF) - length(s) + 1) == s { print $NF; exit }' #{proc_net_unix}) end + # Fully qualified: CRI-O resolves no short names without a configured + # search registry, and would never pull the image. + # renovate: datasource=docker depName=lfncnti/cluster-tools + IMAGE = "docker.io/lfncnti/cluster-tools:v1.0.8" + class ManifestTemplate def initialize end diff --git a/src/modules/cluster_tools/templates/manifest.yml.ecr b/src/modules/cluster_tools/templates/manifest.yml.ecr index 369c09d24..22d5c186f 100644 --- a/src/modules/cluster_tools/templates/manifest.yml.ecr +++ b/src/modules/cluster_tools/templates/manifest.yml.ecr @@ -25,8 +25,7 @@ spec: - name: cluster-tools # Fully qualified: CRI-O resolves no short names without a # configured search registry, and would never pull the image. - # renovate: datasource=docker depName=lfncnti/cluster-tools - image: docker.io/lfncnti/cluster-tools:v1.0.8 + image: <%= ClusterTools::IMAGE %> # The tag is pinned, so there is nothing to refresh: pulling it on # every node only adds a wait, and a registry that is slow or # rate-limits leaves a node without the pod the tests read the diff --git a/src/tasks/airgap.cr b/src/tasks/airgap.cr new file mode 100644 index 000000000..79a104396 --- /dev/null +++ b/src/tasks/airgap.cr @@ -0,0 +1,288 @@ +require "sam" +require "file_utils" +require "../modules/cluster_tools" +require "./utils/utils.cr" +require "./utils/image_tag.cr" +require "./utils/litmus_manager.cr" +require "./setup/constants.cr" + +# Helpers for the air-gap bundle: the files airgap_prepare downloads on a +# connected host and the image references a disconnected cluster needs. +module Airgap + IMAGE_LINE = /^\s*(?:-\s*)?image:\s*["']?([^"'\s]+)["']?(?:\s+#.*)?\s*$/ + RUNNER_IMAGE_VALUE = /name:\s*CHAOS_RUNNER_IMAGE\s*\n\s*value:\s*["']?([^"'\s]+)["']?/ + # A --cnf-config at the root of a checkout must not send the scan through the + # whole repository (or the run's own cnti/ workspace, which is scanned as + # CNF_DIR when the CNF is installed). + SKIP_DIRS = [".git", "node_modules", "vendor", "target", "cnti"] + + def self.fault_url(fault : String) : String + "#{LitmusManager.chaos_charts}/faults/kubernetes/#{fault}/fault.yaml" + end + + def self.host_artifacts : Array(String) + [Setup::HELM_URL, + Setup::KUBESCAPE_URL, + Setup::KUBESCAPE_FRAMEWORK_URL, + Setup::KUBESCAPE_CONTROLS_URL, + Kyverno.download_url, + LitmusManager::LITMUS_OPERATOR] + + LitmusManager::FAULTS.map { |fault| fault_url(fault) } + end + + def self.fault_manifests : Array(String) + LitmusManager::FAULTS.map { |fault| Offline.cache_path(fault_url(fault)) } + end + + def self.headers_for(url : String) : HTTP::Headers? + uri = URI.parse(url) + return nil unless uri.scheme.try(&.downcase) == "https" + host = uri.host.to_s.downcase + github_host = {"github.com", "githubusercontent.com"}.any? do |domain| + host == domain || host.ends_with?(".#{domain}") + end + return nil unless github_host + token = ENV["GITHUB_TOKEN"]? + return nil if token.nil? || token.empty? + HTTP::Headers{"Authorization" => "Bearer #{token}"} + end + + # The image references in a YAML stream, from both "image: x" and "- image: x". + # The Litmus operator also names its runner in an environment variable. The + # lines are read first so a manifest with Helm templating still yields its + # static images; a parsed document then resolves anchors and aliases. + def self.images_from_yaml(content : String) : Array(String) + images = [] of String + seen = Set(String).new + content.lines.each do |line| + match = line.match(IMAGE_LINE) + add_image(match[1], images, seen) if match + end + if match = content.match(RUNNER_IMAGE_VALUE) + add_image(match[1], images, seen) + end + begin + YAML.parse_all(content).each { |document| images_in(document, images, seen) } + rescue YAML::ParseException + end + images + end + + private def self.add_image(image : String, images : Array(String), seen : Set(String)) : Nil + image = image.strip + return if image.empty? || image.includes?("{{") + return if image.starts_with?('&') || image.starts_with?('*') + images << image if seen.add?(image) + end + + private def self.images_in(node : YAML::Any, images : Array(String), seen : Set(String)) : Nil + if hash = node.as_h? + add_image(hash["image"]?.try(&.as_s?).to_s, images, seen) + if hash["name"]?.try(&.as_s?) == "CHAOS_RUNNER_IMAGE" + add_image(hash["value"]?.try(&.as_s?).to_s, images, seen) + end + hash.each_value { |value| images_in(value, images, seen) } + elsif array = node.as_a? + array.each { |value| images_in(value, images, seen) } + end + end + + # The container name and image of each container declared in a YAML stream. + # Compatibility tests take a configured tag and apply it to that deployed + # image's repository, so the bundle needs the resulting image reference too. + def self.container_images_from_yaml(content : String) : Array(NamedTuple(name: String, image: String)) + found = [] of NamedTuple(name: String, image: String) + YAML.parse_all(content).each { |document| container_images_in(document, found) } + found + rescue YAML::ParseException + [] of NamedTuple(name: String, image: String) + end + + private def self.container_images_in(node : YAML::Any, found : Array(NamedTuple(name: String, image: String))) : Nil + if hash = node.as_h? + name = hash["name"]?.try(&.as_s?) + image = hash["image"]?.try(&.as_s?) + if name && image && !image.includes?("{{") + found << {name: name, image: image} + end + hash.each_value { |value| container_images_in(value, found) } + elsif array = node.as_a? + array.each { |value| container_images_in(value, found) } + end + end + + # Every rolling/rollback target the compatibility tests can request, paired + # with the repository of the container's deployed image. A templated image + # cannot be resolved here; the caller still needs to add it after rendering. + def self.compatibility_images(config_path : String, deployment_dirs : Array(String)) : Array(String) + return [] of String unless File.exists?(config_path) + configured = CNFInstall::Config.parse_cnf_config_from_file(config_path).common.container_names + return [] of String if configured.empty? + + images_by_container = {} of String => Set(String) + deployment_dirs.uniq.each do |dir| + next unless Dir.exists?(dir) + yaml_files(dir) do |file| + container_images_from_yaml(File.read(file)).each do |pair| + images_by_container[pair[:name]] ||= Set(String).new + images_by_container[pair[:name]] << pair[:image] + end + rescue ex + Log.warn { "Could not read #{file} while collecting compatibility images: #{ex.message}" } + end + end + + targets = Set(String).new + configured.each do |container| + images_by_container[container.name]?.try &.each do |image| + repository = ImageTag.repository_of(image) + next if repository.empty? + [container.rolling_update_test_tag, + container.rolling_downgrade_test_tag, + container.rolling_version_change_test_tag, + container.rollback_from_tag].each do |tag| + targets << "#{repository}:#{tag}" unless tag.empty? + end + end + end + targets.to_a.sort + end + + # Suite-owned images plus every image declared in the Litmus operator and + # fault manifests and in the given deployment directories. The caller loads + # them into the cluster; the suite never pulls them from the internet in + # offline mode. + def self.collect_images(operator_path : String, deployment_dirs : Array(String) = [] of String, config_path : String? = nil) : Array(String) + images = Set(String).new + images << ClusterTools::IMAGE + + ([operator_path] + fault_manifests).each do |path| + next unless File.exists?(path) + images.concat(images_from_yaml(File.read(path))) + end + + deployment_dirs.uniq.each do |dir| + next unless Dir.exists?(dir) + yaml_files(dir) do |file| + images.concat(images_from_yaml(File.read(file))) + rescue ex + Log.warn { "Could not read #{file} while collecting images: #{ex.message}" } + end + end + + images.concat(compatibility_images(config_path, deployment_dirs)) if config_path + images.to_a.sort + end + + private def self.yaml_files(dir : String, &block : String ->) : Nil + Dir.each_child(dir) do |entry| + path = File.join(dir, entry) + if File.directory?(path) + next if entry.starts_with?(".") || SKIP_DIRS.includes?(entry) + yaml_files(path, &block) + elsif entry.ends_with?(".yml") || entry.ends_with?(".yaml") + yield path + end + rescue ex + Log.warn { "Could not scan #{path} for images: #{ex.message}" } + end + end + + # Copies the pinned Kyverno policies into the bundle. The CLI itself is a + # normal download and is fetched by host_artifacts. + def self.prepare_policies : String + source = Kyverno.policies_repo_path + unless ToolInstall.current?(Kyverno::POLICIES_BRANCH, source) + unless Kyverno.download_policies_repo + raise "could not download the Kyverno policies from github.com/kyverno/policies" + end + end + + destination = Offline.bundle_path("kyverno-policies") + FileUtils.rm_rf(destination) + FileUtils.cp_r(source, destination) + destination + end +end + +desc "Download the host-side files an air-gapped run needs into the offline bundle" +task "airgap_prepare" do |_, args| + if Offline.enabled? + stdout_failure "airgap_prepare needs network access: unset #{Offline::MODE_ENV} and run it on a connected host." + exit 1 + end + + StatusLine.push "Preparing the air-gap bundle in #{Offline.bundle_dir}..." + failures = [] of String + FileUtils.mkdir_p(Offline.downloads_dir) + + Airgap.host_artifacts.each do |url| + destination = Offline.cache_path(url) + if Offline.cached?(url) + Log.info { "Already in the air-gap bundle: #{url_without_credentials(url)}" } + next + end + + begin + download_file(url, destination, headers: Airgap.headers_for(url)) + stdout_info "Fetched #{File.basename(destination)}" + rescue ex + failures << "#{url_without_credentials(url)}: #{ex.message}" + end + end + + begin + Airgap.prepare_policies + stdout_info "Fetched the Kyverno policies" + rescue ex + failures << "Kyverno policies: #{ex.message}" + end + + deployment_dirs = [] of String + config_path = nil + if configured = args.named["cnf-config"]? + path = configured.to_s + config_path = Dir.exists?(path) ? File.join(path, CONFIG_FILE) : path + deployment_dirs << (Dir.exists?(path) ? path : File.dirname(path)) + end + deployment_dirs << CNF_DIR if Dir.exists?(CNF_DIR) + + images = Airgap.collect_images(Offline.cache_path(LitmusManager::LITMUS_OPERATOR), deployment_dirs, config_path) + File.write(Offline.images_file, images.join("\n") + "\n") + stdout_info "Wrote #{images.size} image reference(s) to #{Offline.images_file}" + + StatusLine.pop + if failures.empty? + stdout_success "Air-gap bundle ready: #{Offline.bundle_dir}" + else + failures.each { |failure| stdout_failure "Could not prepare #{failure}" } + stdout_failure "Air-gap bundle is incomplete: #{Offline.bundle_dir}" + exit 1 + end +end + +desc "Show what the offline bundle contains and what is missing" +task "airgap_status" do |_, args| + stdout_info "Offline mode: #{Offline.enabled? ? "enabled" : "disabled"}" + stdout_info "Bundle: #{Offline.bundle_dir}" + + missing = Airgap.host_artifacts.reject { |url| Offline.cached?(url) } + stdout_info "Host artifacts: #{Airgap.host_artifacts.size - missing.size}/#{Airgap.host_artifacts.size} present" + missing.each do |url| + stdout_warning "Missing: #{url_without_credentials(url)} -> #{Offline.cache_path(url)}" + end + + policies = Offline.bundle_path("kyverno-policies") + if Dir.exists?(policies) + stdout_info "Kyverno policies: present" + else + stdout_warning "Missing: #{policies}" + end + + if File.exists?(Offline.images_file) + images = File.read_lines(Offline.images_file).reject(&.empty?) + stdout_info "Images: #{images.size} listed in #{Offline.images_file}" + else + stdout_warning "Missing: #{Offline.images_file}" + end +end diff --git a/src/tasks/setup/litmus_setup.cr b/src/tasks/setup/litmus_setup.cr index 02ddf060d..a5c4c92a7 100644 --- a/src/tasks/setup/litmus_setup.cr +++ b/src/tasks/setup/litmus_setup.cr @@ -28,7 +28,11 @@ namespace "setup" do begin KubectlClient::Utils.label("namespace", LitmusManager::LITMUS_NAMESPACE, ["pod-security.kubernetes.io/enforce=privileged"]) - KubectlClient::Apply.file(LitmusManager::LITMUS_OPERATOR, namespace: LitmusManager::LITMUS_NAMESPACE) + KubectlClient::Apply.file(LitmusManager.operator_manifest, namespace: LitmusManager::LITMUS_NAMESPACE) + rescue ex : Offline::MissingArtifactError + logger.error { "Error while installing Litmus tool: #{ex.message}" } + stdout_failure(ex.message) + exit(1) rescue ex : KubectlClient::ShellCMD::K8sClientCMDException logger.error { "Error while installing Litmus tool: #{ex.message}" } stdout_failure(failed_msg) @@ -52,7 +56,11 @@ namespace "setup" do end begin - KubectlClient::Delete.file(LitmusManager::LITMUS_OPERATOR, namespace: LitmusManager::LITMUS_NAMESPACE) + KubectlClient::Delete.file(LitmusManager.operator_manifest, namespace: LitmusManager::LITMUS_NAMESPACE) + rescue ex : Offline::MissingArtifactError + logger.error { "Error while deleting Litmus operator: #{ex.message}" } + stdout_failure("Task 'uninstall_litmus' failed") + exit(1) rescue ex : KubectlClient::ShellCMD::K8sClientCMDException logger.error { "Error while deleting Litmus operator: #{ex.message}" } stdout_failure("Task 'uninstall_litmus' failed") diff --git a/src/tasks/utils/cli_help.cr b/src/tasks/utils/cli_help.cr index d060058e2..8b8e85aff 100644 --- a/src/tasks/utils/cli_help.cr +++ b/src/tasks/utils/cli_help.cr @@ -97,6 +97,7 @@ module CLIHelp add.call("Getting started", [ "setup", "cnf_install", "cnf_uninstall", "validate_config", + "airgap_prepare", "airgap_status", "cnf_setup", "cnf_cleanup", "uninstall_all", "tools_uninstall", ]) add.call("Test suites", ["all", "workload", "essential", "cert", "static"]) diff --git a/src/tasks/utils/kyverno.cr b/src/tasks/utils/kyverno.cr index fbd3a8284..ebece2bf3 100644 --- a/src/tasks/utils/kyverno.cr +++ b/src/tasks/utils/kyverno.cr @@ -71,6 +71,20 @@ module Kyverno end def self.download_policies_repo + FileUtils.mkdir_p(File.dirname(policies_repo_path)) + if Offline.enabled? + source = Offline.bundle_path("kyverno-policies") + unless Dir.exists?(source) + raise Offline::MissingArtifactError.new( + "Offline mode: the Kyverno policies are not in the air-gap bundle at #{source}. " \ + "Run `cnti-testsuite airgap_prepare` on a connected host." + ) + end + delete_policies_repo + FileUtils.cp_r(source, policies_repo_path) + return true + end + delete_policies_repo url = "https://github.com/kyverno/policies.git" result = GitClient.clone("--depth 1 --branch #{POLICIES_BRANCH} #{url} #{policies_repo_path}") diff --git a/src/tasks/utils/litmus_manager.cr b/src/tasks/utils/litmus_manager.cr index 2e0010cab..71febc241 100644 --- a/src/tasks/utils/litmus_manager.cr +++ b/src/tasks/utils/litmus_manager.cr @@ -24,18 +24,59 @@ module LitmusManager NODE_LABEL = "kubernetes.io/hostname" #https://raw.githubusercontent.com/litmuschaos/chaos-operator/v2.14.x/deploy/operator.yaml LITMUS_OPERATOR = "https://litmuschaos.github.io/litmus/litmus-operator-v#{LitmusManager::Version}.yaml" + # The faults the workload tests install. airgap_prepare fetches their + # manifests so a disconnected run can install them from the bundle. + FAULTS = [ + "disk-fill", + "pod-delete", + "pod-dns-error", + "pod-io-stress", + "pod-memory-hog", + "pod-network-corruption", + "pod-network-duplication", + "pod-network-latency", + ] + # for node drain; live with the chaos templates, not in the CWD def self.downloaded_operator_file : String File.join(chaos_manifests_path, "litmus-operator-downloaded.yaml") end + # The operator manifest to apply. Online it is the released URL, as before; + # offline it is the bundle's cached copy written next to the experiments. + def self.operator_manifest : String + return LITMUS_OPERATOR unless Offline.enabled? + path = downloaded_operator_file + download_file(LITMUS_OPERATOR, path) + use_local_images(path) + end + + # The released manifests pull every pod image, which fails on a cluster that + # cannot reach a registry even when the images were loaded into its nodes. + # Rewrite that policy in the copy the offline run applies, keeping the + # bundle's original manifest untouched. + def self.use_local_images(path : String) : String + return path unless Offline.enabled? + content = File.read(path) + patched = content.gsub(OFFLINE_PULL_POLICY, "\\1imagePullPolicy: IfNotPresent") + # The pod-dns-error fault omits imagePullPolicy, and the chaos runner then + # defaults it to Always, so its experiment job waits for a pull that cannot + # happen offline. Add the policy the fault should have had. + unless patched.matches?(/^[ \t]*imagePullPolicy:/m) + patched = patched.gsub(OFFLINE_MISSING_PULL_POLICY, "\\0\\1imagePullPolicy: IfNotPresent\n") + end + File.write(path, patched) if patched != content + path + end + def self.modified_operator_file : String File.join(chaos_manifests_path, "litmus-operator-modified.yaml") end + LITMUS_NAMESPACE = "litmus" LITMUS_K8S_DOMAIN = "litmuschaos.io" - - + OFFLINE_PULL_POLICY = /^(\s*)imagePullPolicy:\s*["']?Always["']?\s*$/m + OFFLINE_MISSING_PULL_POLICY = /^([ \t]*)image:[^\n]*\n(?![ \t]*imagePullPolicy:)/m # Label pair that selects exactly the pods of `resource`, so a chaos engine # appinfo targets this workload and not every pod sharing a broad selector. @@ -584,6 +625,7 @@ module LitmusManager Log.info {"filepath: #{filepath}"} download_file(url, filepath) + use_local_images(filepath) filepath end diff --git a/src/tasks/utils/offline.cr b/src/tasks/utils/offline.cr new file mode 100644 index 000000000..6057f6110 --- /dev/null +++ b/src/tasks/utils/offline.cr @@ -0,0 +1,92 @@ +require "digest/sha256" +require "file_utils" +require "log" +require "uri" + +# Air-gap support. Every external host-side fetch goes through download_file, +# so an offline run resolves those fetches from a bundle prepared on a +# connected host instead of reaching the internet. +module Offline + MODE_ENV = "CNTI_TESTSUITE_OFFLINE" + DIR_ENV = "CNTI_TESTSUITE_OFFLINE_DIR" + + # The download in the bundle is named after its URL, not just its basename: + # one source URL is fetched to different output paths (a Litmus fault is + # downloaded once per target workload), and two URLs can share a basename. + class MissingArtifactError < Exception + end + + def self.enabled? : Bool + case ENV[MODE_ENV]?.try(&.strip.downcase) + when "1", "true", "yes", "on" + true + else + false + end + end + + def self.bundle_dir : String + configured = ENV[DIR_ENV]?.try(&.strip) + return configured if configured && !configured.empty? + File.join(testsuite_resources_dir, "airgap") + end + + def self.bundle_path(*parts : String) : String + File.join([bundle_dir] + parts.to_a) + end + + def self.downloads_dir : String + bundle_path("downloads") + end + + def self.images_file : String + bundle_path("images.txt") + end + + def self.cache_path(url : String) : String + name = File.basename(URI.parse(url).path) + name = "download" if name.empty? + digest = Digest::SHA256.hexdigest(url)[0, 16] + File.join(downloads_dir, "#{digest}-#{name}") + end + + def self.cached?(url : String) : Bool + path = cache_path(url) + File.exists?(path) && File.size(path) > 0 + end + + # Copies the bundle's copy of `url` to output_path. Raises when the bundle + # does not have it, naming both the URL and the directory to add it to. + def self.fetch_cached(url : String, output_path : String) : Bool + source = cache_path(url) + diagnostic_url = url_without_credentials(url) + unless cached?(url) + raise MissingArtifactError.new( + "Offline mode: #{diagnostic_url} is not in the air-gap bundle at #{bundle_dir}. " \ + "Run `cnti-testsuite airgap_prepare` on a connected host, or copy the file to #{source}." + ) + end + + FileUtils.mkdir_p(File.dirname(output_path)) + unless same_path?(source, output_path) + FileUtils.cp(source, output_path) + end + Log.info { "Offline mode: resolved #{diagnostic_url} from #{source}" } + true + end + + # Copies an existing file into the bundle's cache. airgap_prepare downloads + # straight into the cache so an interrupted preparation keeps every artifact + # it already fetched; this is the way to seed a bundle by hand (and in specs). + def self.store(url : String, source : String) : String + destination = cache_path(url) + return destination if same_path?(destination, source) + FileUtils.mkdir_p(File.dirname(destination)) + FileUtils.cp(source, destination) + destination + end + + private def self.same_path?(first : String, second : String) : Bool + File.expand_path(first) == File.expand_path(second) + end +end diff --git a/src/tasks/utils/release_upgrade.cr b/src/tasks/utils/release_upgrade.cr index 631bbe6c8..e52eb2454 100644 --- a/src/tasks/utils/release_upgrade.cr +++ b/src/tasks/utils/release_upgrade.cr @@ -133,6 +133,28 @@ module ReleaseUpgrade chosen end + # Whether the chart of `version` is on disk, so moving the release to it + # does not need a pull. + def self.chart_cached?(release : Release, version : String) : Bool + installed_path = File.join(DEPLOYMENTS_DIR, release.name, release.chart_name) + return true if version == release.installed && Dir.exists?(installed_path) + Dir.exists?(File.join(DEPLOYMENTS_DIR, "#{release.name}-#{version}", release.chart_name)) + end + + # In offline mode the repository cannot be listed and a chart that is not + # on disk cannot be pulled. Nil when the move can be judged from local files. + def self.offline_reason(release : Release) : String? + return nil unless Offline.enabled? + configured = release.config.previous_chart_version + if configured.empty? + return "offline mode cannot list the chart versions of #{release.chart_name}; set previous_chart_version and prepare that chart locally" + end + unless chart_cached?(release, configured) && chart_cached?(release, release.installed) + return "offline mode cannot pull the chart #{release.chart_name} #{configured} or #{release.installed}; prepare both locally" + end + nil + end + def self.repository_versions(release : Release) : Array(Helm::ChartVersion)? cfg = release.config if release.oci? diff --git a/src/tasks/utils/utils.cr b/src/tasks/utils/utils.cr index c3cb0114e..43e88eaa5 100644 --- a/src/tasks/utils/utils.cr +++ b/src/tasks/utils/utils.cr @@ -8,6 +8,7 @@ require "option_parser" require "../constants.cr" require "semantic_version" require "./dockerd.cr" +require "./offline.cr" require "./kyverno.cr" require "./tool_install.cr" require "./workload_diagnostics.cr" @@ -17,6 +18,7 @@ require "./suite_namespaces.cr" require "./timeouts.cr" require "./cnf_installation/config.cr" require "ecr" +require "base64" require "http/client" require "../../modules/net_retry" @@ -553,16 +555,36 @@ end def download_file(url : String, output_path : String, redirect_limit : Int = 5, headers : HTTP::Headers? = nil) + # An air-gapped run never reaches the internet: the bundle prepared on a + # connected host is the only source. + if Offline.enabled? + return Offline.fetch_cached(url, output_path) + end + # The display label comes from the caller's URL: redirect targets (GitHub # serves release assets under opaque UUID names) and temp output files both # make meaningless labels. - label = File.basename(URI.parse(url).path) + uri = URI.parse(url) + label = File.basename(uri.path) label = File.basename(output_path) if label.empty? + headers = add_basic_auth(uri, headers) NetRetry.with_retries("download #{url_without_credentials(url)}") do follow_and_download(url, output_path, redirect_limit, headers, label) end end +# An authenticated mirror puts its credentials in the URL. Crystal's +# HTTP::Client.new(uri) does not apply them, so turn them into Basic auth. A +# caller-supplied Authorization header wins. +private def add_basic_auth(uri : URI, headers : HTTP::Headers?) : HTTP::Headers? + return headers unless user = uri.user + result = headers.try(&.dup) || HTTP::Headers.new + unless result.has_key?("Authorization") + result["Authorization"] = "Basic #{Base64.strict_encode("#{user}:#{uri.password}")}" + end + result +end + # Streams the response straight to `.part` (so a long download # leaves visible, growing evidence on disk) and renames on completion; a # size-announcing line on stdout tells the user what a first run is fetching. @@ -639,6 +661,23 @@ private def follow_and_download(url : String, output_path : String, if location = redirect_location # Location may be relative; resolve it against the URI that redirected. - return follow_and_download(uri.resolve(location).to_s, output_path, redirect_limit - 1, headers, label) + target = uri.resolve(location) + uri_scheme = uri.scheme.try(&.downcase) + target_scheme = target.scheme.try(&.downcase) + if headers && uri_scheme == "https" && target_scheme != "https" + raise "Refusing to send credentials to #{target.scheme}://#{target.host} after an HTTPS redirect" + end + redirect_headers = headers + if redirect_headers + target_host = target.host.to_s.downcase + uri_host = uri.host.to_s.downcase + target_port = target.port || (target_scheme == "https" ? 443 : 80) + uri_port = uri.port || (uri_scheme == "https" ? 443 : 80) + if target_host != uri_host || target_port != uri_port + redirect_headers = redirect_headers.dup + redirect_headers.delete("Authorization") + end + end + return follow_and_download(target.to_s, output_path, redirect_limit - 1, redirect_headers, label) end end diff --git a/src/tasks/workload/compatibility.cr b/src/tasks/workload/compatibility.cr index e1981e5d2..e77cc7a31 100644 --- a/src/tasks/workload/compatibility.cr +++ b/src/tasks/workload/compatibility.cr @@ -22,6 +22,7 @@ ROLLING_VERSION_CHANGE_TEST_NAMES.each do |tn| Log.for(t.name).debug { "container_names: #{container_names}" } rolled = 0 unconfigured = 0 + unconfigured_reasons = Set(String).new batch = Set(String).new # Workloads an operator owns take their image from its custom resource # (#2810): listed, not rolled, not counted. @@ -70,7 +71,8 @@ ROLLING_VERSION_CHANGE_TEST_NAMES.each do |tn| # (#2577). configured_tag = config_container.try(&.get_container_tag(tn)) || "" tag = rollout_target_tag(result, configured_tag, "#{tn}_test_tag", - "#{resource["kind"]}/#{resource["name"]} in #{namespace}", container_name, container.as_h["image"].as_s) + "#{resource["kind"]}/#{resource["name"]} in #{namespace}", container_name, + container.as_h["image"].as_s, unconfigured_reasons) unless tag unconfigured += 1 next true @@ -129,9 +131,16 @@ ROLLING_VERSION_CHANGE_TEST_NAMES.each do |tn| result.append_description("#{release.label}: no Deployment, StatefulSet or DaemonSet of the release to roll out; not judged") next end + if reason = ReleaseUpgrade.offline_reason(release) + unconfigured += 1 + unconfigured_reasons << "#{release.label}: #{reason}" + result.append_remediation("Run the rollout on a connected host, or prepare both chart versions locally and set previous_chart_version in cnti-testsuite.yaml") + next + end previous = ReleaseUpgrade.previous_version(release, result) unless previous unconfigured += 1 + unconfigured_reasons << "#{release.label} has no previous release" next end # The downgrade is the judged move; the update is judged on the way @@ -167,7 +176,7 @@ ROLLING_VERSION_CHANGE_TEST_NAMES.each do |tn| elsif rolled == 0 && unconfigured == 0 && unprepared.empty? && !(batch.empty? && owned.empty? && skew.empty? && idle.empty?) result.na(rollout_not_applicable("a #{pretty_test_name}", batch, owned, skew, idle)) elsif rolled == 0 && (unconfigured > 0 || !unprepared.empty?) - result.skipped(rollout_skipped("#{tn}_test_tag", unconfigured, unprepared)) + result.skipped(rollout_skipped("#{tn}_test_tag", unconfigured, unprepared, unconfigured_reasons)) elsif task_response && restored result.passed("CNF for #{pretty_test_name_capitalized} Passed") else @@ -186,6 +195,7 @@ scored_task "rollback", rolled = 0 unconfigured = 0 + unconfigured_reasons = Set(String).new batch = Set(String).new owned = Set(String).new @@ -228,7 +238,8 @@ scored_task "rollback", config_container = container_names.find{|x| x.name == container_name } if container_names configured_tag = config_container.try(&.get_container_tag("rollback_from")) || "" rollback_from_tag = rollout_target_tag(result, configured_tag, "rollback_from_tag", - "#{resource_kind}/#{resource_name} in #{namespace}", container_name, container.as_h["image"].as_s) + "#{resource_kind}/#{resource_name} in #{namespace}", container_name, + container.as_h["image"].as_s, unconfigured_reasons) unless rollback_from_tag unconfigured += 1 next true @@ -314,9 +325,16 @@ scored_task "rollback", result.append_description("#{release.label}: no Deployment, StatefulSet or DaemonSet of the release to roll out; not judged") next end + if reason = ReleaseUpgrade.offline_reason(release) + unconfigured += 1 + unconfigured_reasons << "#{release.label}: #{reason}" + result.append_remediation("Run the rollback on a connected host, or prepare both chart versions locally and set previous_chart_version in cnti-testsuite.yaml") + next + end previous = ReleaseUpgrade.previous_version(release, result) unless previous unconfigured += 1 + unconfigured_reasons << "#{release.label} has no previous release" next end if reason = ReleaseUpgrade.move(release, previous, result, "rollback", false) @@ -340,7 +358,7 @@ scored_task "rollback", elsif rolled == 0 && unconfigured == 0 && unprepared.empty? && !(batch.empty? && owned.empty? && skew.empty? && idle.empty?) result.na(rollout_not_applicable("a rollback", batch, owned, skew, idle)) elsif rolled == 0 && (unconfigured > 0 || !unprepared.empty?) - result.skipped(rollout_skipped("rollback_from_tag", unconfigured, unprepared)) + result.skipped(rollout_skipped("rollback_from_tag", unconfigured, unprepared, unconfigured_reasons)) elsif task_response && restored result.passed("CNF Rollback Passed") else @@ -628,9 +646,13 @@ end # Why a rollout test was skipped: nothing to roll was configured, or found # in a registry, or a chart's previous release could not be installed. -def rollout_skipped(tag_key : String, unconfigured : Int32, unprepared : Array(String)) : String +def rollout_skipped(tag_key : String, unconfigured : Int32, unprepared : Array(String), reasons : Set(String)) : String parts = [] of String - parts << "No usable #{tag_key} configured for any container, and no previous release to take from the registry: see the remediation" if unconfigured > 0 + if unconfigured > 0 + why = reasons.to_a.sort.join("; ") + why = "no previous release to take from the registry" unless Offline.enabled? && !why.empty? + parts << "No usable #{tag_key} configured for any container, and #{why}: see the remediation" + end parts << "The release before could not be installed for chart#{unprepared.size == 1 ? "" : "s"} #{unprepared.join(", ")}, so there is nothing to judge: see the details" unless unprepared.empty? parts.join(". ") end @@ -651,16 +673,22 @@ end # comes first; a configured tag equal to the deployed one rolls out nothing # and is a configuration gap. Without one, the image's previous release in # its registry (#2816): the closest older tag of the same form. -def rollout_target_tag(result, configured : String, tag_key : String, label : String, container_name : String, image : String) : String? +def rollout_target_tag(result, configured : String, tag_key : String, label : String, container_name : String, image : String, reasons : Set(String)) : String? deployed = ImageTag.digest?(image) ? nil : ImageTag.tag_of(image) unless configured.empty? if configured == ImageTag.tag_of(image) + reasons << "#{tag_key} is the deployed tag" result.append_remediation("#{tag_key} #{configured} for container #{container_name} is the deployed tag, so nothing would roll out: set it to another released version") return nil end return configured end repository = ImageTag.repository_of(image) + if Offline.enabled? && deployed && ImageTag.version_form(deployed) + reasons << "offline mode cannot list the releases of #{repository}" + result.append_remediation("No #{tag_key} configured for container #{container_name}, and offline mode cannot list the releases of #{repository}: add container_names with a #{tag_key} to cnti-testsuite.yaml") + return nil + end tags = if deployed && ImageTag.version_form(deployed) ROLLOUT_REGISTRY_TAGS.fetch(repository) { ROLLOUT_REGISTRY_TAGS[repository] = ClusterTools.image_tags(repository) } end @@ -680,6 +708,7 @@ def rollout_target_tag(result, configured : String, tag_key : String, label : St else "#{repository} has no older release of the form of #{deployed}" end + reasons << why result.append_remediation("No #{tag_key} configured for container #{container_name}, and #{why}: add container_names with a #{tag_key} to cnti-testsuite.yaml") nil end @@ -816,12 +845,16 @@ scored_task "helm_chart_published", deps: ["setup:install_local_helm"], emoji: "βŽˆπŸ“¦πŸŒ" do |t, args| CNFManager::Task.task_runner(args, task: t) do |args, config, result| - helm = Helm::Binary.get charts = config.deployments.helm_charts if charts.empty? result.na("The CNF has no Helm chart deployment; only charts can be published in a repository") next end + if Offline.enabled? && charts.any? { |deployment| deployment.registry_url.empty? } + result.skipped("Offline mode: a chart served from a Helm repository is looked up there, which this run cannot reach") + next + end + helm = Helm::Binary.get # Every chart is reported with where it was looked for and what came # back; each one the repository does not know is a finding (#2598). diff --git a/src/tasks/workload/microservice.cr b/src/tasks/workload/microservice.cr index 80725274b..40f05d315 100644 --- a/src/tasks/workload/microservice.cr +++ b/src/tasks/workload/microservice.cr @@ -322,6 +322,11 @@ scored_task "reasonable_image_size", type: CNFManager::TestType::Bonus, emoji: "βš–πŸ‘€" do |t, args| CNFManager::Task.task_runner(args, task: t) do |args, config, result| + if Offline.enabled? + result.skipped("Offline mode: the image size is measured by pulling the image from its registry") + next + end + docker_insecure_registries = config.common.docker_insecure_registries || [] of String unless Dockerd.install(docker_insecure_registries) result.skipped("Skipping reasonable_image_size: Dockerd tool failed to install") diff --git a/src/tasks/workload/security.cr b/src/tasks/workload/security.cr index 7826d2e62..b608a86d5 100644 --- a/src/tasks/workload/security.cr +++ b/src/tasks/workload/security.cr @@ -827,6 +827,26 @@ scored_task "sbom_available", type: CNFManager::TestType::Normal, emoji: "πŸ“‹πŸ”’" do |t, args| CNFManager::Task.task_runner(args, task: t) do |args, config, result| + # Applicability comes before offline mode: a CNF that declares no image + # is N/A, not skipped, even when its registry would be unreachable. + declares_images = false + CNFManager.cnf_workload_resources(args, config) do |resource| + pod_spec = CNFManager.pod_spec_of(resource) + declares_images ||= CNFManager.all_containers_of(pod_spec).any? do |container| + image = container["image"]?.try(&.as_s?) + !image.nil? && !image.empty? + end + end + unless declares_images + result.na("The CNF declares no container images; SBOM availability does not apply") + next + end + + if Offline.enabled? + result.skipped("Offline mode: an SBOM is read from the image's registry, which this run cannot reach") + next + end + begin ClusterTools.install rescue ex : ClusterTools::NotReadyException