Skip to content

Commit d7152fd

Browse files
authored
Merge pull request #3591 from dheerajodha/EC-1948
test: add acceptance coverage for the ITS pipeline
2 parents 332234d + ffd908e commit d7152fd

10 files changed

Lines changed: 319 additions & 3 deletions

File tree

‎acceptance/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,3 +157,24 @@ podman machine settings.
157157
* Restart the vm
158158
* `podman machine stop`
159159
* `podman machine start`
160+
161+
## ITS pipeline coverage
162+
163+
`features/pipeline_validate_image.feature` runs the checked-out ITS pipeline in
164+
kind using task bundles built from the same checkout. Only task bundle locations
165+
are redirected to the test registry; pipeline parameters, task ordering, result
166+
references, and task names are preserved. Scenarios cover trusted signed images,
167+
image lookup failures, and untrusted signatures. Both failure cases exercise both
168+
values of `STRICT`, asserting the pipeline's `TEST_OUTPUT` as well as its
169+
completion status.
170+
171+
Run these scenarios with the normal acceptance prerequisites:
172+
173+
```bash
174+
CGO_ENABLED=0 make feature_pipeline_validate_image
175+
```
176+
177+
They also run through `make acceptance` in the existing PR Checks workflow,
178+
including PRs that change only `pipelines/`. Keep task-test ConfigMap fixtures
179+
away from `konflux-info/cluster-config`: the pipeline uses that default location,
180+
so populating it with dummy keyless URLs would affect concurrent pipeline tests.

‎acceptance/kubernetes/kind/kind.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ type testState struct {
7979
namespace string
8080
policy string
8181
taskRun string
82+
pipelineRun string
8283
snapshot string
8384
registry string
8485
snapshotDigest string
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
// Copyright The Conforma Contributors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
// SPDX-License-Identifier: Apache-2.0
16+
17+
package kind
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
"time"
25+
26+
"github.com/tektoncd/cli/pkg/formatted"
27+
pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
28+
tekton "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/typed/pipeline/v1"
29+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/apimachinery/pkg/util/wait"
31+
"sigs.k8s.io/yaml"
32+
33+
"github.com/conforma/cli/acceptance/kubernetes/types"
34+
"github.com/conforma/cli/acceptance/testenv"
35+
)
36+
37+
// localPipeline loads the checked-out definition, changing only bundle locations
38+
// to use the tasks built from this checkout by buildTaskBundleImage.
39+
func localPipeline(version, name, bundle string) (*pipeline.Pipeline, error) {
40+
content, err := os.ReadFile(filepath.Join("pipelines", name, version, name+".yaml"))
41+
if err != nil {
42+
return nil, err
43+
}
44+
var definition pipeline.Pipeline
45+
if err := yaml.UnmarshalStrict(content, &definition); err != nil {
46+
return nil, err
47+
}
48+
for i := range definition.Spec.Tasks {
49+
task := &definition.Spec.Tasks[i]
50+
if task.TaskRef == nil || task.TaskRef.Resolver != "bundles" {
51+
return nil, fmt.Errorf("pipeline task %q must use a bundle resolver", task.Name)
52+
}
53+
found := false
54+
for j := range task.TaskRef.Params {
55+
param := &task.TaskRef.Params[j]
56+
if param.Name == "bundle" {
57+
param.Value = pipeline.ParamValue{Type: pipeline.ParamTypeString, StringVal: bundle}
58+
found = true
59+
}
60+
}
61+
if !found {
62+
return nil, fmt.Errorf("pipeline task %q has no bundle parameter", task.Name)
63+
}
64+
}
65+
return &definition, nil
66+
}
67+
68+
// RunPipeline runs the repository pipeline against the local task bundles.
69+
func (k *kindCluster) RunPipeline(ctx context.Context, version, name string, params map[string]string) error {
70+
t := testenv.FetchState[testState](ctx)
71+
bundle := fmt.Sprintf("registry.image-registry.svc.cluster.local:%d/ec-task-bundle:%s", k.registryPort, version)
72+
definition, err := localPipeline(version, name, bundle)
73+
if err != nil {
74+
return err
75+
}
76+
tkn, err := tekton.NewForConfig(k.config)
77+
if err != nil {
78+
return err
79+
}
80+
tknParams := make([]pipeline.Param, 0, len(params))
81+
for n, v := range params {
82+
tknParams = append(tknParams, stringParam(ctx, n, v, t))
83+
}
84+
pr, err := tkn.PipelineRuns(t.namespace).Create(ctx, &pipeline.PipelineRun{
85+
ObjectMeta: metav1.ObjectMeta{GenerateName: "acceptance-pipelinerun-"},
86+
Spec: pipeline.PipelineRunSpec{
87+
PipelineSpec: &definition.Spec,
88+
Params: tknParams,
89+
TaskRunTemplate: pipeline.PipelineTaskRunTemplate{ServiceAccountName: "default"},
90+
Timeouts: &pipeline.TimeoutFields{Pipeline: &metav1.Duration{Duration: 10 * time.Minute}},
91+
},
92+
}, metav1.CreateOptions{})
93+
if err != nil {
94+
return err
95+
}
96+
t.pipelineRun = pr.Name
97+
return nil
98+
}
99+
100+
// AwaitUntilPipelineIsDone polls with a deadline. A timeout or API failure is
101+
// always an error, never an expected negative validation result.
102+
func (k *kindCluster) AwaitUntilPipelineIsDone(ctx context.Context) (*types.PipelineInfo, error) {
103+
t := testenv.FetchState[testState](ctx)
104+
tkn, err := tekton.NewForConfig(k.config)
105+
if err != nil {
106+
return nil, err
107+
}
108+
var pr *pipeline.PipelineRun
109+
err = wait.PollUntilContextTimeout(ctx, time.Second, 11*time.Minute, true, func(ctx context.Context) (bool, error) {
110+
var err error
111+
pr, err = tkn.PipelineRuns(t.namespace).Get(ctx, t.pipelineRun, metav1.GetOptions{})
112+
if err != nil {
113+
return false, err
114+
}
115+
return pr.IsDone(), nil
116+
})
117+
if err != nil {
118+
return nil, fmt.Errorf("waiting for PipelineRun %s/%s: %w", t.namespace, t.pipelineRun, err)
119+
}
120+
results := map[string]any{}
121+
for _, result := range pr.Status.Results {
122+
results[result.Name] = paramValue(result.Value)
123+
}
124+
// Preserve condition messages (including resolution/validation failures) in
125+
// assertion errors instead of reporting just a boolean status.
126+
return &types.PipelineInfo{
127+
Name: pr.Name,
128+
Status: fmt.Sprintf("%s: %v", formatted.Condition(pr.Status.Conditions), pr.Status.Conditions),
129+
Successful: pr.IsSuccessful(),
130+
Results: results,
131+
}, nil
132+
}

‎acceptance/kubernetes/kubernetes.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,7 @@ func stepEnvVarShouldBe(ctx context.Context, stepName, envName, want string) err
494494

495495
// AddStepsTo adds cluster-related steps to the context
496496
func AddStepsTo(sc *godog.ScenarioContext) {
497+
addPipelineStepsTo(sc)
497498
sc.Step(`^a stub cluster running$`, startAndSetupState(stub.Start))
498499
sc.Step(`^a cluster running$`, startAndSetupState(kind.Start))
499500
sc.Step(`^a working namespace$`, createNamespace)

‎acceptance/kubernetes/pipeline.go‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Copyright The Conforma Contributors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
// SPDX-License-Identifier: Apache-2.0
16+
17+
package kubernetes
18+
19+
import (
20+
"context"
21+
"encoding/json"
22+
"fmt"
23+
24+
"github.com/cucumber/godog"
25+
26+
"github.com/conforma/cli/acceptance/testenv"
27+
)
28+
29+
func runPipeline(ctx context.Context, version, name string, params *godog.Table) error {
30+
c := testenv.FetchState[ClusterState](ctx)
31+
if err := mustBeUp(ctx, *c); err != nil {
32+
return err
33+
}
34+
values := map[string]string{}
35+
for _, row := range params.Rows {
36+
values[row.Cells[0].Value] = row.Cells[1].Value
37+
}
38+
return c.cluster.RunPipeline(ctx, version, name, values)
39+
}
40+
41+
func pipelineShouldComplete(ctx context.Context, outcome string) error {
42+
c := testenv.FetchState[ClusterState](ctx)
43+
if err := mustBeUp(ctx, *c); err != nil {
44+
return err
45+
}
46+
info, err := c.cluster.AwaitUntilPipelineIsDone(ctx)
47+
if err != nil {
48+
return err
49+
}
50+
if info.Successful != (outcome == "succeed") {
51+
return fmt.Errorf("PipelineRun %s should %s, got %s", info.Name, outcome, info.Status)
52+
}
53+
return nil
54+
}
55+
56+
func pipelineReportShouldHaveResult(ctx context.Context, expected string) error {
57+
c := testenv.FetchState[ClusterState](ctx)
58+
if err := mustBeUp(ctx, *c); err != nil {
59+
return err
60+
}
61+
info, err := c.cluster.AwaitUntilPipelineIsDone(ctx)
62+
if err != nil {
63+
return err
64+
}
65+
raw, ok := info.Results["TEST_OUTPUT"].(string)
66+
if !ok {
67+
return fmt.Errorf("PipelineRun %s has no string TEST_OUTPUT result: %s", info.Name, info.Status)
68+
}
69+
var report struct {
70+
Result string `json:"result"`
71+
}
72+
if err := json.Unmarshal([]byte(raw), &report); err != nil {
73+
return fmt.Errorf("invalid pipeline TEST_OUTPUT %q: %w", raw, err)
74+
}
75+
if report.Result != expected {
76+
return fmt.Errorf("pipeline TEST_OUTPUT result: want %q, got %s", expected, raw)
77+
}
78+
return nil
79+
}
80+
81+
func addPipelineStepsTo(sc *godog.ScenarioContext) {
82+
sc.Step(`^version ([\d.]+) of the pipeline named "([^"]*)" is run with parameters:$`, runPipeline)
83+
sc.Step(`^the pipeline should (succeed|fail)$`, pipelineShouldComplete)
84+
sc.Step(`^the pipeline TEST_OUTPUT should report "([^"]*)"$`, pipelineReportShouldHaveResult)
85+
}

‎acceptance/kubernetes/stub/stub.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,3 +214,11 @@ func (s stubCluster) Registry(ctx context.Context) (string, error) {
214214
func (s stubCluster) BuildSnapshotArtifact(ctx context.Context, content string) (context.Context, error) {
215215
return ctx, nil
216216
}
217+
218+
func (s stubCluster) RunPipeline(_ context.Context, _, _ string, _ map[string]string) error {
219+
return errors.New("can't run pipelines when using the stub Kubernetes")
220+
}
221+
222+
func (s stubCluster) AwaitUntilPipelineIsDone(context.Context) (*types.PipelineInfo, error) {
223+
return nil, errors.New("can't run pipelines when using the stub Kubernetes")
224+
}

‎acceptance/kubernetes/types/types.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ type Cluster interface {
2727
CreateNamespace(context.Context) (context.Context, error)
2828
CreateNamedPolicy(context.Context, string, string) error
2929
CreatePolicy(context.Context, string) error
30+
RunPipeline(context.Context, string, string, map[string]string) error
31+
AwaitUntilPipelineIsDone(context.Context) (*PipelineInfo, error)
3032
RunTask(context.Context, string, string, string, map[string]string) error
3133
AwaitUntilTaskIsDone(context.Context) (bool, error)
3234
TaskInfo(context.Context) (*TaskInfo, error)
@@ -52,3 +54,11 @@ type Step struct {
5254
Logs string
5355
EnvVars map[string]string
5456
}
57+
58+
// PipelineInfo describes a completed PipelineRun.
59+
type PipelineInfo struct {
60+
Name string
61+
Status string
62+
Successful bool
63+
Results map[string]any
64+
}

‎features/__snapshots__/task_validate_image.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ true
452452
---
453453

454454
[TestFeatures/Collect keyless signing parameters from ConfigMap:collect-signing-params - 1]
455-
Reading ConfigMap konflux-info/cluster-config
455+
Reading ConfigMap konflux-info/cluster-config-keyless
456456
ConfigMap found, extracting keyless signing parameters
457457
results.keylessSigningEnabled: true
458458
results.defaultOIDCIssuer: https://kubernetes.default.svc
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
Feature: Verify images through the ITS pipeline
2+
Exercise the repository pipeline with locally built task bundles in kind.
3+
4+
Background:
5+
Given a cluster running
6+
And stub rekord running
7+
And stub tuf running
8+
And a working namespace
9+
And a key pair named "known"
10+
And a cluster policy with content:
11+
```
12+
{
13+
"publicKey": ${known_PUBLIC_KEY}
14+
}
15+
```
16+
17+
Scenario: ITS pipeline validates a signed image
18+
Given an image named "acceptance/its-signed"
19+
And a valid image signature of "acceptance/its-signed" image signed by the "known" key
20+
And a valid attestation of "acceptance/its-signed" signed by the "known" key
21+
When version 0.1 of the pipeline named "enterprise-contract" is run with parameters:
22+
| SNAPSHOT | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-signed"}]} |
23+
| PUBLIC_KEY | ${known_PUBLIC_KEY} |
24+
| POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME} |
25+
Then the pipeline should succeed
26+
And the pipeline TEST_OUTPUT should report "SUCCESS"
27+
28+
Scenario Outline: ITS pipeline handles image lookup failures according to STRICT
29+
When version 0.1 of the pipeline named "enterprise-contract" is run with parameters:
30+
| SNAPSHOT | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-missing"}]} |
31+
| PUBLIC_KEY | ${known_PUBLIC_KEY} |
32+
| POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME} |
33+
| STRICT | <strict> |
34+
Then the pipeline should <outcome>
35+
And the pipeline TEST_OUTPUT should report "FAILURE"
36+
37+
Examples:
38+
| strict | outcome |
39+
| true | fail |
40+
| false | succeed |
41+
42+
Scenario Outline: ITS pipeline handles untrusted signatures according to STRICT
43+
Given a key pair named "untrusted"
44+
And an image named "acceptance/its-untrusted"
45+
And a valid image signature of "acceptance/its-untrusted" image signed by the "untrusted" key
46+
And a valid attestation of "acceptance/its-untrusted" signed by the "untrusted" key
47+
When version 0.1 of the pipeline named "enterprise-contract" is run with parameters:
48+
| SNAPSHOT | {"components": [{"containerImage": "${REGISTRY}/acceptance/its-untrusted"}]} |
49+
| PUBLIC_KEY | ${known_PUBLIC_KEY} |
50+
| POLICY_CONFIGURATION | ${NAMESPACE}/${POLICY_NAME} |
51+
| STRICT | <strict> |
52+
Then the pipeline should <outcome>
53+
And the pipeline TEST_OUTPUT should report "FAILURE"
54+
55+
Examples:
56+
| strict | outcome |
57+
| true | fail |
58+
| false | succeed |

‎features/task_validate_image.feature‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -547,7 +547,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
547547
# https://github.com/redhat-appstudio/tsf-cli/blob/84561ca6c9/installer/charts/tsf-konflux/templates/konflux.yaml#L51-L65
548548
# Note: These scenarios might run in parallel so let's use a different config map
549549
# for each scenario so we don't have to worry about them clashing with each other
550-
And a ConfigMap "cluster-config" in namespace "konflux-info" with content:
550+
And a ConfigMap "cluster-config-keyless" in namespace "konflux-info" with content:
551551
# tufExternalUrl should be ignored here because tufInternalUrl takes precedence
552552
```
553553
{
@@ -562,7 +562,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
562562
}
563563
```
564564
When version 0.1 of the task named "collect-keyless-params" is run with parameters:
565-
| configMapName | cluster-config |
565+
| configMapName | cluster-config-keyless |
566566
Then the task should succeed
567567
And the task logs for step "collect-signing-params" should match the snapshot
568568
And the task result "keylessSigningEnabled" should equal "true"

0 commit comments

Comments
 (0)