-
Notifications
You must be signed in to change notification settings - Fork 14
e2e: add kds-pcs-downtime test #1999
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
davidweisse
wants to merge
3
commits into
main
Choose a base branch
from
dav/kds-pcs-downtime
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,7 @@ on: | |
| - gpu | ||
| - imagepuller-auth | ||
| - imagestore | ||
| - kds-pcs-downtime | ||
| - memdump | ||
| - multiple-cpus | ||
| - openssl | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -52,6 +52,7 @@ jobs: | |
| - gpu | ||
| - imagepuller-auth | ||
| - imagestore | ||
| - kds-pcs-downtime | ||
| - memdump | ||
| - openssl | ||
| - peerrecovery | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,214 @@ | ||
| // Copyright 2025 Edgeless Systems GmbH | ||
| // SPDX-License-Identifier: BUSL-1.1 | ||
|
|
||
| //go:build e2e | ||
|
|
||
| package kdspcsdowntime | ||
|
|
||
| import ( | ||
| "context" | ||
| "flag" | ||
| "fmt" | ||
| "net" | ||
| "net/http" | ||
| "os" | ||
| "sync/atomic" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/edgelesssys/contrast/e2e/internal/contrasttest" | ||
| "github.com/edgelesssys/contrast/e2e/internal/kubeclient" | ||
| "github.com/edgelesssys/contrast/internal/kuberesource" | ||
| "github.com/edgelesssys/contrast/internal/manifest" | ||
| "github.com/edgelesssys/contrast/internal/platforms" | ||
| "github.com/elazarl/goproxy" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| const ( | ||
| kdsAddr = "kdsintf.amd.com:443" | ||
| pcsAddr = "api.trustedservices.intel.com:443" | ||
| ) | ||
|
|
||
| func TestKDSPCSDowntime(t *testing.T) { | ||
| platform, err := platforms.FromString(contrasttest.Flags.PlatformStr) | ||
| require.NoError(t, err) | ||
| ct := contrasttest.New(t) | ||
|
|
||
| runtimeHandler, err := manifest.RuntimeHandler(platform) | ||
| require.NoError(t, err) | ||
| resources := kuberesource.CoordinatorBundle() | ||
| resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler) | ||
| resources = kuberesource.AddPortForwarders(resources) | ||
| ct.Init(t, resources) | ||
|
|
||
| proxy := goproxy.NewProxyHttpServer() | ||
| server := http.Server{Handler: proxy} | ||
| errCh := make(chan error) | ||
|
|
||
| // If set to true, connections to KDS and PCS will be blocked by the proxy. | ||
| var blockKDSPCS atomic.Bool | ||
| // connectionProxied will be set to true if the proxy performs an HTTP CONNECT to the address of KDS or PCS. | ||
| var connectionProxied atomic.Bool | ||
| proxy.ConnectDial = func(network string, addr string) (net.Conn, error) { | ||
| t.Logf("Proxying connection: %q", addr) | ||
| if (addr == kdsAddr || addr == pcsAddr) && blockKDSPCS.Load() { | ||
| t.Logf("Blocking connection to KDS/PCS %q", addr) | ||
| connectionProxied.Store(true) | ||
| return nil, fmt.Errorf("connection to KDS/PCS %q blocked by test proxy", addr) | ||
| } | ||
| return (&net.Dialer{}).DialContext(t.Context(), network, addr) | ||
| } | ||
|
|
||
| proxyListener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:") | ||
| require.NoError(t, err) | ||
|
|
||
| t.Cleanup(func() { | ||
| require.NoError(t, server.Close()) | ||
| err := <-errCh | ||
| require.ErrorIs(t, err, http.ErrServerClosed) | ||
| }) | ||
|
|
||
| go func() { | ||
| errCh <- server.Serve(proxyListener) | ||
| }() | ||
|
|
||
| t.Setenv("https_proxy", proxyListener.Addr().String()) | ||
|
|
||
| require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests") | ||
| require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests") | ||
|
|
||
| t.Run("kds downtime", func(t *testing.T) { | ||
| if !platforms.IsSNP(platform) { | ||
| t.Skip("KDS downtime test is only applicable to SEV-SNP workloads") | ||
| } | ||
|
|
||
| require := require.New(t) | ||
|
|
||
| ctx, cancel := context.WithTimeout(t.Context(), ct.FactorPlatformTimeout(3*time.Minute)) | ||
| defer cancel() | ||
|
|
||
| require.NoError(ct.Kubeclient.WaitForCoordinator(ctx, ct.Namespace)) | ||
|
|
||
| // | ||
| // Look at dev-docs/endorsement-caching.md for table of different cases. | ||
| // | ||
|
|
||
| // Coordinator and CLI cache are empty at the beginning. | ||
|
|
||
| coordinatorPods, err := ct.Kubeclient.PodsFromOwner(ctx, ct.Namespace, "StatefulSet", "coordinator") | ||
| require.NoError(err) | ||
| require.NotEmpty(coordinatorPods, "pod not found: %s/%s", ct.Namespace, "coordinator") | ||
|
|
||
| // Block coordinator access to KDS. | ||
| etcHosts, stderr, err := ct.Kubeclient.Exec(ctx, ct.Namespace, coordinatorPods[0].Name, []string{"/bin/sh", "-c", "cat /etc/hosts"}) | ||
| require.NoError(err, "stderr: %q", stderr) | ||
| _, stderr, err = ct.Kubeclient.Exec(ctx, ct.Namespace, coordinatorPods[0].Name, []string{"/bin/sh", "-c", "echo 127.0.0.1 kdsintf.amd.com >> /etc/hosts"}) | ||
| require.NoError(err, "stderr: %q", stderr) | ||
|
|
||
| // Block CLI access to KDS. | ||
| blockKDSPCS.Store(true) | ||
|
|
||
| // Set should fail because neither coordinator nor CLI can reach KDS and there is no cached data. | ||
| // Set loop considers context deadline exceeded from KDS as a retriable error. | ||
| // Lower the timeout so the set loop doesn't exceed the test timeout. | ||
| setCtx, setCancel := context.WithTimeout(ctx, ct.FactorPlatformTimeout(1*time.Minute)) | ||
| defer setCancel() | ||
| err = ct.RunSet(setCtx) | ||
| t.Logf("Set error: %v", err) | ||
| require.ErrorContains(err, "transport: authentication handshake failed: context deadline exceeded") | ||
| require.True(connectionProxied.Load(), "expected connection to KDS to be proxied") | ||
| connectionProxied.Store(false) | ||
|
|
||
| // Unblock coordinator access to KDS. | ||
| _, stderr, err = ct.Kubeclient.Exec(ctx, ct.Namespace, coordinatorPods[0].Name, []string{"/bin/sh", "-c", fmt.Sprintf("echo '%s' > /etc/hosts", etcHosts)}) | ||
| require.NoError(err, "updating /etc/hosts: stderr: %q", stderr) | ||
|
|
||
| // Set should succeed because coordinator can reach KDS. | ||
| require.NoError(ct.RunSet(ctx)) | ||
|
|
||
| // Block coordinator access to KDS again. | ||
| _, stderr, err = ct.Kubeclient.Exec(ctx, ct.Namespace, coordinatorPods[0].Name, []string{"/bin/sh", "-c", "echo 127.0.0.1 kdsintf.amd.com >> /etc/hosts"}) | ||
| require.NoError(err, "updating /etc/hosts: stderr: %q", stderr) | ||
|
|
||
| // Verify should succeed because certs are now cached by coordinator. | ||
| require.NoError(ct.RunVerify(ctx)) | ||
|
|
||
| // Clear coordinator cache by restarting it. | ||
| require.NoError(ct.Kubeclient.Restart(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator")) | ||
| require.NoError(ct.Kubeclient.WaitForCoordinator(ctx, ct.Namespace)) | ||
|
|
||
| coordinatorPods, err = ct.Kubeclient.PodsFromOwner(ctx, ct.Namespace, "StatefulSet", "coordinator") | ||
| require.NoError(err) | ||
| require.NotEmpty(coordinatorPods, "pod not found: %s/%s", ct.Namespace, "coordinator") | ||
|
|
||
| // Block coordinator access to KDS. | ||
| _, stderr, err = ct.Kubeclient.Exec(ctx, ct.Namespace, coordinatorPods[0].Name, []string{"/bin/sh", "-c", "echo 127.0.0.1 kdsintf.amd.com >> /etc/hosts"}) | ||
| require.NoError(err, "updating /etc/hosts: stderr: %q", stderr) | ||
|
|
||
| // Unblock CLI access to KDS. | ||
| blockKDSPCS.Store(false) | ||
|
|
||
| // Recover should succeed because CLI can reach KDS. | ||
| require.NoError(ct.RunRecover(ctx)) | ||
|
|
||
| // Block CLI access to KDS again. | ||
| blockKDSPCS.Store(true) | ||
|
|
||
| // Verify should succeed because CLI has now cached the certs. | ||
| require.NoError(ct.RunVerify(ctx)) | ||
| }) | ||
|
|
||
| t.Run("pcs downtime", func(t *testing.T) { | ||
| if !platforms.IsTDX(platform) { | ||
| t.Skip("PCS downtime test is only applicable to TDX workloads") | ||
| } | ||
|
|
||
| require := require.New(t) | ||
|
|
||
| ctx, cancel := context.WithTimeout(t.Context(), ct.FactorPlatformTimeout(2*time.Minute)) | ||
| defer cancel() | ||
|
|
||
| c := kubeclient.NewForTest(t) | ||
|
|
||
| require.NoError(c.WaitForCoordinator(ctx, ct.Namespace)) | ||
|
|
||
| // | ||
| // We can't test PCS downtime on the issuer side, since PCS/PCCS are accessed from the host. | ||
| // Look at dev-docs/endorsement-caching.md for table of different cases. | ||
| // | ||
|
|
||
| // CLI cache is empty at the beginning. Block CLI access to PCS. | ||
| blockKDSPCS.Store(true) | ||
|
|
||
| // Set should fail because the CLI can't reach the PCS and there is no cached data. | ||
| // Set loop considers context deadline exceeded from PCS as a retriable error. | ||
| // Lower the timeout so the set loop doesn't exceed the test timeout. | ||
| setCtx, setCancel := context.WithTimeout(ctx, ct.FactorPlatformTimeout(1*time.Minute)) | ||
| defer setCancel() | ||
| err = ct.RunSet(setCtx) | ||
| t.Logf("Set error: %v", err) | ||
| require.ErrorContains(err, "transport: authentication handshake failed: context deadline exceeded") | ||
| require.True(connectionProxied.Load(), "expected connection to PCS to be proxied") | ||
| connectionProxied.Store(false) | ||
|
|
||
| // Unblock CLI access to PCS. | ||
| blockKDSPCS.Store(false) | ||
|
|
||
| // Set should succeed because the CLI can reach PCS. | ||
| require.NoError(ct.RunSet(ctx)) | ||
|
|
||
| // Block CLI access to PCS again. | ||
| blockKDSPCS.Store(true) | ||
|
|
||
| // Verify should succeed because collateral is now cached by CLI. | ||
| require.NoError(ct.RunVerify(ctx)) | ||
| }) | ||
| } | ||
|
|
||
| func TestMain(m *testing.M) { | ||
| contrasttest.RegisterFlags() | ||
| flag.Parse() | ||
|
|
||
| os.Exit(m.Run()) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.