diff --git a/pkg/testsuites/standard_suites.go b/pkg/testsuites/standard_suites.go index 13894410bcb5..2a33d4826d91 100644 --- a/pkg/testsuites/standard_suites.go +++ b/pkg/testsuites/standard_suites.go @@ -260,7 +260,7 @@ var staticSuites = []ginkgo.TestSuite{ return strings.Contains(name, "[Suite:openshift/network/ipsec") }, Parallelism: 1, - TestTimeout: 120 * time.Minute, + TestTimeout: 60 * time.Minute, }, { Name: "openshift/network/stress", diff --git a/test/extended/networking/ipsec.go b/test/extended/networking/ipsec.go index 563ae54079a1..3dbf8a781129 100644 --- a/test/extended/networking/ipsec.go +++ b/test/extended/networking/ipsec.go @@ -1,9 +1,11 @@ package networking import ( + "bufio" "context" "fmt" "os" + "regexp" "strings" "time" @@ -11,11 +13,15 @@ import ( exutil "github.com/openshift/origin/test/extended/util" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + "k8s.io/kubernetes/test/e2e/framework/statefulset" admissionapi "k8s.io/pod-security-admission/api" g "github.com/onsi/ginkgo/v2" @@ -38,12 +44,14 @@ const ( nsCertMachineConfigName = "99-worker-north-south-ipsec-config" leftNodeIPsecPolicyName = "left-node-ipsec-policy" rightNodeIPsecPolicyName = "right-node-ipsec-policy" - leftNodeIPsecConfigYaml = "ipsec-left-node.yaml" - rightNodeIPsecConfigYaml = "ipsec-right-node.yaml" + leftNodeIPsecNNCPYaml = "ipsec-left-node.yaml" + rightNodeIPsecNNCPYaml = "ipsec-right-node.yaml" ovnNamespace = "openshift-ovn-kubernetes" ovnIPsecDsName = "ovn-ipsec-host" ) +var gvrNodeNetworkConfigurationPolicy = schema.GroupVersionResource{Group: "nmstate.io", Version: "v1", Resource: "nodenetworkconfigurationpolicies"} + // TODO: consider bringing in the NNCP api. var nodeIPsecConfigManifest = ` kind: NodeNetworkConfigurationPolicy @@ -132,10 +140,10 @@ func getIPsecMode(oc *exutil.CLI) (v1.IPsecMode, error) { return mode, nil } -// ensureIPsecEnabled this function ensure IPsec is enabled by making sure ovn-ipsec-host daemonset +// ensureIPsecFullEnabled this function ensure IPsec is enabled by making sure ovn-ipsec-host daemonset // is completely ready on the cluster and cluster operators are coming back into ready state // once ipsec rollout is complete. -func ensureIPsecEnabled(oc *exutil.CLI) error { +func ensureIPsecFullEnabled(oc *exutil.CLI) error { return wait.PollUntilContextTimeout(context.Background(), ipsecRolloutWaitInterval, ipsecRolloutWaitDuration, true, func(ctx context.Context) (bool, error) { done, err := areMachineConfigPoolsReadyWithIPsec(oc) @@ -159,12 +167,22 @@ func ensureIPsecEnabled(oc *exutil.CLI) error { }) } -// ensureIPsecMachineConfigRolloutComplete this function ensures ipsec machine config extension is rolled out +// ensureIPsecExternalEnabled this function ensures ipsec machine config extension is rolled out // on all of master and worked nodes and cluster operators are coming back into ready state // once ipsec rollout is complete. -func ensureIPsecMachineConfigRolloutComplete(oc *exutil.CLI) error { +func ensureIPsecExternalEnabled(oc *exutil.CLI) error { return wait.PollUntilContextTimeout(context.Background(), ipsecRolloutWaitInterval, ipsecRolloutWaitDuration, true, func(ctx context.Context) (bool, error) { + // Make sure ovn-ipsec-host daemonset is not deployed. When IPsec mode + // is changed from Full to External mode, then it may take a while to + // delete daemonset. + ds, err := getDaemonSet(oc, ovnNamespace, ovnIPsecDsName) + if err != nil && !isConnResetErr(err) { + return false, err + } + if ds != nil { + return false, nil + } done, err := areMachineConfigPoolsReadyWithIPsec(oc) if err != nil && !isConnResetErr(err) { return false, err @@ -212,15 +230,16 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { f := oc.KubeFramework() waitForIPsecNSConfigApplied := func() { - o.Eventually(func() bool { + g.GinkgoHelper() + o.Eventually(func(g o.Gomega) bool { out, err := oc.AsAdmin().Run("get").Args("NodeNetworkConfigurationPolicy/"+leftNodeIPsecPolicyName, "-o", "yaml").Output() - o.Expect(err).NotTo(o.HaveOccurred()) + g.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("rendered left node network config policy:\n%s", out) if !strings.Contains(out, "1/1 nodes successfully configured") { return false } out, err = oc.AsAdmin().Run("get").Args("NodeNetworkConfigurationPolicy/"+rightNodeIPsecPolicyName, "-o", "yaml").Output() - o.Expect(err).NotTo(o.HaveOccurred()) + g.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("rendered right node network config policy:\n%s", out) return strings.Contains(out, "1/1 nodes successfully configured") }, 30*time.Second).Should(o.BeTrue()) @@ -340,6 +359,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } cleanupTestPods := func(config *testConfig) { + g.GinkgoHelper() err := e2epod.DeletePodWithWait(context.Background(), f.ClientSet, config.srcNodeConfig.pingPod) o.Expect(err).NotTo(o.HaveOccurred()) config.srcNodeConfig.pingPod = nil @@ -356,6 +376,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } checkForGeneveOnlyPodTraffic := func(config *testConfig) { + g.GinkgoHelper() err := setupTestPods(config, false) o.Expect(err).NotTo(o.HaveOccurred()) defer func() { @@ -371,9 +392,11 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { o.Expect(err).To(o.HaveOccurred()) err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp) o.Expect(err).To(o.HaveOccurred()) + err = nil } checkForESPOnlyPodTraffic := func(config *testConfig) { + g.GinkgoHelper() err := setupTestPods(config, false) o.Expect(err).NotTo(o.HaveOccurred()) defer func() { @@ -387,9 +410,11 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { o.Expect(err).NotTo(o.HaveOccurred()) err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, geneve) o.Expect(err).To(o.HaveOccurred()) + err = nil } checkPodTraffic := func(mode v1.IPsecMode) { + g.GinkgoHelper() if mode == v1.IPsecModeFull { checkForESPOnlyPodTraffic(config) } else { @@ -398,6 +423,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } checkNodeTraffic := func(mode v1.IPsecMode) { + g.GinkgoHelper() err := setupTestPods(config, true) o.Expect(err).NotTo(o.HaveOccurred()) defer func() { @@ -412,6 +438,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { o.Expect(err).NotTo(o.HaveOccurred()) err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp) o.Expect(err).To(o.HaveOccurred()) + err = nil return } else { err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp) @@ -424,10 +451,17 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { // the selected nodes. ipsecMode, err := getIPsecMode(oc) o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(ipsecMode).To(o.Equal(v1.IPsecModeFull)) srcNode, dstNode := &testNodeConfig{}, &testNodeConfig{} config = &testConfig{ipsecMode: ipsecMode, srcNodeConfig: srcNode, dstNodeConfig: dstNode} + + // Deploy nmstate handler which is used for rolling out IPsec config + // via NodeNetworkConfigurationPolicy. + g.By("deploy nmstate handler") + err = deployNmstateHandler(oc) + o.Expect(err).NotTo(o.HaveOccurred()) }) g.BeforeEach(func() { @@ -455,54 +489,93 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } } o.Expect(config.dstNodeConfig.nodeIP).NotTo(o.BeEmpty()) + + g.By("configure IPsec certs on the worker nodes") + // The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024. + // so proceed with test if system date is before April 10, 2034. Otherwise fail the test. + if !time.Now().Before(certExpirationDate) { + framework.Failf("certficates in the Machine Config are expired, Please consider recreating those certificates") + } + nsCertMachineConfig, err := createIPsecCertsMachineConfig(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(nsCertMachineConfig).NotTo(o.BeNil()) + o.Eventually(func(g o.Gomega) bool { + pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) + g.Expect(err).NotTo(o.HaveOccurred()) + return areMachineConfigPoolsReadyWithMachineConfig(pools, nsCertMachineConfigName) + }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) + // wait for ovn-ipsec-host pod to get rolled out after certs installation. + waitForIPsecConfigToComplete(oc, config.ipsecMode) }) g.AfterEach(func() { + ipsecMode, err := getIPsecMode(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + if g.CurrentSpecReport().Failed() { + if ipsecMode == v1.IPsecModeFull { + var ipsecPods []string + srcIPsecPod, err := findIPsecPodonNode(oc, config.srcNodeConfig.nodeName) + o.Expect(err).NotTo(o.HaveOccurred()) + ipsecPods = append(ipsecPods, srcIPsecPod) + dstIPsecPod, err := findIPsecPodonNode(oc, config.dstNodeConfig.nodeName) + o.Expect(err).NotTo(o.HaveOccurred()) + ipsecPods = append(ipsecPods, dstIPsecPod) + for _, ipsecPod := range ipsecPods { + dumpPodCommand(ovnNamespace, ipsecPod, "cat /etc/ipsec.conf") + dumpPodCommand(ovnNamespace, ipsecPod, "cat /etc/ipsec.d/openshift.conf") + dumpPodCommand(ovnNamespace, ipsecPod, "ipsec status") + dumpPodCommand(ovnNamespace, ipsecPod, "ipsec trafficstatus") + dumpPodCommand(ovnNamespace, ipsecPod, "ip xfrm state") + dumpPodCommand(ovnNamespace, ipsecPod, "ip xfrm policy") + } + } + exutil.DumpPodStatesInNamespace(nmstateNamespace, oc) + exutil.DumpPodLogsStartingWithInNamespace("nmstate-handler", nmstateNamespace, oc) + exutil.DumpPodLogsStartingWithInNamespace("nmstate-operator", nmstateNamespace, oc) + } + g.By("remove left node ipsec configuration") + oc.AsAdmin().Run("delete").Args("-f", leftNodeIPsecNNCPYaml).Execute() + o.Eventually(func(g o.Gomega) bool { + _, err := oc.AdminDynamicClient().Resource(gvrNodeNetworkConfigurationPolicy).Get(context.Background(), + leftNodeIPsecPolicyName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return true + } + g.Expect(err).NotTo(o.HaveOccurred()) + return false + }).Should(o.Equal(true)) + + g.By("remove right node ipsec configuration") + oc.AsAdmin().Run("delete").Args("-f", rightNodeIPsecNNCPYaml).Execute() + o.Eventually(func(g o.Gomega) bool { + _, err := oc.AdminDynamicClient().Resource(gvrNodeNetworkConfigurationPolicy).Get(context.Background(), + rightNodeIPsecPolicyName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return true + } + g.Expect(err).NotTo(o.HaveOccurred()) + return false + }).Should(o.Equal(true)) + + // Removal of IPsec certs are needed otherwise worker nodes still keeping + // stale ip xfrm state and policy entries created for north south traffic. g.By("removing IPsec certs from worker nodes") - err := deleteNSCertMachineConfig(oc) + err = deleteNSCertMachineConfig(oc) o.Expect(err).NotTo(o.HaveOccurred()) - o.Eventually(func() bool { + o.Eventually(func(g o.Gomega) bool { pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) - o.Expect(err).NotTo(o.HaveOccurred()) + g.Expect(err).NotTo(o.HaveOccurred()) return areMachineConfigPoolsReadyWithoutMachineConfig(pools, nsCertMachineConfigName) }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) - g.By("remove right node ipsec configuration") - oc.AsAdmin().Run("delete").Args("-f", rightNodeIPsecConfigYaml).Execute() - - g.By("remove left node ipsec configuration") - oc.AsAdmin().Run("delete").Args("-f", leftNodeIPsecConfigYaml).Execute() - - g.By("undeploy nmstate handler") - undeployNmstateHandler(oc) - - // Restore the cluster back into original state after running all the tests. + // Restore the cluster back into original state after running each test. g.By("restoring ipsec config into original state") err = configureIPsecMode(oc, config.ipsecMode) o.Expect(err).NotTo(o.HaveOccurred()) waitForIPsecConfigToComplete(oc, config.ipsecMode) }) - g.DescribeTable("check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) { - o.Expect(config).NotTo(o.BeNil()) - g.By("validate traffic before changing IPsec configuration") - checkPodTraffic(config.ipsecMode) - // This test does not enable N/S ipsec config, so node traffic behaves as it were disabled - checkNodeTraffic(v1.IPsecModeDisabled) - g.By(fmt.Sprintf("configure IPsec in %s mode and validate traffic", mode)) - err := configureIPsecMode(oc, mode) - o.Expect(err).NotTo(o.HaveOccurred()) - waitForIPsecConfigToComplete(oc, mode) - checkPodTraffic(mode) - // This test does not enable N/S ipsec config, so node traffic behaves as it were disabled - checkNodeTraffic(v1.IPsecModeDisabled) - }, - g.Entry("with IPsec in full mode", v1.IPsecModeFull), - g.Entry("with IPsec in external mode", v1.IPsecModeExternal), - g.Entry("with IPsec in disabled mode", v1.IPsecModeDisabled), - ) - - g.DescribeTable("check traffic for north south IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) { + g.DescribeTable("check traffic [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) { o.Expect(config).NotTo(o.BeNil()) g.By("validate traffic before changing IPsec configuration") @@ -511,8 +584,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { checkNodeTraffic(v1.IPsecModeDisabled) g.By(fmt.Sprintf("configure IPsec in %s mode and validate traffic", mode)) - // Change IPsec mode to External and packet capture on the node's interface - // must be geneve encapsulated ones. + // Change IPsec mode to given mode and do packet capture on the node's interface err := configureIPsecMode(oc, mode) o.Expect(err).NotTo(o.HaveOccurred()) waitForIPsecConfigToComplete(oc, mode) @@ -520,42 +592,29 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { // N/S ipsec config is not in effect yet, so node traffic behaves as it were disabled checkNodeTraffic(v1.IPsecModeDisabled) - g.By("configure IPsec certs on the worker nodes") - // The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024. - // so proceed with test if system date is before April 10, 2034. Otherwise fail the test. - if !time.Now().Before(certExpirationDate) { - framework.Failf("certficates in the Machine Config are expired, Please consider recreating those certificates") + // TODO: remove this block when https://issues.redhat.com/browse/RHEL-67307 is fixed. + if mode == v1.IPsecModeFull { + g.By(fmt.Sprintf("skip testing IPsec NS configuration with %s mode due to nmstate bug RHEL-67307", mode)) + return } - nsCertMachineConfig, err := createIPsecCertsMachineConfig(oc) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(nsCertMachineConfig).NotTo(o.BeNil()) - o.Eventually(func() bool { - pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) - o.Expect(err).NotTo(o.HaveOccurred()) - return areMachineConfigPoolsReadyWithMachineConfig(pools, nsCertMachineConfigName) - }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) - - // Deploy nmstate handler which is used for rolling out IPsec config - // via NodeNetworkConfigurationPolicy. - g.By("deploy nmstate handler") - err = deployNmstateHandler(oc) - o.Expect(err).NotTo(o.HaveOccurred()) g.By("rollout IPsec configuration via nmstate") + err = ensureNmstateHandlerRunning(oc) + o.Expect(err).NotTo(o.HaveOccurred()) leftConfig := fmt.Sprintf(nodeIPsecConfigManifest, leftNodeIPsecPolicyName, config.srcNodeConfig.nodeName, config.srcNodeConfig.nodeIP, leftServerCertName, config.dstNodeConfig.nodeIP) - err = os.WriteFile(leftNodeIPsecConfigYaml, []byte(leftConfig), 0644) + err = os.WriteFile(leftNodeIPsecNNCPYaml, []byte(leftConfig), 0644) o.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("desired left node network config policy:\n%s", leftConfig) - err = oc.AsAdmin().Run("apply").Args("-f", leftNodeIPsecConfigYaml).Execute() + err = oc.AsAdmin().Run("apply").Args("-f", leftNodeIPsecNNCPYaml).Execute() o.Expect(err).NotTo(o.HaveOccurred()) rightConfig := fmt.Sprintf(nodeIPsecConfigManifest, rightNodeIPsecPolicyName, config.dstNodeConfig.nodeName, config.dstNodeConfig.nodeIP, rightServerCertName, config.srcNodeConfig.nodeIP) - err = os.WriteFile(rightNodeIPsecConfigYaml, []byte(rightConfig), 0644) + err = os.WriteFile(rightNodeIPsecNNCPYaml, []byte(rightConfig), 0644) o.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("desired right node network config policy:\n%s", rightConfig) - err = oc.AsAdmin().Run("apply").Args("-f", rightNodeIPsecConfigYaml).Execute() + err = oc.AsAdmin().Run("apply").Args("-f", rightNodeIPsecNNCPYaml).Execute() o.Expect(err).NotTo(o.HaveOccurred()) g.By("wait for nmstate to roll out") @@ -568,20 +627,56 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { }, g.Entry("with IPsec in full mode", v1.IPsecModeFull), g.Entry("with IPsec in external mode", v1.IPsecModeExternal), + // TODO add test for v1.IPsecModeDisabled mode once IPsec tests stabilized in CI. ) }) }) func waitForIPsecConfigToComplete(oc *exutil.CLI, ipsecMode v1.IPsecMode) { + g.GinkgoHelper() switch ipsecMode { case v1.IPsecModeDisabled: err := ensureIPsecDisabled(oc) o.Expect(err).NotTo(o.HaveOccurred()) case v1.IPsecModeExternal: - err := ensureIPsecMachineConfigRolloutComplete(oc) + err := ensureIPsecExternalEnabled(oc) o.Expect(err).NotTo(o.HaveOccurred()) case v1.IPsecModeFull: - err := ensureIPsecEnabled(oc) + err := ensureIPsecFullEnabled(oc) o.Expect(err).NotTo(o.HaveOccurred()) } } + +func findIPsecPodonNode(oc *exutil.CLI, nodeName string) (string, error) { + out, err := runOcWithRetry(oc.AsAdmin(), "get", + "pods", + "-o", "name", + "-n", ovnNamespace, + "--field-selector", fmt.Sprintf("spec.nodeName=%s", nodeName), + "-l", "app=ovn-ipsec") + if err != nil { + return "", err + } + outReader := bufio.NewScanner(strings.NewReader(out)) + re := regexp.MustCompile("^pod/(.*)") + var podName string + for outReader.Scan() { + match := re.FindSubmatch([]byte(outReader.Text())) + if len(match) != 2 { + continue + } + podName = string(match[1]) + break + } + if podName == "" { + return "", fmt.Errorf("could not find a valid ovn-ipsec-host pod on node '%s'", nodeName) + } + return podName, nil +} + +func dumpPodCommand(namespace, name, cmd string) { + g.GinkgoHelper() + stdout, err := e2eoutput.RunHostCmdWithRetries(namespace, name, cmd, statefulset.StatefulSetPoll, statefulset.StatefulPodTimeout) + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf(name + ": " + strings.Join(strings.Split(stdout, "\n"), fmt.Sprintf("\n%s: ", name))) +} diff --git a/test/extended/networking/util.go b/test/extended/networking/util.go index 8b5efb700eed..963e9fb84558 100644 --- a/test/extended/networking/util.go +++ b/test/extended/networking/util.go @@ -713,11 +713,15 @@ func deployNmstateHandler(oc *exutil.CLI) error { return fmt.Errorf("nmstate operator is not running: %v", err) } nmStateConfigYaml := exutil.FixturePath("testdata", "ipsec", nmstateConfigureManifestFile) - err = oc.AsAdmin().Run("create").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute() + err = oc.AsAdmin().Run("apply").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute() if err != nil { return fmt.Errorf("error configuring nmstate: %v", err) } - err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, + return ensureNmstateHandlerRunning(oc) +} + +func ensureNmstateHandlerRunning(oc *exutil.CLI) error { + err := wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, func(ctx context.Context) (bool, error) { // Ensure nmstate handler is running. return isDaemonSetRunning(oc, nmstateNamespace, "nmstate-handler") @@ -732,26 +736,6 @@ func deployNmstateHandler(oc *exutil.CLI) error { return nil } -func undeployNmstateHandler(oc *exutil.CLI) error { - nmStateConfigYaml := exutil.FixturePath("testdata", "ipsec", nmstateConfigureManifestFile) - err := oc.AsAdmin().Run("delete").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute() - if err != nil { - return fmt.Errorf("error deleting nmstate configuration: %v", err) - } - err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, - func(ctx context.Context) (bool, error) { - _, err := oc.AdminKubeClient().AppsV1().DaemonSets(nmstateNamespace).Get(context.Background(), "nmstate-handler", metav1.GetOptions{}) - if err != nil && apierrors.IsNotFound(err) { - return true, nil - } - return false, err - }) - if err != nil { - return fmt.Errorf("failed to remove nmstate crd and handler: %v", err) - } - return nil -} - func waitForDeploymentComplete(oc *exutil.CLI, namespace, name string) error { deployment, err := oc.AdminKubeClient().AppsV1().Deployments(namespace).Get(context.Background(), name, metav1.GetOptions{}) if err != nil { @@ -782,12 +766,19 @@ func getDaemonSet(oc *exutil.CLI, namespace, name string) (*appsv1.DaemonSet, er } func createIPsecCertsMachineConfig(oc *exutil.CLI) (*mcfgv1.MachineConfig, error) { + nsCertMachineConfig, err := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigs().Get(context.Background(), + nsCertMachineConfigName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + if err == nil { + return nsCertMachineConfig, nil + } ipSecCertsMachineConfig := exutil.FixturePath("testdata", "ipsec", nsCertMachineConfigFile) - err := oc.AsAdmin().Run("create").Args("-f", ipSecCertsMachineConfig).Execute() + err = oc.AsAdmin().Run("create").Args("-f", ipSecCertsMachineConfig).Execute() if err != nil { return nil, fmt.Errorf("error deploying IPsec certs Machine Config: %v", err) } - var nsCertMachineConfig *mcfgv1.MachineConfig err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, func(ctx context.Context) (bool, error) { nsCertMachineConfig, err = oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigs().Get(context.Background(), nsCertMachineConfigName, metav1.GetOptions{}) diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index d5b95cc582d0..4200d8e227c6 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -1477,15 +1477,9 @@ var Annotations = map[string]string{ "[sig-network][Feature:EgressRouterCNI] when using openshift ovn-kubernetes should ensure ipv6 egressrouter cni resources are created [apigroup:operator.openshift.io]": " [Suite:openshift/conformance/parallel]", - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in disabled mode": "", + "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in external mode": "", - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in external mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in full mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for north south IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in external mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for north south IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in full mode": "", + "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in full mode": "", "[sig-network][Feature:MultiNetworkPolicy][Serial][apigroup:operator.openshift.io] should enforce a network policies on secondary network IPv4": " [Suite:openshift/conformance/serial]", diff --git a/test/extended/util/clusterversion.go b/test/extended/util/clusterversion.go index c66209c1ba8b..d2061bc2cc83 100644 --- a/test/extended/util/clusterversion.go +++ b/test/extended/util/clusterversion.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" configv1 "github.com/openshift/api/config/v1" @@ -68,9 +69,10 @@ func DetermineImageFromRelease(ctx context.Context, oc *CLI, imageTagName string if len(releaseImage) == 0 { return "", fmt.Errorf("cannot determine release image from ClusterVersion resource") } + podName := "extract-release-imagerefs" podClient := e2epod.PodClientNS(oc.KubeFramework(), oc.Namespace()) podClient.CreateSync(ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "extract-release-imagerefs"}, + ObjectMeta: metav1.ObjectMeta{Name: podName}, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { @@ -81,8 +83,15 @@ func DetermineImageFromRelease(ctx context.Context, oc *CLI, imageTagName string }, }, }) - defer podClient.Delete(ctx, "extract-release-imagerefs", metav1.DeleteOptions{}) - imageRefsString := e2epod.ExecShellInContainer(oc.KubeFramework(), "extract-release-imagerefs", "imagerefs", "cat /release-manifests/image-references") + defer func() { + podClient.Delete(ctx, podName, metav1.DeleteOptions{}) + err = e2epod.WaitForPodNotFoundInNamespace(ctx, oc.kubeFramework.ClientSet, + podName, oc.Namespace(), e2epod.PodDeleteTimeout) + if err != nil { + framework.Logf("pod %q is still found in namespace %q", podName, oc.Namespace()) + } + }() + imageRefsString := e2epod.ExecShellInContainer(oc.KubeFramework(), podName, "imagerefs", "cat /release-manifests/image-references") imageRefs := struct { Spec struct { Tags []struct {