diff --git a/pkg/testsuites/standard_suites.go b/pkg/testsuites/standard_suites.go index 13894410bcb5..2a33d4826d91 100644 --- a/pkg/testsuites/standard_suites.go +++ b/pkg/testsuites/standard_suites.go @@ -260,7 +260,7 @@ var staticSuites = []ginkgo.TestSuite{ return strings.Contains(name, "[Suite:openshift/network/ipsec") }, Parallelism: 1, - TestTimeout: 120 * time.Minute, + TestTimeout: 60 * time.Minute, }, { Name: "openshift/network/stress", diff --git a/test/extended/networking/ipsec.go b/test/extended/networking/ipsec.go index 563ae54079a1..4a433274d438 100644 --- a/test/extended/networking/ipsec.go +++ b/test/extended/networking/ipsec.go @@ -1,21 +1,27 @@ package networking import ( + "bufio" "context" "fmt" "os" + "regexp" "strings" "time" v1 "github.com/openshift/api/operator/v1" + mg "github.com/openshift/origin/test/extended/machine_config" exutil "github.com/openshift/origin/test/extended/util" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + "k8s.io/kubernetes/test/e2e/framework/statefulset" admissionapi "k8s.io/pod-security-admission/api" g "github.com/onsi/ginkgo/v2" @@ -34,16 +40,17 @@ const ( ipsecRolloutWaitDuration = 40 * time.Minute ipsecRolloutWaitInterval = 1 * time.Minute nmstateConfigureManifestFile = "nmstate.yaml" - nsCertMachineConfigFile = "ipsec-nsconfig-machine-config.yaml" nsCertMachineConfigName = "99-worker-north-south-ipsec-config" leftNodeIPsecPolicyName = "left-node-ipsec-policy" rightNodeIPsecPolicyName = "right-node-ipsec-policy" - leftNodeIPsecConfigYaml = "ipsec-left-node.yaml" - rightNodeIPsecConfigYaml = "ipsec-right-node.yaml" + leftNodeIPsecNNCPYaml = "ipsec-left-node.yaml" + rightNodeIPsecNNCPYaml = "ipsec-right-node.yaml" ovnNamespace = "openshift-ovn-kubernetes" ovnIPsecDsName = "ovn-ipsec-host" ) +var gvrNodeNetworkConfigurationPolicy = schema.GroupVersionResource{Group: "nmstate.io", Version: "v1", Resource: "nodenetworkconfigurationpolicies"} + // TODO: consider bringing in the NNCP api. var nodeIPsecConfigManifest = ` kind: NodeNetworkConfigurationPolicy @@ -94,27 +101,6 @@ const ( icmp trafficType = "icmp" ) -// configureIPsecMode helps to rollout specified IPsec Mode on the cluster. If the cluster is already -// configured with specified mode, then this is almost like no-op for the cluster. -func configureIPsecMode(oc *exutil.CLI, ipsecMode v1.IPsecMode) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - network, err := oc.AdminOperatorClient().OperatorV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{}) - if err != nil { - return err - } - if network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig == nil { - network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig = &v1.IPsecConfig{Mode: ipsecMode} - } else if network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig.Mode != ipsecMode { - network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig.Mode = ipsecMode - } else { - // No changes to existing mode, return without updating networks. - return nil - } - _, err = oc.AdminOperatorClient().OperatorV1().Networks().Update(context.Background(), network, metav1.UpdateOptions{}) - return err - }) -} - func getIPsecMode(oc *exutil.CLI) (v1.IPsecMode, error) { network, err := oc.AdminOperatorClient().OperatorV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{}) if err != nil { @@ -132,10 +118,10 @@ func getIPsecMode(oc *exutil.CLI) (v1.IPsecMode, error) { return mode, nil } -// ensureIPsecEnabled this function ensure IPsec is enabled by making sure ovn-ipsec-host daemonset +// ensureIPsecFullEnabled this function ensure IPsec is enabled by making sure ovn-ipsec-host daemonset // is completely ready on the cluster and cluster operators are coming back into ready state // once ipsec rollout is complete. -func ensureIPsecEnabled(oc *exutil.CLI) error { +func ensureIPsecFullEnabled(oc *exutil.CLI) error { return wait.PollUntilContextTimeout(context.Background(), ipsecRolloutWaitInterval, ipsecRolloutWaitDuration, true, func(ctx context.Context) (bool, error) { done, err := areMachineConfigPoolsReadyWithIPsec(oc) @@ -159,12 +145,22 @@ func ensureIPsecEnabled(oc *exutil.CLI) error { }) } -// ensureIPsecMachineConfigRolloutComplete this function ensures ipsec machine config extension is rolled out +// ensureIPsecExternalEnabled this function ensures ipsec machine config extension is rolled out // on all of master and worked nodes and cluster operators are coming back into ready state // once ipsec rollout is complete. -func ensureIPsecMachineConfigRolloutComplete(oc *exutil.CLI) error { +func ensureIPsecExternalEnabled(oc *exutil.CLI) error { return wait.PollUntilContextTimeout(context.Background(), ipsecRolloutWaitInterval, ipsecRolloutWaitDuration, true, func(ctx context.Context) (bool, error) { + // Make sure ovn-ipsec-host daemonset is not deployed. When IPsec mode + // is changed from Full to External mode, then it may take a while to + // delete daemonset. + ds, err := getDaemonSet(oc, ovnNamespace, ovnIPsecDsName) + if err != nil && !isConnResetErr(err) { + return false, err + } + if ds != nil { + return false, nil + } done, err := areMachineConfigPoolsReadyWithIPsec(oc) if err != nil && !isConnResetErr(err) { return false, err @@ -212,15 +208,16 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { f := oc.KubeFramework() waitForIPsecNSConfigApplied := func() { - o.Eventually(func() bool { + g.GinkgoHelper() + o.Eventually(func(g o.Gomega) bool { out, err := oc.AsAdmin().Run("get").Args("NodeNetworkConfigurationPolicy/"+leftNodeIPsecPolicyName, "-o", "yaml").Output() - o.Expect(err).NotTo(o.HaveOccurred()) + g.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("rendered left node network config policy:\n%s", out) if !strings.Contains(out, "1/1 nodes successfully configured") { return false } out, err = oc.AsAdmin().Run("get").Args("NodeNetworkConfigurationPolicy/"+rightNodeIPsecPolicyName, "-o", "yaml").Output() - o.Expect(err).NotTo(o.HaveOccurred()) + g.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("rendered right node network config policy:\n%s", out) return strings.Contains(out, "1/1 nodes successfully configured") }, 30*time.Second).Should(o.BeTrue()) @@ -340,6 +337,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } cleanupTestPods := func(config *testConfig) { + g.GinkgoHelper() err := e2epod.DeletePodWithWait(context.Background(), f.ClientSet, config.srcNodeConfig.pingPod) o.Expect(err).NotTo(o.HaveOccurred()) config.srcNodeConfig.pingPod = nil @@ -356,6 +354,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } checkForGeneveOnlyPodTraffic := func(config *testConfig) { + g.GinkgoHelper() err := setupTestPods(config, false) o.Expect(err).NotTo(o.HaveOccurred()) defer func() { @@ -371,9 +370,11 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { o.Expect(err).To(o.HaveOccurred()) err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp) o.Expect(err).To(o.HaveOccurred()) + err = nil } checkForESPOnlyPodTraffic := func(config *testConfig) { + g.GinkgoHelper() err := setupTestPods(config, false) o.Expect(err).NotTo(o.HaveOccurred()) defer func() { @@ -387,9 +388,11 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { o.Expect(err).NotTo(o.HaveOccurred()) err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, geneve) o.Expect(err).To(o.HaveOccurred()) + err = nil } checkPodTraffic := func(mode v1.IPsecMode) { + g.GinkgoHelper() if mode == v1.IPsecModeFull { checkForESPOnlyPodTraffic(config) } else { @@ -398,6 +401,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } checkNodeTraffic := func(mode v1.IPsecMode) { + g.GinkgoHelper() err := setupTestPods(config, true) o.Expect(err).NotTo(o.HaveOccurred()) defer func() { @@ -412,6 +416,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { o.Expect(err).NotTo(o.HaveOccurred()) err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp) o.Expect(err).To(o.HaveOccurred()) + err = nil return } else { err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp) @@ -424,10 +429,50 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { // the selected nodes. ipsecMode, err := getIPsecMode(oc) o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(ipsecMode).NotTo(o.Equal(v1.IPsecModeDisabled)) srcNode, dstNode := &testNodeConfig{}, &testNodeConfig{} config = &testConfig{ipsecMode: ipsecMode, srcNodeConfig: srcNode, dstNodeConfig: dstNode} + + // Deploy nmstate handler which is used for rolling out IPsec config + // via NodeNetworkConfigurationPolicy. + g.By("deploy nmstate handler") + err = deployNmstateHandler(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Update cluster machine configuration object with few more nodeDisruptionPolicy defined + // in test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml file so that worker + // nodes don't go for a reboot while rolling out `99-worker-north-south-ipsec-config` + // machine config which configures certificates for testing IPsec north south traffic. + g.By("deploy machine configuration policy") + err = oc.AsAdmin().Run("apply").Args("-f", nsNodeRebootNoneFixture).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + mg.WaitForBootImageControllerToComplete(oc) + + g.By("configure IPsec certs on the worker nodes") + // The certificates for configuring NS IPsec between two worker nodes are deployed through machine config + // `99-worker-north-south-ipsec-config` which is in the test/extended/testdata/ipsec/nsconfig-machine-config.yaml file. + // This is a butane generated file via a butane config file available with commit: + // https://github.com/openshift/origin/pull/28658/commits/7399006f3750c530cfef51fa1044e941ccb85087 + // The machine config mounts cert files into node's /etc/pki/certs directory and runs ipsec-addcert.sh script + // to import those certs into Libreswan nss db and will be used by Libreswan for IPsec north south connection + // configured via NodeNetworkConfigurationPolicy on the node. + // The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024. + // so proceed with test if system date is before April 10, 2034. Otherwise fail the test. + if !time.Now().Before(certExpirationDate) { + framework.Failf("certficates in the Machine Config are expired, Please consider recreating those certificates") + } + nsCertMachineConfig, err := createIPsecCertsMachineConfig(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(nsCertMachineConfig).NotTo(o.BeNil()) + o.Eventually(func(g o.Gomega) bool { + pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) + g.Expect(err).NotTo(o.HaveOccurred()) + return areMachineConfigPoolsReadyWithMachineConfig(pools, nsCertMachineConfigName) + }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) + // Ensure IPsec mode is still correctly configured. + waitForIPsecConfigToComplete(oc, config.ipsecMode) }) g.BeforeEach(func() { @@ -458,51 +503,56 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { }) g.AfterEach(func() { - g.By("removing IPsec certs from worker nodes") - err := deleteNSCertMachineConfig(oc) + ipsecMode, err := getIPsecMode(oc) o.Expect(err).NotTo(o.HaveOccurred()) - o.Eventually(func() bool { - pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) - o.Expect(err).NotTo(o.HaveOccurred()) - return areMachineConfigPoolsReadyWithoutMachineConfig(pools, nsCertMachineConfigName) - }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) - - g.By("remove right node ipsec configuration") - oc.AsAdmin().Run("delete").Args("-f", rightNodeIPsecConfigYaml).Execute() - + if g.CurrentSpecReport().Failed() { + if ipsecMode == v1.IPsecModeFull { + var ipsecPods []string + srcIPsecPod, err := findIPsecPodonNode(oc, config.srcNodeConfig.nodeName) + o.Expect(err).NotTo(o.HaveOccurred()) + ipsecPods = append(ipsecPods, srcIPsecPod) + dstIPsecPod, err := findIPsecPodonNode(oc, config.dstNodeConfig.nodeName) + o.Expect(err).NotTo(o.HaveOccurred()) + ipsecPods = append(ipsecPods, dstIPsecPod) + for _, ipsecPod := range ipsecPods { + dumpPodCommand(ovnNamespace, ipsecPod, "cat /etc/ipsec.conf") + dumpPodCommand(ovnNamespace, ipsecPod, "cat /etc/ipsec.d/openshift.conf") + dumpPodCommand(ovnNamespace, ipsecPod, "ipsec status") + dumpPodCommand(ovnNamespace, ipsecPod, "ipsec trafficstatus") + dumpPodCommand(ovnNamespace, ipsecPod, "ip xfrm state") + dumpPodCommand(ovnNamespace, ipsecPod, "ip xfrm policy") + } + } + exutil.DumpPodStatesInNamespace(nmstateNamespace, oc) + exutil.DumpPodLogsStartingWithInNamespace("nmstate-handler", nmstateNamespace, oc) + exutil.DumpPodLogsStartingWithInNamespace("nmstate-operator", nmstateNamespace, oc) + } g.By("remove left node ipsec configuration") - oc.AsAdmin().Run("delete").Args("-f", leftNodeIPsecConfigYaml).Execute() - - g.By("undeploy nmstate handler") - undeployNmstateHandler(oc) + oc.AsAdmin().Run("delete").Args("-f", leftNodeIPsecNNCPYaml).Execute() + o.Eventually(func(g o.Gomega) bool { + _, err := oc.AdminDynamicClient().Resource(gvrNodeNetworkConfigurationPolicy).Get(context.Background(), + leftNodeIPsecPolicyName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return true + } + g.Expect(err).NotTo(o.HaveOccurred()) + return false + }).Should(o.Equal(true)) - // Restore the cluster back into original state after running all the tests. - g.By("restoring ipsec config into original state") - err = configureIPsecMode(oc, config.ipsecMode) - o.Expect(err).NotTo(o.HaveOccurred()) - waitForIPsecConfigToComplete(oc, config.ipsecMode) + g.By("remove right node ipsec configuration") + oc.AsAdmin().Run("delete").Args("-f", rightNodeIPsecNNCPYaml).Execute() + o.Eventually(func(g o.Gomega) bool { + _, err := oc.AdminDynamicClient().Resource(gvrNodeNetworkConfigurationPolicy).Get(context.Background(), + rightNodeIPsecPolicyName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return true + } + g.Expect(err).NotTo(o.HaveOccurred()) + return false + }).Should(o.Equal(true)) }) - g.DescribeTable("check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) { - o.Expect(config).NotTo(o.BeNil()) - g.By("validate traffic before changing IPsec configuration") - checkPodTraffic(config.ipsecMode) - // This test does not enable N/S ipsec config, so node traffic behaves as it were disabled - checkNodeTraffic(v1.IPsecModeDisabled) - g.By(fmt.Sprintf("configure IPsec in %s mode and validate traffic", mode)) - err := configureIPsecMode(oc, mode) - o.Expect(err).NotTo(o.HaveOccurred()) - waitForIPsecConfigToComplete(oc, mode) - checkPodTraffic(mode) - // This test does not enable N/S ipsec config, so node traffic behaves as it were disabled - checkNodeTraffic(v1.IPsecModeDisabled) - }, - g.Entry("with IPsec in full mode", v1.IPsecModeFull), - g.Entry("with IPsec in external mode", v1.IPsecModeExternal), - g.Entry("with IPsec in disabled mode", v1.IPsecModeDisabled), - ) - - g.DescribeTable("check traffic for north south IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) { + g.It("check traffic with IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func() { o.Expect(config).NotTo(o.BeNil()) g.By("validate traffic before changing IPsec configuration") @@ -510,52 +560,29 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { // N/S ipsec config is not in effect yet, so node traffic behaves as it were disabled checkNodeTraffic(v1.IPsecModeDisabled) - g.By(fmt.Sprintf("configure IPsec in %s mode and validate traffic", mode)) - // Change IPsec mode to External and packet capture on the node's interface - // must be geneve encapsulated ones. - err := configureIPsecMode(oc, mode) - o.Expect(err).NotTo(o.HaveOccurred()) - waitForIPsecConfigToComplete(oc, mode) - checkPodTraffic(mode) - // N/S ipsec config is not in effect yet, so node traffic behaves as it were disabled - checkNodeTraffic(v1.IPsecModeDisabled) - - g.By("configure IPsec certs on the worker nodes") - // The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024. - // so proceed with test if system date is before April 10, 2034. Otherwise fail the test. - if !time.Now().Before(certExpirationDate) { - framework.Failf("certficates in the Machine Config are expired, Please consider recreating those certificates") + // TODO: remove this block when https://issues.redhat.com/browse/RHEL-67307 is fixed. + if config.ipsecMode == v1.IPsecModeFull { + g.By(fmt.Sprintf("skip testing IPsec NS configuration with %s mode due to nmstate bug RHEL-67307", config.ipsecMode)) + return } - nsCertMachineConfig, err := createIPsecCertsMachineConfig(oc) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(nsCertMachineConfig).NotTo(o.BeNil()) - o.Eventually(func() bool { - pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) - o.Expect(err).NotTo(o.HaveOccurred()) - return areMachineConfigPoolsReadyWithMachineConfig(pools, nsCertMachineConfigName) - }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) - - // Deploy nmstate handler which is used for rolling out IPsec config - // via NodeNetworkConfigurationPolicy. - g.By("deploy nmstate handler") - err = deployNmstateHandler(oc) - o.Expect(err).NotTo(o.HaveOccurred()) g.By("rollout IPsec configuration via nmstate") + err := ensureNmstateHandlerRunning(oc) + o.Expect(err).NotTo(o.HaveOccurred()) leftConfig := fmt.Sprintf(nodeIPsecConfigManifest, leftNodeIPsecPolicyName, config.srcNodeConfig.nodeName, config.srcNodeConfig.nodeIP, leftServerCertName, config.dstNodeConfig.nodeIP) - err = os.WriteFile(leftNodeIPsecConfigYaml, []byte(leftConfig), 0644) + err = os.WriteFile(leftNodeIPsecNNCPYaml, []byte(leftConfig), 0644) o.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("desired left node network config policy:\n%s", leftConfig) - err = oc.AsAdmin().Run("apply").Args("-f", leftNodeIPsecConfigYaml).Execute() + err = oc.AsAdmin().Run("apply").Args("-f", leftNodeIPsecNNCPYaml).Execute() o.Expect(err).NotTo(o.HaveOccurred()) rightConfig := fmt.Sprintf(nodeIPsecConfigManifest, rightNodeIPsecPolicyName, config.dstNodeConfig.nodeName, config.dstNodeConfig.nodeIP, rightServerCertName, config.srcNodeConfig.nodeIP) - err = os.WriteFile(rightNodeIPsecConfigYaml, []byte(rightConfig), 0644) + err = os.WriteFile(rightNodeIPsecNNCPYaml, []byte(rightConfig), 0644) o.Expect(err).NotTo(o.HaveOccurred()) framework.Logf("desired right node network config policy:\n%s", rightConfig) - err = oc.AsAdmin().Run("apply").Args("-f", rightNodeIPsecConfigYaml).Execute() + err = oc.AsAdmin().Run("apply").Args("-f", rightNodeIPsecNNCPYaml).Execute() o.Expect(err).NotTo(o.HaveOccurred()) g.By("wait for nmstate to roll out") @@ -564,24 +591,56 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { g.By("validate IPsec traffic between nodes") // Pod traffic will be encrypted as a result N/S encryption being enabled between this two nodes checkPodTraffic(v1.IPsecModeFull) - checkNodeTraffic(mode) - }, - g.Entry("with IPsec in full mode", v1.IPsecModeFull), - g.Entry("with IPsec in external mode", v1.IPsecModeExternal), - ) + checkNodeTraffic(v1.IPsecModeExternal) + }) }) }) func waitForIPsecConfigToComplete(oc *exutil.CLI, ipsecMode v1.IPsecMode) { + g.GinkgoHelper() switch ipsecMode { case v1.IPsecModeDisabled: err := ensureIPsecDisabled(oc) o.Expect(err).NotTo(o.HaveOccurred()) case v1.IPsecModeExternal: - err := ensureIPsecMachineConfigRolloutComplete(oc) + err := ensureIPsecExternalEnabled(oc) o.Expect(err).NotTo(o.HaveOccurred()) case v1.IPsecModeFull: - err := ensureIPsecEnabled(oc) + err := ensureIPsecFullEnabled(oc) o.Expect(err).NotTo(o.HaveOccurred()) } } + +func findIPsecPodonNode(oc *exutil.CLI, nodeName string) (string, error) { + out, err := runOcWithRetry(oc.AsAdmin(), "get", + "pods", + "-o", "name", + "-n", ovnNamespace, + "--field-selector", fmt.Sprintf("spec.nodeName=%s", nodeName), + "-l", "app=ovn-ipsec") + if err != nil { + return "", err + } + outReader := bufio.NewScanner(strings.NewReader(out)) + re := regexp.MustCompile("^pod/(.*)") + var podName string + for outReader.Scan() { + match := re.FindSubmatch([]byte(outReader.Text())) + if len(match) != 2 { + continue + } + podName = string(match[1]) + break + } + if podName == "" { + return "", fmt.Errorf("could not find a valid ovn-ipsec-host pod on node '%s'", nodeName) + } + return podName, nil +} + +func dumpPodCommand(namespace, name, cmd string) { + g.GinkgoHelper() + stdout, err := e2eoutput.RunHostCmdWithRetries(namespace, name, cmd, statefulset.StatefulSetPoll, statefulset.StatefulPodTimeout) + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf(name + ": " + strings.Join(strings.Split(stdout, "\n"), fmt.Sprintf("\n%s: ", name))) +} diff --git a/test/extended/networking/util.go b/test/extended/networking/util.go index 8b5efb700eed..f7f6367b555e 100644 --- a/test/extended/networking/util.go +++ b/test/extended/networking/util.go @@ -6,6 +6,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -82,6 +83,9 @@ const ( var ( masterRoleMachineConfigLabel = map[string]string{"machineconfiguration.openshift.io/role": "master"} workerRoleMachineConfigLabel = map[string]string{"machineconfiguration.openshift.io/role": "worker"} + ipsecConfigurationBaseDir = exutil.FixturePath("testdata", "ipsec") + nsMachineConfigFixture = filepath.Join(ipsecConfigurationBaseDir, "nsconfig-machine-config.yaml") + nsNodeRebootNoneFixture = filepath.Join(ipsecConfigurationBaseDir, "nsconfig-reboot-none-policy.yaml") ) // IsIPv6 returns true if a group of ips are ipv6. @@ -713,11 +717,15 @@ func deployNmstateHandler(oc *exutil.CLI) error { return fmt.Errorf("nmstate operator is not running: %v", err) } nmStateConfigYaml := exutil.FixturePath("testdata", "ipsec", nmstateConfigureManifestFile) - err = oc.AsAdmin().Run("create").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute() + err = oc.AsAdmin().Run("apply").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute() if err != nil { return fmt.Errorf("error configuring nmstate: %v", err) } - err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, + return ensureNmstateHandlerRunning(oc) +} + +func ensureNmstateHandlerRunning(oc *exutil.CLI) error { + err := wait.PollUntilContextTimeout(context.Background(), poll, 5*time.Minute, true, func(ctx context.Context) (bool, error) { // Ensure nmstate handler is running. return isDaemonSetRunning(oc, nmstateNamespace, "nmstate-handler") @@ -732,26 +740,6 @@ func deployNmstateHandler(oc *exutil.CLI) error { return nil } -func undeployNmstateHandler(oc *exutil.CLI) error { - nmStateConfigYaml := exutil.FixturePath("testdata", "ipsec", nmstateConfigureManifestFile) - err := oc.AsAdmin().Run("delete").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute() - if err != nil { - return fmt.Errorf("error deleting nmstate configuration: %v", err) - } - err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, - func(ctx context.Context) (bool, error) { - _, err := oc.AdminKubeClient().AppsV1().DaemonSets(nmstateNamespace).Get(context.Background(), "nmstate-handler", metav1.GetOptions{}) - if err != nil && apierrors.IsNotFound(err) { - return true, nil - } - return false, err - }) - if err != nil { - return fmt.Errorf("failed to remove nmstate crd and handler: %v", err) - } - return nil -} - func waitForDeploymentComplete(oc *exutil.CLI, namespace, name string) error { deployment, err := oc.AdminKubeClient().AppsV1().Deployments(namespace).Get(context.Background(), name, metav1.GetOptions{}) if err != nil { @@ -782,12 +770,18 @@ func getDaemonSet(oc *exutil.CLI, namespace, name string) (*appsv1.DaemonSet, er } func createIPsecCertsMachineConfig(oc *exutil.CLI) (*mcfgv1.MachineConfig, error) { - ipSecCertsMachineConfig := exutil.FixturePath("testdata", "ipsec", nsCertMachineConfigFile) - err := oc.AsAdmin().Run("create").Args("-f", ipSecCertsMachineConfig).Execute() + nsCertMachineConfig, err := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigs().Get(context.Background(), + nsCertMachineConfigName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + if err == nil { + return nsCertMachineConfig, nil + } + err = oc.AsAdmin().Run("create").Args("-f", nsMachineConfigFixture).Execute() if err != nil { return nil, fmt.Errorf("error deploying IPsec certs Machine Config: %v", err) } - var nsCertMachineConfig *mcfgv1.MachineConfig err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, func(ctx context.Context) (bool, error) { nsCertMachineConfig, err = oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigs().Get(context.Background(), nsCertMachineConfigName, metav1.GetOptions{}) diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index 62295f10ba39..c59c70d6848a 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -345,8 +345,9 @@ // test/extended/testdata/image_ecosystem/perl-hotdeploy/index.pl // test/extended/testdata/image_ecosystem/perl-hotdeploy/lib/My/Test.pm // test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json -// test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml // test/extended/testdata/ipsec/nmstate.yaml +// test/extended/testdata/ipsec/nsconfig-machine-config.yaml +// test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml // test/extended/testdata/kernel/rt-tests-environment.yaml // test/extended/testdata/kernel/rt-tests-pod.yaml // test/extended/testdata/ldap/groupsync/ad/blacklist_ldap.txt @@ -45721,7 +45722,28 @@ func testExtendedTestdataImage_ecosystemPerlHotdeployPerlJson() (*asset, error) return a, nil } -var _testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml = []byte(`# Generated by Butane; do not edit +var _testExtendedTestdataIpsecNmstateYaml = []byte(`apiVersion: nmstate.io/v1 +kind: NMState +metadata: + name: nmstate +`) + +func testExtendedTestdataIpsecNmstateYamlBytes() ([]byte, error) { + return _testExtendedTestdataIpsecNmstateYaml, nil +} + +func testExtendedTestdataIpsecNmstateYaml() (*asset, error) { + bytes, err := testExtendedTestdataIpsecNmstateYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/ipsec/nmstate.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataIpsecNsconfigMachineConfigYaml = []byte(`# Generated by Butane; do not edit apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: @@ -45777,38 +45799,60 @@ spec: name: ipsec-import.service `) -func testExtendedTestdataIpsecIpsecNsconfigMachineConfigYamlBytes() ([]byte, error) { - return _testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml, nil +func testExtendedTestdataIpsecNsconfigMachineConfigYamlBytes() ([]byte, error) { + return _testExtendedTestdataIpsecNsconfigMachineConfigYaml, nil } -func testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml() (*asset, error) { - bytes, err := testExtendedTestdataIpsecIpsecNsconfigMachineConfigYamlBytes() +func testExtendedTestdataIpsecNsconfigMachineConfigYaml() (*asset, error) { + bytes, err := testExtendedTestdataIpsecNsconfigMachineConfigYamlBytes() if err != nil { return nil, err } - info := bindataFileInfo{name: "test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + info := bindataFileInfo{name: "test/extended/testdata/ipsec/nsconfig-machine-config.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _testExtendedTestdataIpsecNmstateYaml = []byte(`apiVersion: nmstate.io/v1 -kind: NMState +var _testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml = []byte(`apiVersion: operator.openshift.io/v1 +kind: MachineConfiguration metadata: - name: nmstate + name: cluster + namespace: openshift-machine-config-operator +spec: + nodeDisruptionPolicy: + files: + - actions: + - type: None + path: /usr/local/bin/ipsec-addcert.sh + - actions: + - type: None + path: /etc/pki/certs/ca.pem + - actions: + - type: None + path: /etc/pki/certs/left_server.p12 + - actions: + - type: None + path: /etc/pki/certs/right_server.p12 + units: + - name: ipsec-import.service + actions: + - restart: + serviceName: ipsec-import.service + type: Restart `) -func testExtendedTestdataIpsecNmstateYamlBytes() ([]byte, error) { - return _testExtendedTestdataIpsecNmstateYaml, nil +func testExtendedTestdataIpsecNsconfigRebootNonePolicyYamlBytes() ([]byte, error) { + return _testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml, nil } -func testExtendedTestdataIpsecNmstateYaml() (*asset, error) { - bytes, err := testExtendedTestdataIpsecNmstateYamlBytes() +func testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml() (*asset, error) { + bytes, err := testExtendedTestdataIpsecNsconfigRebootNonePolicyYamlBytes() if err != nil { return nil, err } - info := bindataFileInfo{name: "test/extended/testdata/ipsec/nmstate.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + info := bindataFileInfo{name: "test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -55617,8 +55661,9 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/image_ecosystem/perl-hotdeploy/index.pl": testExtendedTestdataImage_ecosystemPerlHotdeployIndexPl, "test/extended/testdata/image_ecosystem/perl-hotdeploy/lib/My/Test.pm": testExtendedTestdataImage_ecosystemPerlHotdeployLibMyTestPm, "test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json": testExtendedTestdataImage_ecosystemPerlHotdeployPerlJson, - "test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml": testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml, "test/extended/testdata/ipsec/nmstate.yaml": testExtendedTestdataIpsecNmstateYaml, + "test/extended/testdata/ipsec/nsconfig-machine-config.yaml": testExtendedTestdataIpsecNsconfigMachineConfigYaml, + "test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml": testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml, "test/extended/testdata/kernel/rt-tests-environment.yaml": testExtendedTestdataKernelRtTestsEnvironmentYaml, "test/extended/testdata/kernel/rt-tests-pod.yaml": testExtendedTestdataKernelRtTestsPodYaml, "test/extended/testdata/ldap/groupsync/ad/blacklist_ldap.txt": testExtendedTestdataLdapGroupsyncAdBlacklist_ldapTxt, @@ -56345,8 +56390,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ }}, }}, "ipsec": {nil, map[string]*bintree{ - "ipsec-nsconfig-machine-config.yaml": {testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml, map[string]*bintree{}}, - "nmstate.yaml": {testExtendedTestdataIpsecNmstateYaml, map[string]*bintree{}}, + "nmstate.yaml": {testExtendedTestdataIpsecNmstateYaml, map[string]*bintree{}}, + "nsconfig-machine-config.yaml": {testExtendedTestdataIpsecNsconfigMachineConfigYaml, map[string]*bintree{}}, + "nsconfig-reboot-none-policy.yaml": {testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml, map[string]*bintree{}}, }}, "kernel": {nil, map[string]*bintree{ "rt-tests-environment.yaml": {testExtendedTestdataKernelRtTestsEnvironmentYaml, map[string]*bintree{}}, diff --git a/test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml b/test/extended/testdata/ipsec/nsconfig-machine-config.yaml similarity index 100% rename from test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml rename to test/extended/testdata/ipsec/nsconfig-machine-config.yaml diff --git a/test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml b/test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml new file mode 100644 index 000000000000..011b74c03490 --- /dev/null +++ b/test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml @@ -0,0 +1,26 @@ +apiVersion: operator.openshift.io/v1 +kind: MachineConfiguration +metadata: + name: cluster + namespace: openshift-machine-config-operator +spec: + nodeDisruptionPolicy: + files: + - actions: + - type: None + path: /usr/local/bin/ipsec-addcert.sh + - actions: + - type: None + path: /etc/pki/certs/ca.pem + - actions: + - type: None + path: /etc/pki/certs/left_server.p12 + - actions: + - type: None + path: /etc/pki/certs/right_server.p12 + units: + - name: ipsec-import.service + actions: + - restart: + serviceName: ipsec-import.service + type: Restart diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index dd7cdea17607..6c3054274679 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -1477,15 +1477,7 @@ var Annotations = map[string]string{ "[sig-network][Feature:EgressRouterCNI] when using openshift ovn-kubernetes should ensure ipv6 egressrouter cni resources are created [apigroup:operator.openshift.io]": " [Suite:openshift/conformance/parallel]", - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in disabled mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in external mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in full mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for north south IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in external mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic for north south IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in full mode": "", + "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic with IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]": "", "[sig-network][Feature:Layer2LiveMigration][Suite:openshift/network/virtualization] Kubevirt Virtual Machines Placeholder test for GA": "", diff --git a/test/extended/util/clusterversion.go b/test/extended/util/clusterversion.go index c66209c1ba8b..d2061bc2cc83 100644 --- a/test/extended/util/clusterversion.go +++ b/test/extended/util/clusterversion.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" configv1 "github.com/openshift/api/config/v1" @@ -68,9 +69,10 @@ func DetermineImageFromRelease(ctx context.Context, oc *CLI, imageTagName string if len(releaseImage) == 0 { return "", fmt.Errorf("cannot determine release image from ClusterVersion resource") } + podName := "extract-release-imagerefs" podClient := e2epod.PodClientNS(oc.KubeFramework(), oc.Namespace()) podClient.CreateSync(ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "extract-release-imagerefs"}, + ObjectMeta: metav1.ObjectMeta{Name: podName}, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { @@ -81,8 +83,15 @@ func DetermineImageFromRelease(ctx context.Context, oc *CLI, imageTagName string }, }, }) - defer podClient.Delete(ctx, "extract-release-imagerefs", metav1.DeleteOptions{}) - imageRefsString := e2epod.ExecShellInContainer(oc.KubeFramework(), "extract-release-imagerefs", "imagerefs", "cat /release-manifests/image-references") + defer func() { + podClient.Delete(ctx, podName, metav1.DeleteOptions{}) + err = e2epod.WaitForPodNotFoundInNamespace(ctx, oc.kubeFramework.ClientSet, + podName, oc.Namespace(), e2epod.PodDeleteTimeout) + if err != nil { + framework.Logf("pod %q is still found in namespace %q", podName, oc.Namespace()) + } + }() + imageRefsString := e2epod.ExecShellInContainer(oc.KubeFramework(), podName, "imagerefs", "cat /release-manifests/image-references") imageRefs := struct { Spec struct { Tags []struct { diff --git a/test/extended/util/managed_services.go b/test/extended/util/managed_services.go index a891545c7c6a..ee49124f9753 100644 --- a/test/extended/util/managed_services.go +++ b/test/extended/util/managed_services.go @@ -35,6 +35,7 @@ var ManagedServiceNamespaces = sets.New[string]( "openshift-managed-upgrade-operator", "openshift-marketplace", "openshift-must-gather-operator", + "openshift-nmstate", "openshift-observability-operator", "openshift-ocm-agent-operator", "openshift-operators-redhat",