diff --git a/test/extended/networking/ipsec.go b/test/extended/networking/ipsec.go index 3dbf8a781129..4a433274d438 100644 --- a/test/extended/networking/ipsec.go +++ b/test/extended/networking/ipsec.go @@ -10,6 +10,7 @@ import ( "time" v1 "github.com/openshift/api/operator/v1" + mg "github.com/openshift/origin/test/extended/machine_config" exutil "github.com/openshift/origin/test/extended/util" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" @@ -17,7 +18,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" @@ -40,7 +40,6 @@ const ( ipsecRolloutWaitDuration = 40 * time.Minute ipsecRolloutWaitInterval = 1 * time.Minute nmstateConfigureManifestFile = "nmstate.yaml" - nsCertMachineConfigFile = "ipsec-nsconfig-machine-config.yaml" nsCertMachineConfigName = "99-worker-north-south-ipsec-config" leftNodeIPsecPolicyName = "left-node-ipsec-policy" rightNodeIPsecPolicyName = "right-node-ipsec-policy" @@ -102,27 +101,6 @@ const ( icmp trafficType = "icmp" ) -// configureIPsecMode helps to rollout specified IPsec Mode on the cluster. If the cluster is already -// configured with specified mode, then this is almost like no-op for the cluster. -func configureIPsecMode(oc *exutil.CLI, ipsecMode v1.IPsecMode) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - network, err := oc.AdminOperatorClient().OperatorV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{}) - if err != nil { - return err - } - if network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig == nil { - network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig = &v1.IPsecConfig{Mode: ipsecMode} - } else if network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig.Mode != ipsecMode { - network.Spec.DefaultNetwork.OVNKubernetesConfig.IPsecConfig.Mode = ipsecMode - } else { - // No changes to existing mode, return without updating networks. - return nil - } - _, err = oc.AdminOperatorClient().OperatorV1().Networks().Update(context.Background(), network, metav1.UpdateOptions{}) - return err - }) -} - func getIPsecMode(oc *exutil.CLI) (v1.IPsecMode, error) { network, err := oc.AdminOperatorClient().OperatorV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{}) if err != nil { @@ -451,7 +429,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { // the selected nodes. ipsecMode, err := getIPsecMode(oc) o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(ipsecMode).To(o.Equal(v1.IPsecModeFull)) + o.Expect(ipsecMode).NotTo(o.Equal(v1.IPsecModeDisabled)) srcNode, dstNode := &testNodeConfig{}, &testNodeConfig{} config = &testConfig{ipsecMode: ipsecMode, srcNodeConfig: srcNode, @@ -462,6 +440,39 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { g.By("deploy nmstate handler") err = deployNmstateHandler(oc) o.Expect(err).NotTo(o.HaveOccurred()) + + // Update cluster machine configuration object with few more nodeDisruptionPolicy defined + // in test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml file so that worker + // nodes don't go for a reboot while rolling out `99-worker-north-south-ipsec-config` + // machine config which configures certificates for testing IPsec north south traffic. + g.By("deploy machine configuration policy") + err = oc.AsAdmin().Run("apply").Args("-f", nsNodeRebootNoneFixture).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + mg.WaitForBootImageControllerToComplete(oc) + + g.By("configure IPsec certs on the worker nodes") + // The certificates for configuring NS IPsec between two worker nodes are deployed through machine config + // `99-worker-north-south-ipsec-config` which is in the test/extended/testdata/ipsec/nsconfig-machine-config.yaml file. + // This is a butane generated file via a butane config file available with commit: + // https://github.com/openshift/origin/pull/28658/commits/7399006f3750c530cfef51fa1044e941ccb85087 + // The machine config mounts cert files into node's /etc/pki/certs directory and runs ipsec-addcert.sh script + // to import those certs into Libreswan nss db and will be used by Libreswan for IPsec north south connection + // configured via NodeNetworkConfigurationPolicy on the node. + // The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024. + // so proceed with test if system date is before April 10, 2034. Otherwise fail the test. + if !time.Now().Before(certExpirationDate) { + framework.Failf("certficates in the Machine Config are expired, Please consider recreating those certificates") + } + nsCertMachineConfig, err := createIPsecCertsMachineConfig(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(nsCertMachineConfig).NotTo(o.BeNil()) + o.Eventually(func(g o.Gomega) bool { + pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) + g.Expect(err).NotTo(o.HaveOccurred()) + return areMachineConfigPoolsReadyWithMachineConfig(pools, nsCertMachineConfigName) + }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) + // Ensure IPsec mode is still correctly configured. + waitForIPsecConfigToComplete(oc, config.ipsecMode) }) g.BeforeEach(func() { @@ -489,23 +500,6 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { } } o.Expect(config.dstNodeConfig.nodeIP).NotTo(o.BeEmpty()) - - g.By("configure IPsec certs on the worker nodes") - // The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024. - // so proceed with test if system date is before April 10, 2034. Otherwise fail the test. - if !time.Now().Before(certExpirationDate) { - framework.Failf("certficates in the Machine Config are expired, Please consider recreating those certificates") - } - nsCertMachineConfig, err := createIPsecCertsMachineConfig(oc) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(nsCertMachineConfig).NotTo(o.BeNil()) - o.Eventually(func(g o.Gomega) bool { - pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) - g.Expect(err).NotTo(o.HaveOccurred()) - return areMachineConfigPoolsReadyWithMachineConfig(pools, nsCertMachineConfigName) - }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) - // wait for ovn-ipsec-host pod to get rolled out after certs installation. - waitForIPsecConfigToComplete(oc, config.ipsecMode) }) g.AfterEach(func() { @@ -556,26 +550,9 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { g.Expect(err).NotTo(o.HaveOccurred()) return false }).Should(o.Equal(true)) - - // Removal of IPsec certs are needed otherwise worker nodes still keeping - // stale ip xfrm state and policy entries created for north south traffic. - g.By("removing IPsec certs from worker nodes") - err = deleteNSCertMachineConfig(oc) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Eventually(func(g o.Gomega) bool { - pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel) - g.Expect(err).NotTo(o.HaveOccurred()) - return areMachineConfigPoolsReadyWithoutMachineConfig(pools, nsCertMachineConfigName) - }, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue()) - - // Restore the cluster back into original state after running each test. - g.By("restoring ipsec config into original state") - err = configureIPsecMode(oc, config.ipsecMode) - o.Expect(err).NotTo(o.HaveOccurred()) - waitForIPsecConfigToComplete(oc, config.ipsecMode) }) - g.DescribeTable("check traffic [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) { + g.It("check traffic with IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func() { o.Expect(config).NotTo(o.BeNil()) g.By("validate traffic before changing IPsec configuration") @@ -583,23 +560,14 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { // N/S ipsec config is not in effect yet, so node traffic behaves as it were disabled checkNodeTraffic(v1.IPsecModeDisabled) - g.By(fmt.Sprintf("configure IPsec in %s mode and validate traffic", mode)) - // Change IPsec mode to given mode and do packet capture on the node's interface - err := configureIPsecMode(oc, mode) - o.Expect(err).NotTo(o.HaveOccurred()) - waitForIPsecConfigToComplete(oc, mode) - checkPodTraffic(mode) - // N/S ipsec config is not in effect yet, so node traffic behaves as it were disabled - checkNodeTraffic(v1.IPsecModeDisabled) - // TODO: remove this block when https://issues.redhat.com/browse/RHEL-67307 is fixed. - if mode == v1.IPsecModeFull { - g.By(fmt.Sprintf("skip testing IPsec NS configuration with %s mode due to nmstate bug RHEL-67307", mode)) + if config.ipsecMode == v1.IPsecModeFull { + g.By(fmt.Sprintf("skip testing IPsec NS configuration with %s mode due to nmstate bug RHEL-67307", config.ipsecMode)) return } g.By("rollout IPsec configuration via nmstate") - err = ensureNmstateHandlerRunning(oc) + err := ensureNmstateHandlerRunning(oc) o.Expect(err).NotTo(o.HaveOccurred()) leftConfig := fmt.Sprintf(nodeIPsecConfigManifest, leftNodeIPsecPolicyName, config.srcNodeConfig.nodeName, config.srcNodeConfig.nodeIP, leftServerCertName, config.dstNodeConfig.nodeIP) @@ -623,12 +591,8 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() { g.By("validate IPsec traffic between nodes") // Pod traffic will be encrypted as a result N/S encryption being enabled between this two nodes checkPodTraffic(v1.IPsecModeFull) - checkNodeTraffic(mode) - }, - g.Entry("with IPsec in full mode", v1.IPsecModeFull), - g.Entry("with IPsec in external mode", v1.IPsecModeExternal), - // TODO add test for v1.IPsecModeDisabled mode once IPsec tests stabilized in CI. - ) + checkNodeTraffic(v1.IPsecModeExternal) + }) }) }) diff --git a/test/extended/networking/util.go b/test/extended/networking/util.go index a109ebc06a01..f7f6367b555e 100644 --- a/test/extended/networking/util.go +++ b/test/extended/networking/util.go @@ -6,6 +6,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -82,6 +83,9 @@ const ( var ( masterRoleMachineConfigLabel = map[string]string{"machineconfiguration.openshift.io/role": "master"} workerRoleMachineConfigLabel = map[string]string{"machineconfiguration.openshift.io/role": "worker"} + ipsecConfigurationBaseDir = exutil.FixturePath("testdata", "ipsec") + nsMachineConfigFixture = filepath.Join(ipsecConfigurationBaseDir, "nsconfig-machine-config.yaml") + nsNodeRebootNoneFixture = filepath.Join(ipsecConfigurationBaseDir, "nsconfig-reboot-none-policy.yaml") ) // IsIPv6 returns true if a group of ips are ipv6. @@ -774,8 +778,7 @@ func createIPsecCertsMachineConfig(oc *exutil.CLI) (*mcfgv1.MachineConfig, error if err == nil { return nsCertMachineConfig, nil } - ipSecCertsMachineConfig := exutil.FixturePath("testdata", "ipsec", nsCertMachineConfigFile) - err = oc.AsAdmin().Run("create").Args("-f", ipSecCertsMachineConfig).Execute() + err = oc.AsAdmin().Run("create").Args("-f", nsMachineConfigFixture).Execute() if err != nil { return nil, fmt.Errorf("error deploying IPsec certs Machine Config: %v", err) } diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index 98af26b1df70..f0b5405490c1 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -345,8 +345,9 @@ // test/extended/testdata/image_ecosystem/perl-hotdeploy/index.pl // test/extended/testdata/image_ecosystem/perl-hotdeploy/lib/My/Test.pm // test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json -// test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml // test/extended/testdata/ipsec/nmstate.yaml +// test/extended/testdata/ipsec/nsconfig-machine-config.yaml +// test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml // test/extended/testdata/kernel/rt-tests-environment.yaml // test/extended/testdata/kernel/rt-tests-pod.yaml // test/extended/testdata/ldap/groupsync/ad/blacklist_ldap.txt @@ -45721,7 +45722,28 @@ func testExtendedTestdataImage_ecosystemPerlHotdeployPerlJson() (*asset, error) return a, nil } -var _testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml = []byte(`# Generated by Butane; do not edit +var _testExtendedTestdataIpsecNmstateYaml = []byte(`apiVersion: nmstate.io/v1 +kind: NMState +metadata: + name: nmstate +`) + +func testExtendedTestdataIpsecNmstateYamlBytes() ([]byte, error) { + return _testExtendedTestdataIpsecNmstateYaml, nil +} + +func testExtendedTestdataIpsecNmstateYaml() (*asset, error) { + bytes, err := testExtendedTestdataIpsecNmstateYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/ipsec/nmstate.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataIpsecNsconfigMachineConfigYaml = []byte(`# Generated by Butane; do not edit apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: @@ -45777,38 +45799,60 @@ spec: name: ipsec-import.service `) -func testExtendedTestdataIpsecIpsecNsconfigMachineConfigYamlBytes() ([]byte, error) { - return _testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml, nil +func testExtendedTestdataIpsecNsconfigMachineConfigYamlBytes() ([]byte, error) { + return _testExtendedTestdataIpsecNsconfigMachineConfigYaml, nil } -func testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml() (*asset, error) { - bytes, err := testExtendedTestdataIpsecIpsecNsconfigMachineConfigYamlBytes() +func testExtendedTestdataIpsecNsconfigMachineConfigYaml() (*asset, error) { + bytes, err := testExtendedTestdataIpsecNsconfigMachineConfigYamlBytes() if err != nil { return nil, err } - info := bindataFileInfo{name: "test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + info := bindataFileInfo{name: "test/extended/testdata/ipsec/nsconfig-machine-config.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _testExtendedTestdataIpsecNmstateYaml = []byte(`apiVersion: nmstate.io/v1 -kind: NMState +var _testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml = []byte(`apiVersion: operator.openshift.io/v1 +kind: MachineConfiguration metadata: - name: nmstate + name: cluster + namespace: openshift-machine-config-operator +spec: + nodeDisruptionPolicy: + files: + - actions: + - type: None + path: /usr/local/bin/ipsec-addcert.sh + - actions: + - type: None + path: /etc/pki/certs/ca.pem + - actions: + - type: None + path: /etc/pki/certs/left_server.p12 + - actions: + - type: None + path: /etc/pki/certs/right_server.p12 + units: + - name: ipsec-import.service + actions: + - restart: + serviceName: ipsec-import.service + type: Restart `) -func testExtendedTestdataIpsecNmstateYamlBytes() ([]byte, error) { - return _testExtendedTestdataIpsecNmstateYaml, nil +func testExtendedTestdataIpsecNsconfigRebootNonePolicyYamlBytes() ([]byte, error) { + return _testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml, nil } -func testExtendedTestdataIpsecNmstateYaml() (*asset, error) { - bytes, err := testExtendedTestdataIpsecNmstateYamlBytes() +func testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml() (*asset, error) { + bytes, err := testExtendedTestdataIpsecNsconfigRebootNonePolicyYamlBytes() if err != nil { return nil, err } - info := bindataFileInfo{name: "test/extended/testdata/ipsec/nmstate.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + info := bindataFileInfo{name: "test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -55624,8 +55668,9 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/image_ecosystem/perl-hotdeploy/index.pl": testExtendedTestdataImage_ecosystemPerlHotdeployIndexPl, "test/extended/testdata/image_ecosystem/perl-hotdeploy/lib/My/Test.pm": testExtendedTestdataImage_ecosystemPerlHotdeployLibMyTestPm, "test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json": testExtendedTestdataImage_ecosystemPerlHotdeployPerlJson, - "test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml": testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml, "test/extended/testdata/ipsec/nmstate.yaml": testExtendedTestdataIpsecNmstateYaml, + "test/extended/testdata/ipsec/nsconfig-machine-config.yaml": testExtendedTestdataIpsecNsconfigMachineConfigYaml, + "test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml": testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml, "test/extended/testdata/kernel/rt-tests-environment.yaml": testExtendedTestdataKernelRtTestsEnvironmentYaml, "test/extended/testdata/kernel/rt-tests-pod.yaml": testExtendedTestdataKernelRtTestsPodYaml, "test/extended/testdata/ldap/groupsync/ad/blacklist_ldap.txt": testExtendedTestdataLdapGroupsyncAdBlacklist_ldapTxt, @@ -56352,8 +56397,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ }}, }}, "ipsec": {nil, map[string]*bintree{ - "ipsec-nsconfig-machine-config.yaml": {testExtendedTestdataIpsecIpsecNsconfigMachineConfigYaml, map[string]*bintree{}}, - "nmstate.yaml": {testExtendedTestdataIpsecNmstateYaml, map[string]*bintree{}}, + "nmstate.yaml": {testExtendedTestdataIpsecNmstateYaml, map[string]*bintree{}}, + "nsconfig-machine-config.yaml": {testExtendedTestdataIpsecNsconfigMachineConfigYaml, map[string]*bintree{}}, + "nsconfig-reboot-none-policy.yaml": {testExtendedTestdataIpsecNsconfigRebootNonePolicyYaml, map[string]*bintree{}}, }}, "kernel": {nil, map[string]*bintree{ "rt-tests-environment.yaml": {testExtendedTestdataKernelRtTestsEnvironmentYaml, map[string]*bintree{}}, diff --git a/test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml b/test/extended/testdata/ipsec/nsconfig-machine-config.yaml similarity index 100% rename from test/extended/testdata/ipsec/ipsec-nsconfig-machine-config.yaml rename to test/extended/testdata/ipsec/nsconfig-machine-config.yaml diff --git a/test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml b/test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml new file mode 100644 index 000000000000..011b74c03490 --- /dev/null +++ b/test/extended/testdata/ipsec/nsconfig-reboot-none-policy.yaml @@ -0,0 +1,26 @@ +apiVersion: operator.openshift.io/v1 +kind: MachineConfiguration +metadata: + name: cluster + namespace: openshift-machine-config-operator +spec: + nodeDisruptionPolicy: + files: + - actions: + - type: None + path: /usr/local/bin/ipsec-addcert.sh + - actions: + - type: None + path: /etc/pki/certs/ca.pem + - actions: + - type: None + path: /etc/pki/certs/left_server.p12 + - actions: + - type: None + path: /etc/pki/certs/right_server.p12 + units: + - name: ipsec-import.service + actions: + - restart: + serviceName: ipsec-import.service + type: Restart diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index a60e565472e7..ac7c256cbf76 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -1495,9 +1495,7 @@ var Annotations = map[string]string{ "[sig-network][Feature:EgressRouterCNI] when using openshift ovn-kubernetes should ensure ipv6 egressrouter cni resources are created [apigroup:operator.openshift.io]": " [Suite:openshift/conformance/parallel]", - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in external mode": "", - - "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic [apigroup:config.openshift.io] [Suite:openshift/network/ipsec] with IPsec in full mode": "", + "[sig-network][Feature:IPsec] when using openshift ovn-kubernetes check traffic with IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]": "", "[sig-network][Feature:Layer2LiveMigration][OCPFeatureGate:NetworkSegmentation][Suite:openshift/network/virtualization] primary UDN smoke test when using openshift ovn-kubernetes assert the primary UDN feature works as expected": "",