Move to use newer IPsec DaemonSets when MCP is in paused state

When MCP is in paused state, network operator continues to render older
IPsec daemonsets which blocks network cluster operator not getting upgraded
to newer version. Hence this commit renders newer IPsec daemonsets for
intermediate period. When MCPs are moved to unpaused state and IPsec machine
configs are installed on it, then it goes ahead with rendering only host
flavored IPsec daemonset.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>

Author: pperiyasamy

bindata/network/ovn-kubernetes/common/ipsec-containerized.yaml

@@ -49,6 +49,14 @@ spec:
         - |
           #!/bin/bash
           set -exuo pipefail
+
+{{ if .IPsecCheckForLibreswan }}
+          if rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host has libreswan and therefore ipsec will be configured by ipsec host daemonset, this ovn ipsec container doesnt need to init anything"
+            exit 0
+          fi
+{{ end }}
+
 {{ if .NETWORK_NODE_IDENTITY_ENABLE }}
           # When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig
           # that will be used to talk to the API
@@ -189,6 +197,9 @@ spec:
           name: signer-ca
         - mountPath: /etc/openvswitch
           name: etc-openvswitch
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -220,6 +231,13 @@ spec:
           }
           trap cleanup SIGTERM
 
+{{ if .IPsecCheckForLibreswan }}
+          if rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host has libreswan and therefore ipsec will be configured by ipsec host daemonset, this ovn ipsec container will sleep to infinity"
+            sleep infinity
+          fi
+{{ end }}
+
           # Don't start IPsec until ovnkube-node has finished setting up the node
           counter=0
           until [ -f /etc/cni/net.d/10-ovn-kubernetes.conf ]
@@ -276,6 +294,9 @@ spec:
           name: host-var-log-ovs
         - mountPath: /etc/openvswitch
           name: etc-openvswitch
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -288,6 +309,12 @@ spec:
             - -c
             - |
               #!/bin/bash
+{{ if .IPsecCheckForLibreswan }}
+              if rpm --dbpath=/usr/share/rpm -q libreswan; then
+                echo "host has libreswan and therefore ipsec will be configured by ipsec host daemonset, this ovn ipsec container is always \"alive\""
+                exit 0
+              fi
+{{ end }}
               if [[ $(ipsec whack --trafficstatus | wc -l) -eq 0 ]]; then
                 echo "no ipsec traffic configured"
                 exit 10
@@ -321,6 +348,10 @@ spec:
       - name: host-cni-netd
         hostPath:
           path: "{{.CNIConfDir}}"
+      - name: host-usr-share-rpm
+        hostPath:
+          path: /usr/share/rpm
+          type: DirectoryOrCreate
       tolerations:
       - operator: "Exists"
 {{end}}

bindata/network/ovn-kubernetes/common/ipsec-host.yaml

@@ -50,6 +50,12 @@ spec:
         - |
           #!/bin/bash
           set -exuo pipefail
+{{ if .IPsecCheckForLibreswan }}
+          if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, this ovn ipsec container has nothing to init"
+            exit 0
+          fi
+{{ end }}
 {{ if .NETWORK_NODE_IDENTITY_ENABLE }}
           # When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig
           # that will be used to talk to the API
@@ -194,6 +200,9 @@ spec:
           name: etc-openvswitch
         - mountPath: /etc
           name: host-etc
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -210,6 +219,12 @@ spec:
           #!/bin/bash
           set -exuo pipefail
 
+{{ if .IPsecCheckForLibreswan }}
+          if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, this ovn ipsec container will sleep to infinity"
+            sleep infinity
+          fi
+{{ end }}
 
           # Don't start IPsec until ovnkube-node has finished setting up the node
           counter=0
@@ -268,6 +283,13 @@ spec:
                    # In order to maintain traffic flows during container restart, we
                    # need to ensure that xfrm state and policies are not flushed.
 
+{{ if .IPsecCheckForLibreswan }}
+                   if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+                     echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, preStop wont do anything"
+                     exit 0
+                   fi
+{{ end }}
+
                    # Don't allow ovs monitor to cleanup persistent state
                    kill "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)" 2>/dev/null || true
         env:
@@ -291,6 +313,9 @@ spec:
           name: host-var-lib
         - mountPath: /etc
           name: host-etc
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -303,6 +328,12 @@ spec:
             - -c
             - |
               #!/bin/bash
+{{ if .IPsecCheckForLibreswan }}
+              if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+                echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, this ovn ipsec container is always \"alive\""
+                exit 0
+              fi
+{{ end }}
               if [[ $(ipsec whack --trafficstatus | wc -l) -eq 0 ]]; then
                 echo "no ipsec traffic configured"
                 exit 10
@@ -346,6 +377,10 @@ spec:
           path: /etc
           type: Directory
         name: host-etc
+      - name: host-usr-share-rpm
+        hostPath:
+          path: /usr/share/rpm
+          type: DirectoryOrCreate
       tolerations:
       - operator: "Exists"
 {{end}}

pkg/network/ovn_kubernetes.go

@@ -270,10 +270,11 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	}
 
 	IPsecMachineConfigEnable, OVNIPsecDaemonsetEnable, OVNIPsecEnable, renderIPsecHostDaemonSet, renderIPsecContainerizedDaemonSet,
-		renderIPsecDaemonSetAsCreateWaitOnly := shouldRenderIPsec(c, bootstrapResult)
+		renderIPsecDaemonSetAsCreateWaitOnly, renderBothIPsecDemonSetsWhenAPoolPausedState := shouldRenderIPsec(c, bootstrapResult)
 	data.Data["IPsecMachineConfigEnable"] = IPsecMachineConfigEnable
 	data.Data["OVNIPsecDaemonsetEnable"] = OVNIPsecDaemonsetEnable
 	data.Data["OVNIPsecEnable"] = OVNIPsecEnable
+	data.Data["IPsecCheckForLibreswan"] = renderBothIPsecDemonSetsWhenAPoolPausedState
 
 	// Set progressing to true until IPsec DaemonSet is rendered when EW IPsec config is enabled.
 	// TODO Do a poor man's job mapping machine config pool status to CNO progressing state for now.
@@ -610,7 +611,7 @@ func IsIPsecLegacyAPI(conf *operv1.OVNKubernetesConfig) bool {
 // MachineConfig kind is not supported there.
 // All Other cases are not supported in pre-4.14 deployments.
 func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootstrap.BootstrapResult) (renderCNOIPsecMachineConfig, renderIPsecDaemonSet,
-	renderIPsecOVN, renderIPsecHostDaemonSet, renderIPsecContainerizedDaemonSet, renderIPsecDaemonSetAsCreateWaitOnly bool) {
+	renderIPsecOVN, renderIPsecHostDaemonSet, renderIPsecContainerizedDaemonSet, renderIPsecDaemonSetAsCreateWaitOnly, renderBothIPsecDemonSetsWhenAPoolPausedState bool) {
 	isHypershiftHostedCluster := bootstrapResult.Infra.HostedControlPlane != nil
 	isIpsecUpgrade := bootstrapResult.OVN.IPsecUpdateStatus != nil && bootstrapResult.OVN.IPsecUpdateStatus.LegacyIPsecUpgrade
 	isOVNIPsecActive := bootstrapResult.OVN.IPsecUpdateStatus != nil && bootstrapResult.OVN.IPsecUpdateStatus.OVNIPsecActive
@@ -620,7 +621,9 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 	// On upgrade, we will just remove any existing ipsec deployment without making any
 	// change to them. So during upgrade, we must keep track if IPsec MachineConfigs are
 	// active or not for non Hybrid hosted cluster.
-	isIPsecMachineConfigActive := isIPsecMachineConfigActive(bootstrapResult.Infra)
+	machineConfigPoolPaused := isThereAnyMachineConfigPoolPaused(bootstrapResult.Infra)
+	isIPsecMachineConfigActiveInUnPausedPools := isIPsecMachineConfigActive(bootstrapResult.Infra, true)
+	isIPsecMachineConfigActive := isIPsecMachineConfigActive(bootstrapResult.Infra, false)
 	isIPsecMachineConfigNotActiveOnUpgrade := isIpsecUpgrade && !isIPsecMachineConfigActive && !isHypershiftHostedCluster
 	isMachineConfigClusterOperatorReady := bootstrapResult.Infra.MachineConfigClusterOperatorReady
 	isCNOIPsecMachineConfigPresent := isCNOIPsecMachineConfigPresent(bootstrapResult.Infra)
@@ -629,16 +632,24 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 	// or if EW IPsec config is enabled.
 	renderIPsecDaemonSet = isOVNIPsecActive || mode == operv1.IPsecModeFull
 
+	// When any of the machine pool in paused state and IPsec MachineConfig is not active on paused pools and active on
+	// other unpaused pools then render both host and containerized ipsec deployment until machine config pools are moved
+	// into unpaused state and IPsec MachineConfig is active on all pools.
+	renderBothIPsecDemonSetsWhenAPoolPausedState = machineConfigPoolPaused && isIPsecMachineConfigActiveInUnPausedPools &&
+		!isIPsecMachineConfigActive
+
 	// If ipsec is enabled, we render the host ipsec deployment except for
 	// hypershift hosted clusters and we need to wait for the ipsec MachineConfig
 	// extensions to be active first. We must also render host ipsec deployment
 	// at the time of upgrade though user created IPsec Machine Config is not
 	// present/active.
-	renderIPsecHostDaemonSet = (renderIPsecDaemonSet && isIPsecMachineConfigActive && !isHypershiftHostedCluster) || isIPsecMachineConfigNotActiveOnUpgrade
+	renderIPsecHostDaemonSet = (renderIPsecDaemonSet && isIPsecMachineConfigActive && !isHypershiftHostedCluster) ||
+		isIPsecMachineConfigNotActiveOnUpgrade || renderBothIPsecDemonSetsWhenAPoolPausedState
 
 	// The containerized ipsec deployment is only rendered during upgrades or
 	// for hypershift hosted clusters.
-	renderIPsecContainerizedDaemonSet = (renderIPsecDaemonSet && isHypershiftHostedCluster) || isIPsecMachineConfigNotActiveOnUpgrade
+	renderIPsecContainerizedDaemonSet = (renderIPsecDaemonSet && isHypershiftHostedCluster) || isIPsecMachineConfigNotActiveOnUpgrade ||
+		renderBothIPsecDemonSetsWhenAPoolPausedState
 
 	// MachineConfig IPsec extensions rollout is needed for the ipsec enablement and are used in both External and Full modes.
 	// except  when the containerized deployment is used in hypershift hosted clusters.
@@ -653,7 +664,7 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 
 	// While OVN ipsec is being upgraded and IPsec MachineConfigs deployment is in progress
 	// (or) IPsec config in OVN is being disabled, then ipsec deployment is not updated.
-	renderIPsecDaemonSetAsCreateWaitOnly = isIPsecMachineConfigNotActiveOnUpgrade || (isOVNIPsecActive && !renderIPsecOVN)
+	renderIPsecDaemonSetAsCreateWaitOnly = (isIPsecMachineConfigNotActiveOnUpgrade && !renderBothIPsecDemonSetsWhenAPoolPausedState) || (isOVNIPsecActive && !renderIPsecOVN)
 
 	return
 }
@@ -1479,7 +1490,7 @@ func containsNetworkOwnerRef(ownerRefs []metav1.OwnerReference) bool {
 
 // isIPsecMachineConfigActive returns true if both master and worker's machine config pools are ready with
 // ipsec machine config extension rolled out, otherwise returns false.
-func isIPsecMachineConfigActive(infra bootstrap.InfraStatus) bool {
+func isIPsecMachineConfigActive(infra bootstrap.InfraStatus, checkOnlyUnpausedPool bool) bool {
 	if infra.MasterIPsecMachineConfigs == nil || infra.WorkerIPsecMachineConfigs == nil {
 		// One of the IPsec MachineConfig is not created yet, so return false.
 		return false
@@ -1493,18 +1504,49 @@ func isIPsecMachineConfigActive(infra bootstrap.InfraStatus) bool {
 			hasSourceInMachineConfigStatus(status, machineConfigs)
 	}
 	for _, masterMCP := range infra.MasterMCPs {
+		if checkOnlyUnpausedPool && masterMCP.Spec.Paused {
+			continue
+		}
 		if !ipSecPluginOnPool(masterMCP.Status, infra.MasterIPsecMachineConfigs) {
 			return false
 		}
 	}
 	for _, workerMCP := range infra.WorkerMCPs {
+		if checkOnlyUnpausedPool && workerMCP.Spec.Paused {
+			continue
+		}
 		if !ipSecPluginOnPool(workerMCP.Status, infra.WorkerIPsecMachineConfigs) {
 			return false
 		}
 	}
 	return true
 }
 
+func isThereAnyMachineConfigPoolPaused(infra bootstrap.InfraStatus) bool {
+	if infra.MasterIPsecMachineConfigs == nil || infra.WorkerIPsecMachineConfigs == nil {
+		// One of the IPsec MachineConfig is not created yet, so return false.
+		return false
+	}
+	if len(infra.MasterMCPs) == 0 || len(infra.WorkerMCPs) == 0 {
+		// When none of MachineConfig pools exist, then return false. needed for unit test.
+		return false
+	}
+	isPausedPool := func(pool mcfgv1.MachineConfigPool) bool {
+		return pool.Spec.Paused
+	}
+	for _, masterMCP := range infra.MasterMCPs {
+		if isPausedPool(masterMCP) {
+			return true
+		}
+	}
+	for _, workerMCP := range infra.WorkerMCPs {
+		if isPausedPool(workerMCP) {
+			return true
+		}
+	}
+	return false
+}
+
 func hasSourceInMachineConfigStatus(machineConfigStatus mcfgv1.MachineConfigPoolStatus, machineConfigs []*mcfgv1.MachineConfig) bool {
 	ipSecMachineConfigNames := sets.New[string]()
 	for _, machineConfig := range machineConfigs {