Add toggle for Multi-cluster Pod-to-Pod connectivity (#4605)

Signed-off-by: hujiajing <hjiajing@vmware.com>
antrea-io · Feb 16, 2023 · cf90cfa · cf90cfa
1 parent e793e57
commit cf90cfa
Show file tree

Hide file tree

Showing 13 changed files with 57 additions and 16 deletions.
diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md
@@ -86,6 +86,7 @@ Kubernetes: `>= 1.16.0-0`
 | multicast.igmpQueryInterval | string | `"125s"` | The interval at which the antrea-agent sends IGMP queries to Pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". |
 | multicast.multicastInterfaces | list | `[]` | Names of the interfaces on Nodes that are used to forward multicast traffic. |
 | multicluster.enableGateway | bool | `false` | Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. This feature is supported only with encap mode. |
+| multicluster.enablePodToPodConnectivity | bool | `false` | Enable Multi-cluster Pod to Pod connectivity. |
 | multicluster.enableStretchedNetworkPolicy | bool | `false` | Enable Multi-cluster NetworkPolicy. Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy. |
 | multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is antrea-agent's Namespace. |
 | noSNAT | bool | `false` | Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to the external network. |

diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -344,6 +344,8 @@ multicluster:
 # Enable Multi-cluster NetworkPolicy (ingress rules).
 # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
   enableStretchedNetworkPolicy: {{ .enableStretchedNetworkPolicy }}
+# Enable Pod to Pod connectivity.
+  enablePodToPodConnectivity: {{ .enablePodToPodConnectivity }}
 {{- end }}
 
 {{- if .Values.featureGates.SecondaryNetwork }}

diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml
@@ -336,6 +336,8 @@ multicluster:
   # -- Enable Multi-cluster NetworkPolicy.
   # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
   enableStretchedNetworkPolicy: false
+  # -- Enable Multi-cluster Pod to Pod connectivity.
+  enablePodToPodConnectivity: false
 
 testing:
   ## -- enable code coverage measurement (used when testing Antrea only).

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3240,6 +3240,8 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4297,7 +4299,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
+        checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
       labels:
         app: antrea
         component: antrea-agent
@@ -4538,7 +4540,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
+        checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3240,6 +3240,8 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4297,7 +4299,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
+        checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
       labels:
         app: antrea
         component: antrea-agent
@@ -4539,7 +4541,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
+        checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3240,6 +3240,8 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4297,7 +4299,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: aff837005adc6d91f4b5ace3a87c08cfa49e26c60e284ebd234eea34ce5de91f
+        checksum/config: fca1f2d4967020380202ef0c2394b560055830ee2770e41f791af76b42559659
       labels:
         app: antrea
         component: antrea-agent
@@ -4536,7 +4538,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: aff837005adc6d91f4b5ace3a87c08cfa49e26c60e284ebd234eea34ce5de91f
+        checksum/config: fca1f2d4967020380202ef0c2394b560055830ee2770e41f791af76b42559659
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3253,6 +3253,8 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4310,7 +4312,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e3e3255bb4f4cd13bce262d8c8d5f4aead3e84e52ea1775c34898c69b80fad33
+        checksum/config: ab53bf1e28a67ba5be2b99989a8d28b31d716d79b207a610cd5258ead514eb6b
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -4595,7 +4597,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e3e3255bb4f4cd13bce262d8c8d5f4aead3e84e52ea1775c34898c69b80fad33
+        checksum/config: ab53bf1e28a67ba5be2b99989a8d28b31d716d79b207a610cd5258ead514eb6b
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3240,6 +3240,8 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4297,7 +4299,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 846220b3b64851cea85fb1e374c3ffdb29376ea729a494dad1cb230b3e5efe8c
+        checksum/config: 2c1c5158b6a3ea32eff58bc1e498592e80ebecee07f51b10c722b67afce7b964
       labels:
         app: antrea
         component: antrea-agent
@@ -4536,7 +4538,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 846220b3b64851cea85fb1e374c3ffdb29376ea729a494dad1cb230b3e5efe8c
+        checksum/config: 2c1c5158b6a3ea32eff58bc1e498592e80ebecee07f51b10c722b67afce7b964
       labels:
         app: antrea
         component: antrea-controller

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -342,6 +342,7 @@ func run(o *Options) error {
 			nodeConfig,
 			mcNamespace,
 			o.config.Multicluster.EnableStretchedNetworkPolicy,
+			o.config.Multicluster.EnablePodToPodConnectivity,
 		)
 	}
 	if enableMulticlusterNP {

diff --git a/docs/multicluster/user-guide.md b/docs/multicluster/user-guide.md
@@ -511,9 +511,11 @@ for more information.
 
 Since Antrea v1.9.0, Multi-cluster supports routing Pod traffic across clusters
 through Multi-cluster Gateways. Pod IPs can be reached in all member clusters
-within a ClusterSet. To enable this feature, the cluster's Pod CIDRs must be set in
-ConfigMap `antrea-mc-controller-config` of each member cluster like the example
-below. Note, **Pod CIDRs must not overlap among clusters to enable cross-cluster
+within a ClusterSet. To enable this feature, the cluster's Pod CIDRs must be set
+in ConfigMap `antrea-mc-controller-config` of each member cluster and
+`multicluster.enablePodToPodConnectivity` must be set to `true` in the `antrea-agent`
+configuration.
+Note, **Pod CIDRs must not overlap among clusters to enable cross-cluster
 Pod-to-Pod connectivity**.
 
 ```yaml
@@ -533,6 +535,16 @@ metadata:
   namespace: kube-system
 ```
 
+```yaml
+antrea-controller.conf: |
+  featureGates:
+...
+    Multicluster: true
+...
+  multicluster:
+    enablePodToPodConnectivity: true
+```
+
 You can edit [antrea-multicluster-member.yml](../../multicluster/build/yamls/antrea-multicluster-member.yml),
 or use `kubectl edit` to change the ConfigMap:
 

diff --git a/pkg/agent/multicluster/mc_route_controller.go b/pkg/agent/multicluster/mc_route_controller.go
@@ -77,6 +77,7 @@ type MCRouteController struct {
 	// The Namespace where Antrea Multi-cluster Controller is running.
 	namespace                    string
 	enableStretchedNetworkPolicy bool
+	enablePodToPodConnectivity   bool
 }
 
 func NewMCRouteController(
@@ -89,6 +90,7 @@ func NewMCRouteController(
 	nodeConfig *config.NodeConfig,
 	namespace string,
 	enableStretchedNetworkPolicy bool,
+	enablePodToPodConnectivity bool,
 ) *MCRouteController {
 	controller := &MCRouteController{
 		mcClient:                     mcClient,
@@ -106,6 +108,7 @@ func NewMCRouteController(
 		installedCIImports:           make(map[string]*mcv1alpha1.ClusterInfoImport),
 		namespace:                    namespace,
 		enableStretchedNetworkPolicy: enableStretchedNetworkPolicy,
+		enablePodToPodConnectivity:   enablePodToPodConnectivity,
 	}
 	controller.gwInformer.Informer().AddEventHandlerWithResyncPeriod(
 		cache.ResourceEventHandlerFuncs{
@@ -351,8 +354,10 @@ func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gatewa
 	var ciImportNoChange bool
 	if installedCIImp != nil {
 		oldTunnelPeerIPToRemoteGW := getPeerGatewayIP(installedCIImp.Spec)
-		ciImportNoChange = oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR &&
-			sets.NewString(installedCIImp.Spec.PodCIDRs...).Equal(sets.NewString(ciImport.Spec.PodCIDRs...))
+		ciImportNoChange = oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR
+		if c.enablePodToPodConnectivity {
+			ciImportNoChange = ciImportNoChange && sets.NewString(installedCIImp.Spec.PodCIDRs...).Equal(sets.NewString(ciImport.Spec.PodCIDRs...))
+		}
 	}
 
 	if ciImportNoChange && !activeGWChanged {
@@ -362,7 +367,10 @@ func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gatewa
 
 	klog.InfoS("Adding/updating remote Gateway Node flows for Multi-cluster", "gateway", klog.KObj(activeGW),
 		"node", c.nodeConfig.Name, "peer", tunnelPeerIPToRemoteGW)
-	allCIDRs := append([]string{ciImport.Spec.ServiceCIDR}, ciImport.Spec.PodCIDRs...)
+	allCIDRs := []string{ciImport.Spec.ServiceCIDR}
+	if c.enablePodToPodConnectivity {
+		allCIDRs = append(allCIDRs, ciImport.Spec.PodCIDRs...)
+	}
 	peerConfigs, err := generatePeerConfigs(allCIDRs, tunnelPeerIPToRemoteGW)
 	if err != nil {
 		klog.ErrorS(err, "Parse error for serviceCIDR from remote cluster", "clusterinfoimport", ciImport.Name, "gateway", activeGW.Name)

diff --git a/pkg/agent/multicluster/mc_route_controller_test.go b/pkg/agent/multicluster/mc_route_controller_test.go
@@ -61,6 +61,7 @@ func newMCRouteController(t *testing.T, nodeConfig *config.NodeConfig) (*fakeRou
 		nodeConfig,
 		"default",
 		true,
+		true,
 	)
 	return &fakeRouteController{
 		MCRouteController: c,

diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go
@@ -296,6 +296,10 @@ type MulticlusterConfig struct {
 	// Enable Multi-cluster NetworkPolicy which allows Antrea-native policy ingress rules to select peers
 	// from all clusters in a ClusterSet.
 	EnableStretchedNetworkPolicy bool `yaml:"enableStretchedNetworkPolicy,omitempty"`
+	// Enable Multi-cluster Pod to Pod connectivity which allows one Pod access to another Pod in other member
+	// clusters directly. This feature also requires Pod CIDRs to be provided in the Multi-cluster Controller
+	// configuration.
+	EnablePodToPodConnectivity bool `yaml:"enablePodToPodConnectivity,omitempty"`
 }
 
 type ExternalNodeConfig struct {