From 5086b34ebb001dd7f9875573ddf9be4768995eff Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Tue, 6 Feb 2024 14:33:10 +0800 Subject: [PATCH] Add BGPPolicy controller Signed-off-by: Hongliang Liu --- build/charts/antrea/README.md | 1 + build/charts/antrea/conf/antrea-agent.conf | 8 + .../antrea/templates/agent/bgp-secret.yaml | 6 + .../antrea/templates/agent/clusterrole.yaml | 10 + build/charts/antrea/values.yaml | 4 + build/yamls/antrea-aks.yml | 30 +- build/yamls/antrea-eks.yml | 30 +- build/yamls/antrea-gke.yml | 30 +- build/yamls/antrea-ipsec.yml | 30 +- build/yamls/antrea.yml | 30 +- cmd/antrea-agent/agent.go | 18 + cmd/antrea-agent/options.go | 6 + docs/feature-gates.md | 7 +- go.mod | 18 +- go.sum | 59 +- hack/update-codegen-dockerized.sh | 1 + pkg/agent/bgp/controller.go | 1049 +++++++++++ pkg/agent/bgp/controller_test.go | 1574 +++++++++++++++++ pkg/agent/bgp/engine/gobgp.go | 295 +++ pkg/agent/bgp/engine/gobgp_test.go | 82 + pkg/agent/bgp/engine/interface.go | 114 ++ pkg/agent/bgp/engine/logs.go | 116 ++ pkg/agent/bgp/engine/testing/mock_engine.go | 183 ++ .../networkpolicy/l7engine/reconciler.go | 2 +- .../handlers/featuregates/handler_test.go | 2 + pkg/config/agent/config.go | 7 + pkg/features/antrea_features.go | 6 + test/integration/agent/gobgp_test.go | 415 +++++ 28 files changed, 4114 insertions(+), 19 deletions(-) create mode 100644 build/charts/antrea/templates/agent/bgp-secret.yaml create mode 100644 pkg/agent/bgp/controller.go create mode 100644 pkg/agent/bgp/controller_test.go create mode 100644 pkg/agent/bgp/engine/gobgp.go create mode 100644 pkg/agent/bgp/engine/gobgp_test.go create mode 100644 pkg/agent/bgp/engine/interface.go create mode 100644 pkg/agent/bgp/engine/logs.go create mode 100644 pkg/agent/bgp/engine/testing/mock_engine.go create mode 100644 test/integration/agent/gobgp_test.go diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index ba66fad08fa..8c87ec98b90 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -64,6 +64,7 @@ Kubernetes: `>= 1.19.0-0` | auditLogging.maxAge | int | `28` | MaxAge is the maximum number of days to retain old log files based on the timestamp encoded in their filename. If set to 0, old log files are not removed based on age. | | auditLogging.maxBackups | int | `3` | MaxBackups is the maximum number of old log files to retain. If set to 0, all log files will be retained (unless MaxAge causes them to be deleted). | | auditLogging.maxSize | int | `500` | MaxSize is the maximum size in MB of a log file before it gets rotated. | +| bgpPolicy.secretName | string | `"antrea-bgp-passwords"` | The name of the Secret storing the passwords of BGP peers. | | clientCAFile | string | `""` | File path of the certificate bundle for all the signers that is recognized for incoming client certificates. | | cni.hostBinPath | string | `"/opt/cni/bin"` | Installation path of CNI binaries on the host. | | cni.plugins | object | `{"bandwidth":true,"portmap":true}` | Chained plugins to use alongside antrea-cni. | diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index e140f3c0790..492fc8a5090 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -91,6 +91,9 @@ featureGates: # Enable NodeLatencyMonitor to monitor the latency between Nodes. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "NodeLatencyMonitor" "default" false) }} +# Allow users to advertise Service IPs, Pod IPs, and Egress IPs to BGP peers. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "BGPPolicy" "default" false) }} + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: {{ .Values.ovs.bridgeName | quote }} @@ -443,3 +446,8 @@ secondaryNetwork: {{- end }} {{- end }} + +bgpPolicy: + # The name of the Secret storing passwords of the BGP peers. For each BGP peer, the Secret key is generated by + # concatenating its IP address and AS number, e.g., `192.168.1.1-65521`. + secretName: {{ .Values.bgpPolicy.secretName | quote }} diff --git a/build/charts/antrea/templates/agent/bgp-secret.yaml b/build/charts/antrea/templates/agent/bgp-secret.yaml new file mode 100644 index 00000000000..828ddf3f96e --- /dev/null +++ b/build/charts/antrea/templates/agent/bgp-secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.bgpPolicy.secretName }} + namespace: {{ .Release.Namespace }} +type: Opaque diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index ef1c43e40e8..3935e6aa81e 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -177,6 +177,7 @@ rules: - apiGroups: - crd.antrea.io resources: + - bgppolicies - externalippools - ippools - trafficcontrols @@ -234,3 +235,12 @@ rules: - create - patch - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - {{ .Values.bgpPolicy.secretName }} + verbs: + - get + - watch diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index ce7923d197b..67b68e343fc 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -207,6 +207,10 @@ secondaryNetwork: # [{bridgeName: "br1", physicalInterfaces: ["eth1"]}] ovsBridges: [] +bgpPolicy: + # -- The name of the Secret storing the passwords of BGP peers. + secretName: "antrea-bgp-passwords" + agent: # -- Port for the antrea-agent APIServer to serve on. apiPort: 10350 diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 1d9c4c57eff..88d7dfda956 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3684,6 +3684,14 @@ metadata: labels: app: antrea --- +# Source: antrea/templates/agent/bgp-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: antrea-bgp-passwords + namespace: kube-system +type: Opaque +--- # Source: antrea/templates/agent/secret.yaml apiVersion: v1 kind: Secret @@ -3807,6 +3815,9 @@ data: # Enable NodeLatencyMonitor to monitor the latency between Nodes. # NodeLatencyMonitor: false + # Allow users to advertise Service IPs, Pod IPs, and Egress IPs to BGP peers. + # BGPPolicy: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -4115,6 +4126,11 @@ data: maxAge: 28 # Compress enables gzip compression on rotated files. compress: true + + bgpPolicy: + # The name of the Secret storing passwords of the BGP peers. For each BGP peer, the Secret key is generated by + # concatenating its IP address and AS number, e.g., `192.168.1.1-65521`. + secretName: "antrea-bgp-passwords" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -4445,6 +4461,7 @@ rules: - apiGroups: - crd.antrea.io resources: + - bgppolicies - externalippools - ippools - trafficcontrols @@ -4502,6 +4519,15 @@ rules: - create - patch - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-bgp-passwords + verbs: + - get + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -5110,7 +5136,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: f976029accf54258d01ad907fe19b50ac671eee014cd8aea968c6a0bc7e8f95a + checksum/config: 178df1e0c099cb87d001786b31a8a1e598bfc14e2d6eb974f46dc3bdfc5ef3dc labels: app: antrea component: antrea-agent @@ -5348,7 +5374,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: f976029accf54258d01ad907fe19b50ac671eee014cd8aea968c6a0bc7e8f95a + checksum/config: 178df1e0c099cb87d001786b31a8a1e598bfc14e2d6eb974f46dc3bdfc5ef3dc labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 62e00bc5f1b..67e7e7312e1 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3684,6 +3684,14 @@ metadata: labels: app: antrea --- +# Source: antrea/templates/agent/bgp-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: antrea-bgp-passwords + namespace: kube-system +type: Opaque +--- # Source: antrea/templates/agent/secret.yaml apiVersion: v1 kind: Secret @@ -3807,6 +3815,9 @@ data: # Enable NodeLatencyMonitor to monitor the latency between Nodes. # NodeLatencyMonitor: false + # Allow users to advertise Service IPs, Pod IPs, and Egress IPs to BGP peers. + # BGPPolicy: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -4115,6 +4126,11 @@ data: maxAge: 28 # Compress enables gzip compression on rotated files. compress: true + + bgpPolicy: + # The name of the Secret storing passwords of the BGP peers. For each BGP peer, the Secret key is generated by + # concatenating its IP address and AS number, e.g., `192.168.1.1-65521`. + secretName: "antrea-bgp-passwords" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -4445,6 +4461,7 @@ rules: - apiGroups: - crd.antrea.io resources: + - bgppolicies - externalippools - ippools - trafficcontrols @@ -4502,6 +4519,15 @@ rules: - create - patch - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-bgp-passwords + verbs: + - get + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -5110,7 +5136,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: f976029accf54258d01ad907fe19b50ac671eee014cd8aea968c6a0bc7e8f95a + checksum/config: 178df1e0c099cb87d001786b31a8a1e598bfc14e2d6eb974f46dc3bdfc5ef3dc labels: app: antrea component: antrea-agent @@ -5349,7 +5375,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: f976029accf54258d01ad907fe19b50ac671eee014cd8aea968c6a0bc7e8f95a + checksum/config: 178df1e0c099cb87d001786b31a8a1e598bfc14e2d6eb974f46dc3bdfc5ef3dc labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 22665a948c9..61a7891531f 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3684,6 +3684,14 @@ metadata: labels: app: antrea --- +# Source: antrea/templates/agent/bgp-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: antrea-bgp-passwords + namespace: kube-system +type: Opaque +--- # Source: antrea/templates/agent/secret.yaml apiVersion: v1 kind: Secret @@ -3807,6 +3815,9 @@ data: # Enable NodeLatencyMonitor to monitor the latency between Nodes. # NodeLatencyMonitor: false + # Allow users to advertise Service IPs, Pod IPs, and Egress IPs to BGP peers. + # BGPPolicy: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -4115,6 +4126,11 @@ data: maxAge: 28 # Compress enables gzip compression on rotated files. compress: true + + bgpPolicy: + # The name of the Secret storing passwords of the BGP peers. For each BGP peer, the Secret key is generated by + # concatenating its IP address and AS number, e.g., `192.168.1.1-65521`. + secretName: "antrea-bgp-passwords" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -4445,6 +4461,7 @@ rules: - apiGroups: - crd.antrea.io resources: + - bgppolicies - externalippools - ippools - trafficcontrols @@ -4502,6 +4519,15 @@ rules: - create - patch - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-bgp-passwords + verbs: + - get + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -5110,7 +5136,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 5299e6235e262daf606758cf900766470fcb8dd21a0d707a3ae284548bd8c2b2 + checksum/config: 2b80cc90ebecf3ab2716df9ee529aa0388283256c85d37b440d397b8a3d5c984 labels: app: antrea component: antrea-agent @@ -5346,7 +5372,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 5299e6235e262daf606758cf900766470fcb8dd21a0d707a3ae284548bd8c2b2 + checksum/config: 2b80cc90ebecf3ab2716df9ee529aa0388283256c85d37b440d397b8a3d5c984 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 0c9afaa9857..b96cfb19e54 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3684,6 +3684,14 @@ metadata: labels: app: antrea --- +# Source: antrea/templates/agent/bgp-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: antrea-bgp-passwords + namespace: kube-system +type: Opaque +--- # Source: antrea/templates/agent/ipsec-secret.yaml apiVersion: v1 kind: Secret @@ -3820,6 +3828,9 @@ data: # Enable NodeLatencyMonitor to monitor the latency between Nodes. # NodeLatencyMonitor: false + # Allow users to advertise Service IPs, Pod IPs, and Egress IPs to BGP peers. + # BGPPolicy: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -4128,6 +4139,11 @@ data: maxAge: 28 # Compress enables gzip compression on rotated files. compress: true + + bgpPolicy: + # The name of the Secret storing passwords of the BGP peers. For each BGP peer, the Secret key is generated by + # concatenating its IP address and AS number, e.g., `192.168.1.1-65521`. + secretName: "antrea-bgp-passwords" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -4458,6 +4474,7 @@ rules: - apiGroups: - crd.antrea.io resources: + - bgppolicies - externalippools - ippools - trafficcontrols @@ -4515,6 +4532,15 @@ rules: - create - patch - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-bgp-passwords + verbs: + - get + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -5123,7 +5149,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: ba93df141f512a1f8483114b5994444c7231b298e7e9133483ddc1f4210ec395 + checksum/config: 940dc5c50998e1328e6a4188b3e5125f1005a420f5fec24bdce7d66d43739eb7 checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -5405,7 +5431,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: ba93df141f512a1f8483114b5994444c7231b298e7e9133483ddc1f4210ec395 + checksum/config: 940dc5c50998e1328e6a4188b3e5125f1005a420f5fec24bdce7d66d43739eb7 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index b938d9f83bc..79dc3a41a36 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3684,6 +3684,14 @@ metadata: labels: app: antrea --- +# Source: antrea/templates/agent/bgp-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: antrea-bgp-passwords + namespace: kube-system +type: Opaque +--- # Source: antrea/templates/agent/secret.yaml apiVersion: v1 kind: Secret @@ -3807,6 +3815,9 @@ data: # Enable NodeLatencyMonitor to monitor the latency between Nodes. # NodeLatencyMonitor: false + # Allow users to advertise Service IPs, Pod IPs, and Egress IPs to BGP peers. + # BGPPolicy: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -4115,6 +4126,11 @@ data: maxAge: 28 # Compress enables gzip compression on rotated files. compress: true + + bgpPolicy: + # The name of the Secret storing passwords of the BGP peers. For each BGP peer, the Secret key is generated by + # concatenating its IP address and AS number, e.g., `192.168.1.1-65521`. + secretName: "antrea-bgp-passwords" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -4445,6 +4461,7 @@ rules: - apiGroups: - crd.antrea.io resources: + - bgppolicies - externalippools - ippools - trafficcontrols @@ -4502,6 +4519,15 @@ rules: - create - patch - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-bgp-passwords + verbs: + - get + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -5110,7 +5136,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: aca23e21519e0fc112647f23d3ce6f92a3dea0bc7ebf1c6d7a7eed2dbe80f0a3 + checksum/config: b9fbb71804b11c687284d14ab8fa26b267fc04e1884c7a732b938e58002174bc labels: app: antrea component: antrea-agent @@ -5346,7 +5372,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: aca23e21519e0fc112647f23d3ce6f92a3dea0bc7ebf1c6d7a7eed2dbe80f0a3 + checksum/config: b9fbb71804b11c687284d14ab8fa26b267fc04e1884c7a732b938e58002174bc labels: app: antrea component: antrea-controller diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 83a2d05286c..138ea01916d 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -34,6 +34,7 @@ import ( mcinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions" "antrea.io/antrea/pkg/agent" "antrea.io/antrea/pkg/agent/apiserver" + "antrea.io/antrea/pkg/agent/bgp" "antrea.io/antrea/pkg/agent/client" "antrea.io/antrea/pkg/agent/cniserver" "antrea.io/antrea/pkg/agent/cniserver/ipam" @@ -742,6 +743,23 @@ func run(o *Options) error { } } + if features.DefaultFeatureGate.Enabled(features.BGPPolicy) { + bgpPolicyInformer := crdInformerFactory.Crd().V1alpha1().BGPPolicies() + bgpController, err := bgp.NewBGPPolicyController(nodeInformer, + serviceInformer, + egressInformer, + bgpPolicyInformer, + endpointSliceInformer, + k8sClient, + o.config.BGPPolicy.SecretName, + nodeConfig, + networkConfig) + if err != nil { + return err + } + go bgpController.Run(stopCh) + } + if features.DefaultFeatureGate.Enabled(features.TrafficControl) { tcController := trafficcontrol.NewTrafficControlController(ofClient, ifaceStore, diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 7e1ec94de2b..48e3239b4fe 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -61,6 +61,7 @@ const ( defaultAuditLogsMaxAge = 28 defaultAuditLogsCompressed = true defaultPacketInRate = 500 + defaultBGPPolicySecretName = "antrea-bgp-passwords" //nolint ) var defaultIGMPQueryVersions = []int{1, 2, 3} @@ -494,6 +495,11 @@ func (o *Options) setK8sNodeDefaultOptions() { o.config.Egress.MaxEgressIPsPerNode = defaultMaxEgressIPsPerNode } } + if features.DefaultFeatureGate.Enabled(features.BGPPolicy) { + if o.config.BGPPolicy.SecretName == "" { + o.config.BGPPolicy.SecretName = defaultBGPPolicySecretName + } + } } func (o *Options) validateEgressConfig(encapMode config.TrafficEncapModeType) error { diff --git a/docs/feature-gates.md b/docs/feature-gates.md index 287d9dacc58..ccfdd1bdc16 100644 --- a/docs/feature-gates.md +++ b/docs/feature-gates.md @@ -31,7 +31,7 @@ edit the Agent configuration in the ## List of Available Features | Feature Name | Component | Default | Stage | Alpha Release | Beta Release | GA Release | Extra Requirements | Notes | -| ----------------------------- | ------------------ | ------- | ----- | ------------- | ------------ | ---------- | ------------------ | --------------------------------------------- | +|-------------------------------| ------------------ | ------- | ----- |---------------| ------------ | ---------- |--------------------| --------------------------------------------- | | `AntreaProxy` | Agent | `true` | GA | v0.8 | v0.11 | v1.14 | Yes | Must be enabled for Windows. | | `EndpointSlice` | Agent | `true` | GA | v0.13.0 | v1.11 | v1.14 | Yes | | | `TopologyAwareHints` | Agent | `true` | Beta | v1.8 | v1.12 | N/A | Yes | | @@ -59,6 +59,7 @@ edit the Agent configuration in the | `EgressSeparateSubnet` | Agent | `false` | Alpha | v1.15 | N/A | N/A | No | | | `NodeNetworkPolicy` | Agent | `false` | Alpha | v1.15 | N/A | N/A | Yes | | | `L7FlowExporter` | Agent | `false` | Alpha | v1.15 | N/A | N/A | Yes | | +| `BGPPolicy` | Agent | `false` | Alpha | v2.1 | N/A | N/A | No | | ## Description and Requirements of Features @@ -438,3 +439,7 @@ Refer to this [document](network-flow-visibility.md#l7-visibility) for more info #### Requirements for this Feature - Linux Nodes only. + +### BGPPolicy + +`BGPPolicy` allows users to advertise Service IPs, Pod IPs, and Egress IPs to external BGP peers. diff --git a/go.mod b/go.mod index bb7bc11e6d6..6651464a08e 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 + github.com/osrg/gobgp/v3 v3.26.0 github.com/pkg/sftp v1.13.6 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/common v0.47.0 @@ -94,7 +95,7 @@ require ( github.com/alexflint/go-filemutex v1.2.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect - github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect + github.com/armon/go-metrics v0.4.0 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.12 // indirect @@ -123,6 +124,9 @@ require ( github.com/contiv/libovsdb v0.0.0-20170227191248-d0061a53e358 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/eapache/channels v1.1.0 // indirect + github.com/eapache/queue v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -150,20 +154,23 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-immutable-radix v1.0.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-sockaddr v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/k-sone/critbitgo v1.4.0 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -172,6 +179,7 @@ require ( github.com/mdlayher/netlink v1.7.2 // indirect github.com/mdlayher/socket v0.5.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -180,6 +188,7 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/paulmach/orb v0.8.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pion/dtls/v2 v2.2.4 // indirect @@ -196,7 +205,11 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/viper v1.16.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/ti-mo/netfilter v0.5.2 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect @@ -228,6 +241,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect k8s.io/cli-runtime v0.29.2 // indirect k8s.io/kms v0.29.2 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect diff --git a/go.sum b/go.sum index 2a6a46d87f3..daf4a3dbe3f 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,7 @@ github.com/ClickHouse/clickhouse-go/v2 v2.6.1 h1:82UzCrD8cYEb/Bs/LOO3dlBZZyL+Slv github.com/ClickHouse/clickhouse-go/v2 v2.6.1/go.mod h1:SvXuWqDsiHJE3VAn2+3+nz9W9exOSigyskcs4DAcxJQ= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= @@ -52,7 +53,9 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexflint/go-filemutex v1.2.0 h1:1v0TJPDtlhgpW4nJ+GvxCLSlUDC3+gW0CQQvlmfDR/s= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= @@ -62,8 +65,8 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8Q= +github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -128,6 +131,7 @@ github.com/cenkalti/rpc2 v0.0.0-20180727162946-9642ea02d0aa h1:t+iWhuJE2aropY4ux github.com/cenkalti/rpc2 v0.0.0-20180727162946-9642ea02d0aa/go.mod h1:v2npkhrXyk5BCnkNIiPdRI23Uq6uWPUQGL2hnRcRr/M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= @@ -137,6 +141,8 @@ github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= @@ -180,6 +186,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -190,6 +198,10 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k= +github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -215,6 +227,8 @@ github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -234,6 +248,7 @@ github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6v github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI= github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -298,6 +313,8 @@ github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85n github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -400,12 +417,15 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9K github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack/v2 v2.1.1 h1:xQEY9yB2wnHitoSzk/B9UjXWRQ67QKu5AOm8aFp8N3I= github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -415,6 +435,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/memberlist v0.5.1 h1:mk5dRuzeDNis2bi6LLoQIXfMH7JQvAzt3mQD0vNZZUo= github.com/hashicorp/memberlist v0.5.1/go.mod h1:zGDXV6AqbDTKTM6yxW0I4+JtFzZAJVoIPvss4hV8F24= @@ -450,11 +471,14 @@ github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE= +github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s= github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0 h1:MjRRgZyTGo90G+UrwlDQjU+uG4Z7By65qvQxGoILT/8= github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0= github.com/k8snetworkplumbingwg/sriov-cni v2.1.0+incompatible h1:5comk9qUB9j99Oc+rvnm92RWWe9urdJ1TP3cXM3fmmc= @@ -486,6 +510,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -542,6 +568,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= @@ -584,14 +612,18 @@ github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/osrg/gobgp/v3 v3.26.0 h1:/iHaQKNgp0dRI3/RGt/j60aUeoGng6CL0VATVfQXEPE= +github.com/osrg/gobgp/v3 v3.26.0/go.mod h1:ZGeSti9mURR/o5hf5R6T1FM5g1yiEBZbhP+TuqYJUpI= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulmach/orb v0.8.0 h1:W5XAt5yNPNnhaMNEf0xNSkBMJ1LzOzdk2MRlB6EN0Vs= github.com/paulmach/orb v0.8.0/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= @@ -617,6 +649,7 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -628,11 +661,13 @@ github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k= github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= @@ -672,12 +707,16 @@ github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -685,6 +724,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -702,8 +743,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/ti-mo/conntrack v0.5.1 h1:opEwkFICnDbQc0BUXl73PHBK0h23jEIFVjXsqvF4GY0= github.com/ti-mo/conntrack v0.5.1/go.mod h1:T6NCbkMdVU4qEIgwL0njA6lw/iCAbzchlnwm1Sa314o= github.com/ti-mo/netfilter v0.5.2 h1:CTjOwFuNNeZ9QPdRXt1MZFLFUf84cKtiQutNauHWd40= @@ -713,6 +757,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -903,6 +948,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1042,6 +1088,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= @@ -1053,6 +1101,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/hack/update-codegen-dockerized.sh b/hack/update-codegen-dockerized.sh index 26b77491603..d9d59cdc643 100755 --- a/hack/update-codegen-dockerized.sh +++ b/hack/update-codegen-dockerized.sh @@ -37,6 +37,7 @@ ANTREA_PKG="antrea.io/antrea" ANTREA_PROTO_PKG="antrea_io.antrea" MOCKGEN_TARGETS=( + "pkg/agent/bgp/engine Interface testing" "pkg/agent/cniserver SriovNet testing" "pkg/agent/cniserver/ipam IPAMDriver testing" "pkg/agent/flowexporter/connections ConnTrackDumper,NetFilterConnTrack testing" diff --git a/pkg/agent/bgp/controller.go b/pkg/agent/bgp/controller.go new file mode 100644 index 00000000000..ad754ab8fd5 --- /dev/null +++ b/pkg/agent/bgp/controller.go @@ -0,0 +1,1049 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bgp + +import ( + "context" + "fmt" + "reflect" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + discoveryinformers "k8s.io/client-go/informers/discovery/v1" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" + "k8s.io/utils/strings/slices" + + "antrea.io/antrea/pkg/agent/bgp/engine" + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" + "antrea.io/antrea/pkg/apis/crd/v1beta1" + crdinformersv1a1 "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha1" + crdinformersv1b1 "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1beta1" + crdlistersv1a1 "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" + crdlistersv1b1 "antrea.io/antrea/pkg/client/listers/crd/v1beta1" + "antrea.io/antrea/pkg/features" + "antrea.io/antrea/pkg/util/env" + utilipset "antrea.io/antrea/pkg/util/sets" +) + +const ( + controllerName = "BGPPolicyController" + // How long to wait before retrying the processing of a TrafficControl change. + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + // Default number of workers processing a TrafficControl change. + defaultWorkers = 4 + // Disable resyncing. + resyncPeriod time.Duration = 0 +) + +const ( + bgpRouteIDAnnotation = "antrea.io/bgp-route-id" +) + +const ( + ipv4Suffix = "/32" + ipv6Suffix = "/128" +) + +var ( + protocolIPv4 = utilnet.IPv4 + protocolIPv6 = utilnet.IPv6 + + newBGPServerFn = engine.NewGoBGPServer +) + +type nodeToBGPPolicyBinding struct { + effectiveBP string + alternativeBPs sets.Set[string] +} + +type bgpPolicyState struct { + // The local BGP server created for the BGPPolicy. + bgpServer engine.Interface + // The port on which local the BGP server listens. + listenPort int32 + // The AS number used by the local BGP server. + localASN int32 + // The router ID used by the local BGP server. + routerID string + // Routes to be advertised to BGP peers. + routes map[utilnet.IPFamily]sets.Set[engine.Route] + // peers maps IP families to concatenated strings of BGP peer IP addresses and ASNs. + // Example: "192.168.77.100-65000", "2001::1-65000". + peers map[utilnet.IPFamily]sets.Set[string] + // peerConfigs maps concatenated string of BGP peer IP addresses and ASN to the configuration of the peer. + peerConfigs map[string]*engine.PeerConfig +} + +type Controller struct { + nodeInformer cache.SharedIndexInformer + nodeLister corelisters.NodeLister + nodeListerSynced cache.InformerSynced + + serviceInformer cache.SharedIndexInformer + serviceLister corelisters.ServiceLister + serviceListerSynced cache.InformerSynced + + egressInformer cache.SharedIndexInformer + egressLister crdlistersv1b1.EgressLister + egressListerSynced cache.InformerSynced + + bgpPolicyInformer cache.SharedIndexInformer + bgpPolicyLister crdlistersv1a1.BGPPolicyLister + bgpPolicyListerSynced cache.InformerSynced + + endpointSliceLister discoverylisters.EndpointSliceLister + endpointSliceListerSynced cache.InformerSynced + + bgpPolicyBinding *nodeToBGPPolicyBinding + bgpPolicyBindingMutex sync.RWMutex + + bgpPolicyStates map[string]*bgpPolicyState + bgpPolicyStatesMutex sync.RWMutex + + k8sClient kubernetes.Interface + bgpPeerPasswordsSecret string + bgpPeerPasswords map[string]string + bgpPeerPasswordsMutex sync.RWMutex + + nodeName string + enabledIPv4 bool + enabledIPv6 bool + podIPv4CIDR string + podIPv6CIDR string + nodeIPv4Addr string + ipProtocols []utilnet.IPFamily + + egressEnabled bool + + queue workqueue.RateLimitingInterface +} + +func NewBGPPolicyController(nodeInformer coreinformers.NodeInformer, + serviceInformer coreinformers.ServiceInformer, + egressInformer crdinformersv1b1.EgressInformer, + bgpPolicyInformer crdinformersv1a1.BGPPolicyInformer, + endpointSliceInformer discoveryinformers.EndpointSliceInformer, + k8sClient kubernetes.Interface, + bgpPeerPasswordsSecret string, + nodeConfig *config.NodeConfig, + networkConfig *config.NetworkConfig) (*Controller, error) { + c := &Controller{ + nodeInformer: nodeInformer.Informer(), + nodeLister: nodeInformer.Lister(), + nodeListerSynced: nodeInformer.Informer().HasSynced, + serviceInformer: serviceInformer.Informer(), + serviceLister: serviceInformer.Lister(), + serviceListerSynced: serviceInformer.Informer().HasSynced, + egressInformer: egressInformer.Informer(), + egressLister: egressInformer.Lister(), + egressListerSynced: egressInformer.Informer().HasSynced, + bgpPolicyInformer: bgpPolicyInformer.Informer(), + bgpPolicyLister: bgpPolicyInformer.Lister(), + bgpPolicyListerSynced: bgpPolicyInformer.Informer().HasSynced, + endpointSliceLister: endpointSliceInformer.Lister(), + endpointSliceListerSynced: endpointSliceInformer.Informer().HasSynced, + bgpPolicyBinding: &nodeToBGPPolicyBinding{alternativeBPs: sets.Set[string]{}}, + bgpPolicyStates: make(map[string]*bgpPolicyState), + k8sClient: k8sClient, + bgpPeerPasswordsSecret: bgpPeerPasswordsSecret, + bgpPeerPasswords: make(map[string]string), + nodeName: nodeConfig.Name, + enabledIPv4: networkConfig.IPv4Enabled, + enabledIPv6: networkConfig.IPv6Enabled, + podIPv4CIDR: nodeConfig.PodIPv4CIDR.String(), + podIPv6CIDR: nodeConfig.PodIPv6CIDR.String(), + nodeIPv4Addr: nodeConfig.NodeIPv4Addr.IP.String(), + egressEnabled: features.DefaultFeatureGate.Enabled(features.Egress), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "bgpPolicyGroup"), + } + c.bgpPolicyInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.addBGPPolicy, + UpdateFunc: c.updateBGPPolicy, + DeleteFunc: c.deleteBGPPolicy, + }, + resyncPeriod, + ) + c.serviceInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.addService, + UpdateFunc: c.updateService, + DeleteFunc: c.deleteService, + }, + resyncPeriod, + ) + c.egressInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.addEgress, + UpdateFunc: c.updateEgress, + DeleteFunc: c.deleteEgress, + }, + resyncPeriod, + ) + c.nodeInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: nil, + UpdateFunc: c.updateNode, + DeleteFunc: nil, + }, + resyncPeriod, + ) + if c.enabledIPv4 { + c.ipProtocols = append(c.ipProtocols, protocolIPv4) + } + if c.enabledIPv6 { + c.ipProtocols = append(c.ipProtocols, protocolIPv6) + } + return c, nil +} + +// watchSecretChanges uses watch API directly to watch for the changes of the specific Secret. +func (c *Controller) watchSecretChanges(endCh <-chan struct{}) error { + ns := env.GetAntreaNamespace() + watcher, err := c.k8sClient.CoreV1().Secrets(ns).Watch(context.TODO(), metav1.SingleObject(metav1.ObjectMeta{ + Namespace: ns, + Name: c.bgpPeerPasswordsSecret, + })) + if err != nil { + return fmt.Errorf("failed to create Secret watcher: %v", err) + } + + ch := watcher.ResultChan() + defer watcher.Stop() + klog.InfoS("Starting watching Secret changes", "Secret", fmt.Sprintf("%s/%s", ns, c.bgpPeerPasswordsSecret)) + for { + select { + case event, ok := <-ch: + if !ok { + return nil + } + // Update BGP peer passwords. + klog.InfoS("Processing Secret event", "Secret", fmt.Sprintf("%s/%s", ns, c.bgpPeerPasswordsSecret)) + func() { + c.bgpPeerPasswordsMutex.Lock() + defer c.bgpPeerPasswordsMutex.Unlock() + + secretObj := event.Object.(*corev1.Secret) + c.bgpPeerPasswords = make(map[string]string) + for key, data := range secretObj.Data { + c.bgpPeerPasswords[key] = string(data) + } + }() + func() { + c.bgpPolicyBindingMutex.RLock() + defer c.bgpPolicyBindingMutex.RUnlock() + if c.bgpPolicyBinding.effectiveBP != "" { + c.queue.Add(c.bgpPolicyBinding.effectiveBP) + } + }() + case <-endCh: + return nil + } + } +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting", "controllerName", controllerName) + defer klog.InfoS("Shutting down", "controllerName", controllerName) + + if !cache.WaitForNamedCacheSync(controllerName, + stopCh, + c.nodeListerSynced, + c.serviceListerSynced, + c.egressListerSynced, + c.bgpPolicyListerSynced, + c.endpointSliceListerSynced) { + return + } + + go wait.NonSlidingUntil(func() { + if err := c.watchSecretChanges(stopCh); err != nil { + klog.ErrorS(err, "Watch Secret error", "secret", c.bgpPeerPasswordsSecret) + } + }, time.Second*10, stopCh) + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.worker, time.Second, stopCh) + } + + <-stopCh +} + +func (c *Controller) worker() { + for c.processNextWorkItem() { + } +} + +func (c *Controller) processNextWorkItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(obj) + + if key, ok := obj.(string); !ok { + // As the item in the work queue is actually invalid, we call Forget here else we'd go into a loop of attempting + // to process a work item that is invalid. This should not happen. + c.queue.Forget(obj) + klog.Errorf("Expected string in work queue but got %#v", obj) + return true + } else if err := c.syncBGPPolicy(key); err == nil { + // If no error occurs we Forget this item, so it does not get queued again until another change happens. + c.queue.Forget(key) + } else { + // Put the item back on the work queue to handle any transient errors. + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Syncing BGPPolicy failed, requeue", "BGPPolicy", key) + } + return true +} + +func (c *Controller) syncBGPPolicy(bpName string) error { + startTime := time.Now() + defer func() { + klog.V(2).InfoS("Finished syncing BGPPolicy", "BGPPolicy", bpName, "durationTime", time.Since(startTime)) + }() + + bp, err := c.bgpPolicyLister.Get(bpName) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + // If the BGPPolicy is deleted or not applied to the current Node anymore, do some cleanup. + if bp == nil || !c.matchedCurrentNode(bp) { + bpState, exists := c.getBGPPolicyState(bpName) + // If this is the effective BGPPolicy for the current Node, the BGPPolicy state should exist, and do some cleanup. + if exists { + // Stop the BGP server process on the current Node. + if err := bpState.bgpServer.Stop(); err != nil { + return err + } + // Delete the BGPPolicy state. + c.deleteBGPPolicyState(bpName) + } + + // Unbind the BGPPolicy from the current Node. + // If the BGPPolicy is the effective one for the current Node, try to pop a new effective BGPPolicy from the + // alternatives list to the queue. If the BGPPolicy is an alternative, remove it from the alternatives list. + if newEffectiveBP := c.unbindNodeFromBGPPolicy(bpName); newEffectiveBP != "" { + c.queue.Add(newEffectiveBP) + } + return nil + } + + // Bind the BGPPolicy to the current Node. If the BGPPolicy is not as the effective one, just return. + if asEffective := c.bindNodeToBGPPolicy(bpName); !asEffective { + klog.InfoS("The Node has already got an effective BGPPolicy. This is as an alternative", "BGPPolicy", bpName) + return nil + } + + // Retrieve the listen port, local AS number, and router ID from the current BGPPolicy to start the BGP server. + routerID, err := c.getRouterID() + if err != nil { + return err + } + listenPort := engine.DefaultBGPListenPort + if bp.Spec.ListenPort != nil { + listenPort = *bp.Spec.ListenPort + } + localASN := bp.Spec.LocalASN + + var needUpdateBGPServer bool + // Get the BGPPolicy state. + bpState, exists := c.getBGPPolicyState(bpName) + if !exists { + // If the BGPPolicy state doesn't exist, meaning the BGPPolicy has not been enforced on the current Node, create + // state for the BGPPolicy, and then start the BGP server. + bpState = c.newBGPPolicyState(bp) + needUpdateBGPServer = true + } else { + // Check the listen port, local AS number and routerID. If any of them have changed, then start a new BGP server + // and stop the stale one. + needUpdateBGPServer = bpState.listenPort != listenPort || + bpState.localASN != localASN || + bpState.routerID != routerID + } + + if needUpdateBGPServer { + // Start the new BGP server. + globalConfig := &engine.GlobalConfig{ + ASN: uint32(localASN), + RouterID: routerID, + ListenPort: listenPort, + } + extraLogFields := map[string]interface{}{ + "ASN": localASN, + "BGPPolicy": bpName, + } + bgpServer, err := newBGPServerFn(globalConfig, extraLogFields) + if err != nil { + return err + } + if err := bgpServer.Start(); err != nil { + return fmt.Errorf("failed to start BGP server: %w", err) + } + + // Stop the stale BGP server if it exists. + if bpState.bgpServer != nil { + if err := bpState.bgpServer.Stop(); err != nil { + klog.ErrorS(err, "Failed to stop stale BGP Server", "BGPPolicy", bpName) + } + } + + // Update the BGPPolicy state. + bpState.bgpServer = bgpServer + bpState.listenPort = listenPort + bpState.localASN = localASN + bpState.routerID = routerID + } + + // Reconcile BGP peers. + peers, peerConfigs, err := c.getPeers(bp.Spec.BGPPeers) + if err != nil { + return err + } + if err := c.reconcileBGPPeers(peers, peerConfigs, bpState, needUpdateBGPServer); err != nil { + return err + } + + // Reconcile advertisements. + routes, err := c.getRoutes(bp.Spec.Advertisements) + if err != nil { + return err + } + if err := c.reconcileRoutes(routes, bpState, needUpdateBGPServer); err != nil { + return err + } + + // Update the BGPPolicy state. + bpState.routes = routes + bpState.peers = peers + bpState.peerConfigs = peerConfigs + + return nil +} + +func getPeerConfigs(peers sets.Set[string], allPeerConfigs map[string]*engine.PeerConfig) []engine.PeerConfig { + peerConfigs := make([]engine.PeerConfig, 0, len(peers)) + for peer := range peers { + peerConfigs = append(peerConfigs, *allPeerConfigs[peer]) + } + return peerConfigs +} + +func (c *Controller) reconcileBGPPeers(curPeers map[utilnet.IPFamily]sets.Set[string], + curPeerConfigs map[string]*engine.PeerConfig, + bpState *bgpPolicyState, + bgpServerUpdated bool) error { + for ipProtocol, peers := range curPeers { + prePeers := bpState.peers[ipProtocol] + + var peersToAdd sets.Set[string] + if !bgpServerUpdated { + peersToAdd = peers.Difference(prePeers) + } else { + peersToAdd = peers + } + peerConfigsToAdd := getPeerConfigs(peersToAdd, curPeerConfigs) + if len(peerConfigsToAdd) != 0 { + if err := bpState.bgpServer.AddPeers(peerConfigsToAdd, ipProtocol); err != nil { + return err + } + } + + if !bgpServerUpdated { + peersToUpdate := sets.New[string]() + for peer := range prePeers.Intersection(peers) { + prevPeerInfo := bpState.peerConfigs[peer] + peerConfig := curPeerConfigs[peer] + if *prevPeerInfo != *peerConfig { + peersToUpdate.Insert(peer) + } + } + peerConfigsToUpdate := getPeerConfigs(peersToUpdate, curPeerConfigs) + if len(peerConfigsToUpdate) != 0 { + if err := bpState.bgpServer.UpdatePeers(peerConfigsToUpdate, ipProtocol); err != nil { + return err + } + } + peersToDelete := prePeers.Difference(peers) + prePeerConfigs := bpState.peerConfigs + peerConfigsToDelete := getPeerConfigs(peersToDelete, prePeerConfigs) + if len(peerConfigsToDelete) != 0 { + if err := bpState.bgpServer.RemovePeers(peerConfigsToDelete); err != nil { + return err + } + } + } + } + return nil +} + +func (c *Controller) reconcileRoutes(allRoutes map[utilnet.IPFamily]sets.Set[engine.Route], + bpState *bgpPolicyState, + bgpServerUpdated bool) error { + for ipProtocol, routes := range allRoutes { + prevRoutes := bpState.routes[ipProtocol] + + var routesToAdvertise sets.Set[engine.Route] + if !bgpServerUpdated { + routesToAdvertise = routes.Difference(prevRoutes) + } else { + routesToAdvertise = routes + } + if routesToAdvertise.Len() != 0 { + if err := bpState.bgpServer.AdvertiseRoutes(routesToAdvertise.UnsortedList(), ipProtocol); err != nil { + return err + } + } + + if !bgpServerUpdated { + routesToWithdraw := prevRoutes.Difference(routes) + if routesToWithdraw.Len() != 0 { + if err := bpState.bgpServer.WithdrawRoutes(routesToWithdraw.UnsortedList(), ipProtocol); err != nil { + return err + } + } + } + } + + return nil +} + +func (c *Controller) getBGPPolicyState(bpName string) (*bgpPolicyState, bool) { + c.bgpPolicyStatesMutex.RLock() + defer c.bgpPolicyStatesMutex.RUnlock() + state, exists := c.bgpPolicyStates[bpName] + return state, exists +} + +func (c *Controller) deleteBGPPolicyState(bpName string) { + c.bgpPolicyStatesMutex.Lock() + defer c.bgpPolicyStatesMutex.Unlock() + delete(c.bgpPolicyStates, bpName) +} + +func (c *Controller) getRouterID() (string, error) { + var routerID string + if !c.enabledIPv4 && c.enabledIPv6 { + nodeObj, _ := c.nodeLister.Get(c.nodeName) + var exists bool + if routerID, exists = nodeObj.GetAnnotations()[bgpRouteIDAnnotation]; !exists { + return "", fmt.Errorf("BGP routerID should be assigned by annotation manually when IPv6 is only enabled") + } + if !utilnet.IsIPv4String(routerID) { + return "", fmt.Errorf("BGP routerID should be an IPv4 address") + } + } else { + routerID = c.nodeIPv4Addr + } + return routerID, nil +} + +func (c *Controller) newBGPPolicyState(bp *v1alpha1.BGPPolicy) *bgpPolicyState { + c.bgpPolicyStatesMutex.Lock() + defer c.bgpPolicyStatesMutex.Unlock() + + routes := make(map[utilnet.IPFamily]sets.Set[engine.Route]) + peers := make(map[utilnet.IPFamily]sets.Set[string]) + peerConfigs := make(map[string]*engine.PeerConfig) + for _, ipProtocol := range c.ipProtocols { + routes[ipProtocol] = sets.New[engine.Route]() + peers[ipProtocol] = sets.New[string]() + } + state := &bgpPolicyState{ + routes: routes, + peers: peers, + peerConfigs: peerConfigs, + } + c.bgpPolicyStates[bp.Name] = state + return state +} + +func (c *Controller) getRoutes(advertisements v1alpha1.Advertisements) (map[utilnet.IPFamily]sets.Set[engine.Route], error) { + allRoutes := make(map[utilnet.IPFamily]sets.Set[engine.Route]) + for _, ipProtocol := range c.ipProtocols { + allRoutes[ipProtocol] = sets.New[engine.Route]() + } + + if advertisements.Service != nil { + if err := c.addServiceRoutes(advertisements.Service, allRoutes); err != nil { + return nil, err + } + } + if advertisements.Egress != nil && c.egressEnabled { + if err := c.addEgressRoutes(allRoutes); err != nil { + return nil, err + } + } + if advertisements.Pod != nil { + c.addPodRoutes(allRoutes) + } + + return allRoutes, nil +} + +func serviceIPTypesToAdvervise(serviceIPTypes []v1alpha1.ServiceIPType) sets.Set[v1alpha1.ServiceIPType] { + ipTypeMap := sets.New[v1alpha1.ServiceIPType]() + for _, ipType := range serviceIPTypes { + ipTypeMap.Insert(ipType) + } + return ipTypeMap +} + +func (c *Controller) addServiceRoutes(advertisement *v1alpha1.ServiceAdvertisement, allRoutes map[utilnet.IPFamily]sets.Set[engine.Route]) error { + ipTypeMap := serviceIPTypesToAdvervise(advertisement.IPTypes) + + services, err := c.serviceLister.List(labels.Everything()) + if err != nil { + return err + } + + var serviceIPs []string + for _, svc := range services { + internalLocal := svc.Spec.InternalTrafficPolicy != nil && *svc.Spec.InternalTrafficPolicy == corev1.ServiceInternalTrafficPolicyLocal + externalLocal := svc.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyLocal + var hasLocalEndpoints bool + if internalLocal || externalLocal { + var err error + hasLocalEndpoints, err = c.hasLocalEndpoints(svc) + if err != nil { + return err + } + } + if ipTypeMap.Has(v1alpha1.ServiceIPTypeClusterIP) { + if internalLocal && hasLocalEndpoints || !internalLocal { + for _, clusterIP := range svc.Spec.ClusterIPs { + serviceIPs = append(serviceIPs, clusterIP) + } + } + } + if ipTypeMap.Has(v1alpha1.ServiceIPTypeExternalIP) { + if externalLocal && hasLocalEndpoints || !externalLocal { + for _, externalIP := range svc.Spec.ExternalIPs { + serviceIPs = append(serviceIPs, externalIP) + } + } + } + if ipTypeMap.Has(v1alpha1.ServiceIPTypeLoadBalancerIP) && svc.Spec.Type == corev1.ServiceTypeLoadBalancer { + if externalLocal && hasLocalEndpoints || !externalLocal { + for _, ingressIP := range svc.Status.LoadBalancer.Ingress { + if ingressIP.IP != "" { + serviceIPs = append(serviceIPs, ingressIP.IP) + } + } + } + } + } + + for _, ip := range serviceIPs { + if c.enabledIPv4 && utilnet.IsIPv4String(ip) { + allRoutes[protocolIPv4].Insert(engine.Route{Prefix: ip + ipv4Suffix}) + } + if c.enabledIPv6 && utilnet.IsIPv6String(ip) { + allRoutes[protocolIPv6].Insert(engine.Route{Prefix: ip + ipv6Suffix}) + } + } + + return nil +} + +func (c *Controller) addEgressRoutes(allRoutes map[utilnet.IPFamily]sets.Set[engine.Route]) error { + egresses, err := c.egressLister.List(labels.Everything()) + if err != nil { + return err + } + + for _, eg := range egresses { + if eg.Status.EgressNode != c.nodeName { + continue + } + ip := eg.Status.EgressIP + if c.enabledIPv4 && utilnet.IsIPv4String(ip) { + allRoutes[protocolIPv4].Insert(engine.Route{Prefix: ip + ipv4Suffix}) + } + if c.enabledIPv6 && utilnet.IsIPv6String(ip) { + allRoutes[protocolIPv6].Insert(engine.Route{Prefix: ip + ipv6Suffix}) + } + } + + return nil +} + +func (c *Controller) addPodRoutes(allRoutes map[utilnet.IPFamily]sets.Set[engine.Route]) { + if c.enabledIPv4 { + allRoutes[protocolIPv4].Insert(engine.Route{Prefix: c.podIPv4CIDR}) + } + if c.enabledIPv6 { + allRoutes[protocolIPv6].Insert(engine.Route{Prefix: c.podIPv6CIDR}) + } +} + +func (c *Controller) hasLocalEndpoints(svc *corev1.Service) (bool, error) { + eps, err := c.endpointSliceLister.EndpointSlices(svc.GetNamespace()).Get(svc.GetName()) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + for _, ep := range eps.Endpoints { + if ep.NodeName != nil && *ep.NodeName == c.nodeName { + return true, nil + } + } + + return false, nil +} + +func (c *Controller) generateBGPPeerConfig(peer *v1alpha1.BGPPeer) (*engine.PeerConfig, error) { + bgpPeerConfig := &engine.PeerConfig{ + Address: peer.Address, + ASN: peer.ASN, + } + if peer.Port != nil { + bgpPeerConfig.Port = *peer.Port + } else { + bgpPeerConfig.Port = engine.DefaultBGPListenPort + } + if peer.GracefulRestartTimeSeconds != nil { + bgpPeerConfig.GracefulRestartTimeSeconds = *peer.GracefulRestartTimeSeconds + } else { + bgpPeerConfig.GracefulRestartTimeSeconds = engine.DefaultBGPGracefulRestartTimeSeconds + } + if peer.MultihopTTL != nil { + bgpPeerConfig.MultihopTTL = *peer.MultihopTTL + } else { + bgpPeerConfig.MultihopTTL = engine.DefaultBGPMultihopTTL + } + + bgpPeerKey := generateBGPPeerKey(peer.Address, peer.ASN) + c.bgpPeerPasswordsMutex.RLock() + defer c.bgpPeerPasswordsMutex.RUnlock() + if password, exists := c.bgpPeerPasswords[bgpPeerKey]; exists { + bgpPeerConfig.AuthPassword = password + } + return bgpPeerConfig, nil +} + +func (c *Controller) getPeers(allPeers []v1alpha1.BGPPeer) (map[utilnet.IPFamily]sets.Set[string], map[string]*engine.PeerConfig, error) { + peers := make(map[utilnet.IPFamily]sets.Set[string]) + peerConfigs := make(map[string]*engine.PeerConfig) + var ipv4Peers, ipv6Peers sets.Set[string] + if c.enabledIPv4 { + peers[utilnet.IPv4] = sets.New[string]() + ipv4Peers = peers[utilnet.IPv4] + } + if c.enabledIPv6 { + peers[utilnet.IPv6] = sets.New[string]() + ipv6Peers = peers[utilnet.IPv6] + } + for i := range allPeers { + key := generateBGPPeerKey(allPeers[i].Address, allPeers[i].ASN) + if c.enabledIPv4 && utilnet.IsIPv4String(allPeers[i].Address) { + ipv4Peers.Insert(key) + } + if c.enabledIPv6 && utilnet.IsIPv6String(allPeers[i].Address) { + ipv6Peers.Insert(key) + } + var err error + if peerConfigs[key], err = c.generateBGPPeerConfig(&allPeers[i]); err != nil { + return nil, nil, err + } + } + return peers, peerConfigs, nil +} + +func generateBGPPeerKey(address string, asn int32) string { + return fmt.Sprintf("%s-%d", address, asn) +} + +func (c *Controller) addBGPPolicy(obj interface{}) { + bp := obj.(*v1alpha1.BGPPolicy) + if !c.matchedCurrentNode(bp) { + return + } + klog.V(2).InfoS("Processing BGPPolicy ADD event", "BGPPolicy", klog.KObj(bp)) + c.queue.Add(bp.Name) +} + +func (c *Controller) updateBGPPolicy(oldObj, obj interface{}) { + oldBP := oldObj.(*v1alpha1.BGPPolicy) + bp := obj.(*v1alpha1.BGPPolicy) + if !c.matchedCurrentNode(bp) && !c.matchedCurrentNode(oldBP) { + return + } + if bp.GetGeneration() != oldBP.GetGeneration() { + klog.V(2).InfoS("Processing BGPPolicy UPDATE event", "BGPPolicy", klog.KObj(bp)) + c.queue.Add(bp.Name) + } +} + +func (c *Controller) deleteBGPPolicy(obj interface{}) { + bp := obj.(*v1alpha1.BGPPolicy) + if !c.matchedCurrentNode(bp) { + return + } + klog.V(2).InfoS("Processing BGPPolicy DELETE event", "BGPPolicy", klog.KObj(bp)) + c.queue.Add(bp.Name) +} + +func getIngressIPs(svc *corev1.Service) []string { + var ips []string + for _, ingress := range svc.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + ips = append(ips, ingress.IP) + } + } + return ips +} + +func (c *Controller) matchedCurrentNode(bp *v1alpha1.BGPPolicy) bool { + nodeSelector, _ := metav1.LabelSelectorAsSelector(&bp.Spec.NodeSelector) + node, _ := c.nodeLister.Get(c.nodeName) + return nodeSelector.Matches(labels.Set(node.GetLabels())) +} + +func (c *Controller) matchedNode(node *corev1.Node, bp *v1alpha1.BGPPolicy) bool { + nodeSel, _ := metav1.LabelSelectorAsSelector(&bp.Spec.NodeSelector) + if !nodeSel.Matches(labels.Set(node.Labels)) { + return false + } + return true +} + +func (c *Controller) filterAffectedBPsByNode(node *corev1.Node) sets.Set[string] { + affectedBPs := sets.New[string]() + allBPs, _ := c.bgpPolicyLister.List(labels.Everything()) + for _, bp := range allBPs { + if c.matchedNode(node, bp) { + affectedBPs.Insert(bp.GetName()) + } + } + return affectedBPs +} + +func (c *Controller) filterAffectedBPsByService(svc *corev1.Service) sets.Set[string] { + affectedBPs := sets.New[string]() + allBPs, _ := c.bgpPolicyLister.List(labels.Everything()) + for _, bp := range allBPs { + if bp.Spec.Advertisements.Service == nil { + continue + } + ipTypeMap := serviceIPTypesToAdvervise(bp.Spec.Advertisements.Service.IPTypes) + + if ipTypeMap.Has(v1alpha1.ServiceIPTypeClusterIP) && len(svc.Spec.ClusterIPs) != 0 || + ipTypeMap.Has(v1alpha1.ServiceIPTypeExternalIP) && len(svc.Spec.ExternalIPs) != 0 || + ipTypeMap.Has(v1alpha1.ServiceIPTypeLoadBalancerIP) && len(getIngressIPs(svc)) != 0 { + if c.matchedCurrentNode(bp) { + affectedBPs.Insert(bp.GetName()) + } + } + } + return affectedBPs +} + +func (c *Controller) filterAffectedBPsByEgress() sets.Set[string] { + affectedBPs := sets.New[string]() + allBPs, _ := c.bgpPolicyLister.List(labels.Everything()) + for _, bp := range allBPs { + if bp.Spec.Advertisements.Egress != nil && c.matchedCurrentNode(bp) { + affectedBPs.Insert(bp.GetName()) + } + } + return affectedBPs +} + +func (c *Controller) addService(obj interface{}) { + svc := obj.(*corev1.Service) + affectedBPs := c.filterAffectedBPsByService(svc) + if len(affectedBPs) == 0 { + return + } + klog.V(2).InfoS("Processing Service ADD event", "Service", klog.KObj(svc)) + for affectedBP := range affectedBPs { + c.queue.Add(affectedBP) + } +} + +func (c *Controller) updateService(oldObj, obj interface{}) { + oldSvc := oldObj.(*corev1.Service) + svc := obj.(*corev1.Service) + + if slices.Equal(oldSvc.Spec.ClusterIPs, svc.Spec.ClusterIPs) && + slices.Equal(oldSvc.Spec.ExternalIPs, svc.Spec.ExternalIPs) && + slices.Equal(getIngressIPs(oldSvc), getIngressIPs(svc)) { + return + } + oldAffectedBPs := c.filterAffectedBPsByService(oldSvc) + newAffectedBPs := c.filterAffectedBPsByService(svc) + affectedBPs := utilipset.MergeString(oldAffectedBPs, newAffectedBPs) + if len(affectedBPs) == 0 { + return + } + klog.V(2).InfoS("Processing Service UPDATE event", "Service", klog.KObj(svc)) + for affectedBP := range affectedBPs { + c.queue.Add(affectedBP) + } +} + +func (c *Controller) deleteService(obj interface{}) { + svc := obj.(*corev1.Service) + affectedBPs := c.filterAffectedBPsByService(svc) + if len(affectedBPs) == 0 { + return + } + klog.V(2).InfoS("Processing Service DELETE event", "Service", klog.KObj(svc)) + for affectedBP := range affectedBPs { + c.queue.Add(affectedBP) + } +} + +func (c *Controller) addEgress(obj interface{}) { + if !c.egressEnabled { + return + } + eg := obj.(*v1beta1.Egress) + if eg.Status.EgressNode != c.nodeName { + return + } + affectedBPs := c.filterAffectedBPsByEgress() + if len(affectedBPs) == 0 { + return + } + klog.V(2).InfoS("Processing Egress ADD event", "Egress", klog.KObj(eg)) + for affectedBP := range affectedBPs { + c.queue.Add(affectedBP) + } +} + +// TODO: if the update of status can be captured +func (c *Controller) updateEgress(oldObj, obj interface{}) { + if !c.egressEnabled { + return + } + oldEg := oldObj.(*v1beta1.Egress) + eg := obj.(*v1beta1.Egress) + if oldEg.Status.EgressNode != c.nodeName && eg.Status.EgressNode != c.nodeName { + return + } + if oldEg.Status.EgressIP == eg.Status.EgressIP { + return + } + affectedBPs := c.filterAffectedBPsByEgress() + if len(affectedBPs) == 0 { + return + } + klog.V(2).InfoS("Processing Egress UPDATE event", "Egress", klog.KObj(eg)) + for affectedBP := range affectedBPs { + c.queue.Add(affectedBP) + } +} + +func (c *Controller) deleteEgress(obj interface{}) { + if !c.egressEnabled { + return + } + eg := obj.(*v1beta1.Egress) + if eg.Status.EgressNode != c.nodeName { + return + } + affectedBPs := c.filterAffectedBPsByEgress() + if len(affectedBPs) == 0 { + return + } + klog.V(2).InfoS("Processing Egress DELETE event", "Service", klog.KObj(eg)) + for affectedBP := range affectedBPs { + c.queue.Add(affectedBP) + } +} + +func (c *Controller) updateNode(oldObj, obj interface{}) { + oldNode := oldObj.(*corev1.Node) + node := obj.(*corev1.Node) + if node.GetName() != c.nodeName { + return + } + if reflect.DeepEqual(node.GetLabels(), oldNode.GetLabels()) && + reflect.DeepEqual(node.GetAnnotations(), oldNode.GetAnnotations()) { + return + } + oldAffectedBPs := c.filterAffectedBPsByNode(oldNode) + newAffectedBPs := c.filterAffectedBPsByNode(node) + affectedBPs := utilipset.SymmetricDifferenceString(oldAffectedBPs, newAffectedBPs) + if len(affectedBPs) == 0 { + return + } + klog.V(2).InfoS("Processing Node UPDATE event", "Node", klog.KObj(node)) + for affectedBP := range affectedBPs { + c.queue.Add(affectedBP) + } +} + +func (c *Controller) bindNodeToBGPPolicy(bpName string) bool { + c.bgpPolicyBindingMutex.Lock() + defer c.bgpPolicyBindingMutex.Unlock() + + binding := c.bgpPolicyBinding + if binding.effectiveBP == "" { + binding.effectiveBP = bpName + return true + } + if binding.effectiveBP == bpName { + return true + } + + if !binding.alternativeBPs.Has(bpName) { + binding.alternativeBPs.Insert(bpName) + } + return false +} + +func (c *Controller) unbindNodeFromBGPPolicy(bpName string) string { + c.bgpPolicyBindingMutex.Lock() + defer c.bgpPolicyBindingMutex.Unlock() + + binding := c.bgpPolicyBinding + if binding.effectiveBP == bpName { + var popped bool + // Select a new effective BGPPolicy. + binding.effectiveBP, popped = binding.alternativeBPs.PopAny() + if !popped { + // Remove the binding information for the Node if there is no alternative BGPPolicies. + binding.effectiveBP = "" + return "" + } + return binding.effectiveBP + } + binding.alternativeBPs.Delete(bpName) + return "" +} diff --git a/pkg/agent/bgp/controller_test.go b/pkg/agent/bgp/controller_test.go new file mode 100644 index 00000000000..889bcbf62e8 --- /dev/null +++ b/pkg/agent/bgp/controller_test.go @@ -0,0 +1,1574 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bgp + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + netutils "k8s.io/utils/net" + "k8s.io/utils/ptr" + + "antrea.io/antrea/pkg/agent/bgp/engine" + bgptest "antrea.io/antrea/pkg/agent/bgp/engine/testing" + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" + crdv1b1 "antrea.io/antrea/pkg/apis/crd/v1beta1" + fakeversioned "antrea.io/antrea/pkg/client/clientset/versioned/fake" + crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" + "antrea.io/antrea/pkg/util/ip" +) + +const testAntreaBGPPasswordSecret = "antrea-bgp-passwords" // #nosec G101 + +var ( + podIPv4CIDR = ip.MustParseCIDR("10.10.0.0/24") + podIPv6CIDR = ip.MustParseCIDR("fec0:10:10::/64") + nodeIPv4Addr = ip.MustParseCIDR("192.168.77.100/24") + + testNodeConfig = &config.NodeConfig{ + PodIPv4CIDR: podIPv4CIDR, + PodIPv6CIDR: podIPv6CIDR, + NodeIPv4Addr: nodeIPv4Addr, + Name: localNodeName, + } + + peer1ASN = int32(65531) + peer1AuthPassword = "bpg-peer1" // #nosec G101 + ipv4Peer1Addr = "192.168.77.251" + ipv6Peer1Addr = "fec0::196:168:77:251" + ipv4Peer1 = generateBGPPeer(ipv4Peer1Addr, peer1ASN, 179, 120) + ipv6Peer1 = generateBGPPeer(ipv6Peer1Addr, peer1ASN, 179, 120) + ipv4Peer1Config = generateBGPPeerConfig(ipv4Peer1Addr, peer1ASN, 179, 120, peer1AuthPassword) + ipv6Peer1Config = generateBGPPeerConfig(ipv6Peer1Addr, peer1ASN, 179, 120, peer1AuthPassword) + + peer2ASN = int32(65532) + peer2AuthPassword = "bpg-peer2" // #nosec G101 + ipv4Peer2Addr = "192.168.77.252" + ipv6Peer2Addr = "fec0::196:168:77:252" + ipv4Peer2 = generateBGPPeer(ipv4Peer2Addr, peer2ASN, 179, 120) + ipv6Peer2 = generateBGPPeer(ipv6Peer2Addr, peer2ASN, 179, 120) + ipv4Peer2Config = generateBGPPeerConfig(ipv4Peer2Addr, peer2ASN, 179, 120, peer2AuthPassword) + ipv6Peer2Config = generateBGPPeerConfig(ipv6Peer2Addr, peer2ASN, 179, 120, peer2AuthPassword) + + updatedIPv4Peer2 = generateBGPPeer(ipv4Peer2Addr, peer2ASN, 179, 60) + updatedIPv6Peer2 = generateBGPPeer(ipv6Peer2Addr, peer2ASN, 179, 60) + updatedIPv4Peer2Config = generateBGPPeerConfig(ipv4Peer2Addr, peer2ASN, 179, 60, peer2AuthPassword) + updatedIPv6Peer2Config = generateBGPPeerConfig(ipv6Peer2Addr, peer2ASN, 179, 60, peer2AuthPassword) + + peer3ASN = int32(65533) + peer3AuthPassword = "bpg-peer3" // #nosec G101 + ipv4Peer3Addr = "192.168.77.253" + ipv6Peer3Addr = "fec0::196:168:77:253" + ipv4Peer3 = generateBGPPeer(ipv4Peer3Addr, peer3ASN, 179, 120) + ipv6Peer3 = generateBGPPeer(ipv6Peer3Addr, peer3ASN, 179, 120) + ipv4Peer3Config = generateBGPPeerConfig(ipv4Peer3Addr, peer3ASN, 179, 120, peer3AuthPassword) + ipv6Peer3Config = generateBGPPeerConfig(ipv6Peer3Addr, peer3ASN, 179, 120, peer3AuthPassword) + + nodeLabels1 = map[string]string{"node": "control-plane"} + nodeLabels2 = map[string]string{"os": "linux"} + nodeLabels3 = map[string]string{"node": "control-plane", "os": "linux"} + nodeAnnotations1 = map[string]string{bgpRouteIDAnnotation: "192.168.77.100"} + nodeAnnotations2 = map[string]string{bgpRouteIDAnnotation: "10.10.0.100"} + + localNodeName = "local" + node = generateNode(localNodeName, nodeLabels1, nodeAnnotations1) + + ipv4EgressIP1 = "192.168.77.200" + ipv6EgressIP1 = "fec0::192:168:77:200" + ipv4EgressIP2 = "192.168.77.201" + ipv6EgressIP2 = "fec0::192:168:77:2001" + + ipv4Egress1 = generateEgress("eg1-4", ipv4EgressIP1, localNodeName) + ipv6Egress1 = generateEgress("eg1-6", ipv6EgressIP1, localNodeName) + ipv4Egress2 = generateEgress("eg2-4", ipv4EgressIP2, "test-remote-node") + ipv6Egress2 = generateEgress("eg2-6", ipv6EgressIP2, "test-remote-node") + + bgpPolicyName1 = "bp-1" + bgpPolicyName2 = "bp-2" + bgpPolicyName3 = "bp-3" + + clusterIPv4 = "10.96.10.10" + externalIPv4 = "192.168.77.100" + loadBalancerIPv4 = "192.168.77.150" + endpointIPv4 = "10.10.0.10" + clusterIPv6 = "fec0::10:96:10:10" + externalIPv6 = "fec0::192:168:77:100" + loadBalancerIPv6 = "fec0::192:168:77:150" + endpointIPv6 = "fec0::10:10:0:10" + + ipv4ClusterIPName1 = "clusterip-4" + ipv4ClusterIPName2 = "clusterip-4-local" + ipv6ClusterIPName1 = "clusterip-6" + ipv6ClusterIPName2 = "clusterip-6-local" + ipv4LoadBalancerName = "loadbalancer-4" + ipv6LoadBalancerName = "loadbalancer-6" + + ipv4ClusterIP1 = generateService(ipv4ClusterIPName1, corev1.ServiceTypeClusterIP, clusterIPv4, externalIPv4, "", false) + ipv4ClusterIP1Eps = generateEndpointSlice(ipv4ClusterIPName1, false, false, endpointIPv4) + ipv4ClusterIP2 = generateService(ipv4ClusterIPName2, corev1.ServiceTypeClusterIP, clusterIPv4, externalIPv4, "", true) + ipv4ClusterIP2Eps = generateEndpointSlice(ipv4ClusterIPName2, false, false, endpointIPv4) + + ipv6ClusterIP1 = generateService(ipv6ClusterIPName1, corev1.ServiceTypeClusterIP, clusterIPv6, externalIPv6, "", false) + ipv6ClusterIP1Eps = generateEndpointSlice(ipv6ClusterIPName1, false, false, endpointIPv6) + ipv6ClusterIP2 = generateService(ipv6ClusterIPName2, corev1.ServiceTypeClusterIP, clusterIPv6, externalIPv6, "", true) + ipv6ClusterIP2Eps = generateEndpointSlice(ipv6ClusterIPName2, false, false, endpointIPv6) + + ipv4LoadBalancer = generateService(ipv4LoadBalancerName, corev1.ServiceTypeLoadBalancer, clusterIPv4, externalIPv4, loadBalancerIPv4, false) + ipv4LoadBalancerEps = generateEndpointSlice(ipv4LoadBalancerName, false, false, endpointIPv4) + ipv6LoadBalancer = generateService(ipv6LoadBalancerName, corev1.ServiceTypeLoadBalancer, clusterIPv6, externalIPv6, loadBalancerIPv6, false) + ipv6LoadBalancerEps = generateEndpointSlice(ipv6LoadBalancerName, false, false, endpointIPv6) + + bgpPeerPasswords = map[string]string{ + generateBGPPeerKey(ipv4Peer1Addr, peer1ASN): peer1AuthPassword, + generateBGPPeerKey(ipv6Peer1Addr, peer1ASN): peer1AuthPassword, + generateBGPPeerKey(ipv4Peer2Addr, peer2ASN): peer2AuthPassword, + generateBGPPeerKey(ipv6Peer2Addr, peer2ASN): peer2AuthPassword, + generateBGPPeerKey(ipv4Peer3Addr, peer3ASN): peer3AuthPassword, + generateBGPPeerKey(ipv6Peer3Addr, peer3ASN): peer3AuthPassword, + } +) + +type fakeController struct { + *Controller + mockController *gomock.Controller + mockBGPServer *bgptest.MockInterface + crdClient *fakeversioned.Clientset + crdInformerFactory crdinformers.SharedInformerFactory + client *fake.Clientset + informerFactory informers.SharedInformerFactory +} + +func (c *fakeController) startInformers(stopCh chan struct{}) { + c.informerFactory.Start(stopCh) + c.informerFactory.WaitForCacheSync(stopCh) + c.crdInformerFactory.Start(stopCh) + c.crdInformerFactory.WaitForCacheSync(stopCh) +} + +func newFakeController(t *testing.T, objects []runtime.Object, crdObjects []runtime.Object, ipv4Enabled, ipv6Enabled bool) *fakeController { + ctrl := gomock.NewController(t) + mockBGPServer := bgptest.NewMockInterface(ctrl) + + client := fake.NewSimpleClientset(objects...) + crdClient := fakeversioned.NewSimpleClientset(crdObjects...) + + crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, 0) + informerFactory := informers.NewSharedInformerFactory(client, 0) + + nodeInformer := informerFactory.Core().V1().Nodes() + serviceInformer := informerFactory.Core().V1().Services() + egressInformer := crdInformerFactory.Crd().V1beta1().Egresses() + endpointSliceInformer := informerFactory.Discovery().V1().EndpointSlices() + bgpPolicyInformer := crdInformerFactory.Crd().V1alpha1().BGPPolicies() + + bgpController, _ := NewBGPPolicyController(nodeInformer, + serviceInformer, + egressInformer, + bgpPolicyInformer, + endpointSliceInformer, + client, + testAntreaBGPPasswordSecret, + testNodeConfig, + &config.NetworkConfig{ + IPv4Enabled: ipv4Enabled, + IPv6Enabled: ipv6Enabled, + }) + bgpController.egressEnabled = true + + return &fakeController{ + Controller: bgpController, + mockController: ctrl, + mockBGPServer: mockBGPServer, + crdClient: crdClient, + crdInformerFactory: crdInformerFactory, + client: client, + informerFactory: informerFactory, + } +} + +func mockNewBGPServer(mockBGPServer engine.Interface) func() { + originBGPServerFn := newBGPServerFn + newBGPServerFn = func(_ *engine.GlobalConfig, _ map[string]interface{}) (engine.Interface, error) { + return mockBGPServer, nil + } + return func() { + newBGPServerFn = originBGPServerFn + } +} + +func TestBGPPolicyAdd(t *testing.T) { + testCases := []struct { + name string + ipv4Enabled bool + ipv6Enabled bool + bp *v1alpha1.BGPPolicy + objects []runtime.Object + crdObjects []runtime.Object + expectedBGPPolicyState *bgpPolicyState + existingEffectiveBP string + expectedEffectiveBP string + expectedAlternativeBPs sets.Set[string] + expectedCalls func(mockBGPServer *bgptest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4, as effective BGPPolicy, advertise ClusterIP", + ipv4Enabled: true, + bp: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + true, + false, + true, + true, + false, + []v1alpha1.BGPPeer{ipv4Peer1}), + objects: []runtime.Object{ipv4ClusterIP1, ipv4ClusterIP1Eps, node}, + expectedBGPPolicyState: generateBGPPolicyState(true, + false, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(clusterIPv4)}, + []*engine.PeerConfig{ipv4Peer1Config}, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv4Peer1Config}, protocolIPv4).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: ipStrToPrefix(clusterIPv4)}}, protocolIPv4).Times(1) + }, + }, + { + name: "IPv6, as effective BGPPolicy, advertise ExternalIP", + ipv6Enabled: true, + bp: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + false, + true, + true, + true, + false, + []v1alpha1.BGPPeer{ipv6Peer1}), + objects: []runtime.Object{ipv6ClusterIP1, ipv6ClusterIP1Eps, node}, + expectedBGPPolicyState: generateBGPPolicyState(false, + true, + 179, + 65000, + "192.168.77.100", + []string{ipStrToPrefix(externalIPv6)}, + []*engine.PeerConfig{ipv6Peer1Config}, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv6Peer1Config}, protocolIPv6).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: ipStrToPrefix(externalIPv6)}}, protocolIPv6).Times(1) + }, + }, + { + name: "IPv4 & IPv6, as effective BGPPolicy, advertise LoadBalancerIP", + ipv4Enabled: true, + ipv6Enabled: true, + bp: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + false, + false, + true, + false, + false, + []v1alpha1.BGPPeer{ipv4Peer1, ipv6Peer1}), + objects: []runtime.Object{ipv4LoadBalancer, ipv4LoadBalancerEps, ipv6LoadBalancer, ipv6LoadBalancerEps, node}, + expectedBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(loadBalancerIPv4), ipStrToPrefix(loadBalancerIPv6)}, + []*engine.PeerConfig{ipv4Peer1Config, ipv6Peer1Config}, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv4Peer1Config}, protocolIPv4).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: ipStrToPrefix(loadBalancerIPv4)}}, protocolIPv4).Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv6Peer1Config}, protocolIPv6).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: ipStrToPrefix(loadBalancerIPv6)}}, protocolIPv6).Times(1) + }, + }, + { + name: "IPv4, as effective BGPPolicy, advertise EgressIP", + ipv4Enabled: true, + bp: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + true, + true, + true, + true, + false, + []v1alpha1.BGPPeer{ipv4Peer1}), + objects: []runtime.Object{node}, + crdObjects: []runtime.Object{ipv4Egress1, ipv4Egress2}, + expectedBGPPolicyState: generateBGPPolicyState(true, + false, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(ipv4EgressIP1)}, + []*engine.PeerConfig{ipv4Peer1Config}, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv4Peer1Config}, protocolIPv4).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: ipStrToPrefix(ipv4EgressIP1)}}, protocolIPv4).Times(1) + }, + }, + { + name: "IPv6, as effective BGPPolicy, advertise Pod CIDR", + ipv6Enabled: true, + bp: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + true, + true, + true, + true, + true, + []v1alpha1.BGPPeer{ipv6Peer1}), + objects: []runtime.Object{node}, + expectedBGPPolicyState: generateBGPPolicyState(false, + true, + 179, + 65000, + "192.168.77.100", + []string{podIPv6CIDR.String()}, + []*engine.PeerConfig{ipv6Peer1Config}, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv6Peer1Config}, protocolIPv6).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: podIPv6CIDR.String()}}, protocolIPv6).Times(1) + }, + }, + { + name: "IPv4 & IPv6, as effective BGPPolicy, not advertise any Service IP due to no local Endpoint", + ipv4Enabled: true, + ipv6Enabled: true, + bp: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 1179, + 65001, + true, + true, + true, + false, + false, + []v1alpha1.BGPPeer{ipv4Peer1, ipv6Peer1}), + objects: []runtime.Object{ipv4ClusterIP2, ipv4ClusterIP2Eps, ipv6ClusterIP2, ipv6ClusterIP2Eps, node}, + expectedBGPPolicyState: generateBGPPolicyState(true, + true, + 1179, + 65001, + nodeIPv4Addr.IP.String(), + nil, + []*engine.PeerConfig{ipv4Peer1Config, ipv6Peer1Config}, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv4Peer1Config}, protocolIPv4).Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv6Peer1Config}, protocolIPv6).Times(1) + }, + }, + { + name: "IPv4, as alternative BGPPolicy", + ipv4Enabled: true, + bp: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + true, + false, + false, + false, + false, + []v1alpha1.BGPPeer{ipv4Peer1}), + existingEffectiveBP: bgpPolicyName2, + objects: []runtime.Object{ipv4ClusterIP1, ipv4ClusterIP1Eps, node}, + expectedEffectiveBP: bgpPolicyName2, + expectedAlternativeBPs: sets.New[string](bgpPolicyName1), + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + c := newFakeController(t, tt.objects, append(tt.crdObjects, tt.bp), tt.ipv4Enabled, tt.ipv6Enabled) + defer mockNewBGPServer(c.mockBGPServer)() + + stopCh := make(chan struct{}) + defer close(stopCh) + c.startInformers(stopCh) + + // Ignore the test BGPPolicy ADD event. + waitEvents(t, 1, c) + + // Fake the effective BGPPolicy. + c.bgpPolicyBinding.effectiveBP = tt.existingEffectiveBP + // Fake the passwords of BGP peers. + c.bgpPeerPasswords = bgpPeerPasswords + + if tt.expectedCalls != nil { + tt.expectedCalls(c.mockBGPServer.EXPECT()) + } + assert.NoError(t, c.syncBGPPolicy(tt.bp.Name)) + + assert.Equal(t, tt.expectedEffectiveBP, c.bgpPolicyBinding.effectiveBP) + assert.Equal(t, tt.expectedAlternativeBPs, c.bgpPolicyBinding.alternativeBPs) + checkBGPPolicyState(t, tt.expectedBGPPolicyState, c.bgpPolicyStates[tt.bp.GetName()]) + }) + } +} + +func TestBGPPolicyUpdate(t *testing.T) { + bp := generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + true, + false, + true, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, + ipv4Peer2, + ipv6Peer1, + ipv6Peer2, + }) + objects := []runtime.Object{ + ipv4ClusterIP2, + ipv4ClusterIP2Eps, + ipv6ClusterIP2, + ipv6ClusterIP2Eps, + ipv4LoadBalancer, + ipv4LoadBalancerEps, + ipv6LoadBalancer, + ipv6LoadBalancerEps, + node, + } + crdObjects := []runtime.Object{ipv4Egress1, + ipv4Egress2, + ipv6Egress1, + ipv6Egress2, + bp, + } + testCases := []struct { + name string + asAlternative bool + updatedBP *v1alpha1.BGPPolicy + expectedBGPPolicyState *bgpPolicyState + expectedEffectiveBP string + expectedAlternativeBPs sets.Set[string] + expectedCalls func(mockBGPServer *bgptest.MockInterfaceMockRecorder) + }{ + { + name: "As effective, update NodeSelector", + updatedBP: generateBGPPolicy(bgpPolicyName1, + nodeLabels2, + 179, + 65000, + true, + false, + true, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, + ipv4Peer2, + ipv6Peer1, + ipv6Peer2, + }), + expectedEffectiveBP: "", + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Stop().Times(1) + }, + }, + { + name: "As effective, update Advertisements", + updatedBP: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + false, + true, + false, + true, + false, + []v1alpha1.BGPPeer{ipv4Peer1, + ipv4Peer2, + ipv6Peer1, + ipv6Peer2, + }), + expectedBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(externalIPv4), + ipStrToPrefix(externalIPv6), + ipStrToPrefix(ipv4EgressIP1), + ipStrToPrefix(ipv6EgressIP1), + }, + []*engine.PeerConfig{ipv4Peer1Config, + ipv6Peer1Config, + ipv4Peer2Config, + ipv6Peer2Config, + }, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(externalIPv4)}, {Prefix: ipStrToPrefix(ipv4EgressIP1)}}), protocolIPv4).Times(1) + mockBGPServer.AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(externalIPv6)}, {Prefix: ipStrToPrefix(ipv6EgressIP1)}}), protocolIPv6).Times(1) + mockBGPServer.WithdrawRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(clusterIPv4)}, {Prefix: ipStrToPrefix(loadBalancerIPv4)}, {Prefix: podIPv4CIDR.String()}}), protocolIPv4).Times(1) + mockBGPServer.WithdrawRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(clusterIPv6)}, {Prefix: ipStrToPrefix(loadBalancerIPv6)}, {Prefix: podIPv6CIDR.String()}}), protocolIPv6).Times(1) + }, + }, + { + name: "As effective, update LocalASN and Advertisements", + updatedBP: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65001, + false, + true, + false, + true, + false, + []v1alpha1.BGPPeer{ipv4Peer1, + ipv4Peer2, + ipv6Peer1, + ipv6Peer2, + }), + expectedBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65001, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(externalIPv4), + ipStrToPrefix(externalIPv6), + ipStrToPrefix(ipv4EgressIP1), + ipStrToPrefix(ipv6EgressIP1), + }, + []*engine.PeerConfig{ipv4Peer1Config, + ipv6Peer1Config, + ipv4Peer2Config, + ipv6Peer2Config, + }, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.Stop().Times(1) + mockBGPServer.AddPeers(gomock.InAnyOrder([]engine.PeerConfig{*ipv4Peer1Config, *ipv4Peer2Config}), protocolIPv4).Times(1) + mockBGPServer.AddPeers(gomock.InAnyOrder([]engine.PeerConfig{*ipv6Peer1Config, *ipv6Peer2Config}), protocolIPv6).Times(1) + mockBGPServer.AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(externalIPv4)}, {Prefix: ipStrToPrefix(ipv4EgressIP1)}}), protocolIPv4).Times(1) + mockBGPServer.AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(externalIPv6)}, {Prefix: ipStrToPrefix(ipv6EgressIP1)}}), protocolIPv6).Times(1) + }, + }, + { + name: "As effective, update ListenPort", + updatedBP: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 1179, + 65000, + true, + false, + true, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, + ipv4Peer2, + ipv6Peer1, + ipv6Peer2, + }), + expectedBGPPolicyState: generateBGPPolicyState(true, + true, + 1179, + 65000, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(clusterIPv4), + ipStrToPrefix(clusterIPv6), + ipStrToPrefix(loadBalancerIPv4), + ipStrToPrefix(loadBalancerIPv6), + podIPv4CIDR.String(), + podIPv6CIDR.String(), + }, + []*engine.PeerConfig{ipv4Peer1Config, + ipv6Peer1Config, + ipv4Peer2Config, + ipv6Peer2Config, + }, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.Stop().Times(1) + mockBGPServer.AddPeers(gomock.InAnyOrder([]engine.PeerConfig{*ipv4Peer1Config, *ipv4Peer2Config}), protocolIPv4).Times(1) + mockBGPServer.AddPeers(gomock.InAnyOrder([]engine.PeerConfig{*ipv6Peer1Config, *ipv6Peer2Config}), protocolIPv6).Times(1) + mockBGPServer.AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(clusterIPv4)}, {Prefix: ipStrToPrefix(loadBalancerIPv4)}, {Prefix: podIPv4CIDR.String()}}), protocolIPv4).Times(1) + mockBGPServer.AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: ipStrToPrefix(clusterIPv6)}, {Prefix: ipStrToPrefix(loadBalancerIPv6)}, {Prefix: podIPv6CIDR.String()}}), protocolIPv6).Times(1) + }, + }, + { + name: "As effective, update BGPPeers", + updatedBP: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + true, + false, + true, + false, + true, + []v1alpha1.BGPPeer{updatedIPv4Peer2, + updatedIPv6Peer2, + ipv4Peer3, + ipv6Peer3}), + expectedBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(clusterIPv4), + ipStrToPrefix(clusterIPv6), + ipStrToPrefix(loadBalancerIPv4), + ipStrToPrefix(loadBalancerIPv6), + podIPv4CIDR.String(), + podIPv6CIDR.String(), + }, + []*engine.PeerConfig{updatedIPv4Peer2Config, + updatedIPv6Peer2Config, + ipv4Peer3Config, + ipv6Peer3Config, + }, + ), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv4Peer3Config}, protocolIPv4).Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv6Peer3Config}, protocolIPv6).Times(1) + mockBGPServer.RemovePeers([]engine.PeerConfig{*ipv4Peer1Config}).Times(1) + mockBGPServer.RemovePeers([]engine.PeerConfig{*ipv6Peer1Config}).Times(1) + mockBGPServer.UpdatePeers([]engine.PeerConfig{*updatedIPv4Peer2Config}, protocolIPv4).Times(1) + mockBGPServer.UpdatePeers([]engine.PeerConfig{*updatedIPv6Peer2Config}, protocolIPv6).Times(1) + }, + }, + { + name: "As alternative, update NodeSelector", + asAlternative: true, + updatedBP: generateBGPPolicy(bgpPolicyName1, + nodeLabels2, + 179, + 65000, + true, + false, + true, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, + ipv4Peer2, + ipv6Peer1, + ipv6Peer2, + }), + expectedEffectiveBP: bgpPolicyName2, + expectedAlternativeBPs: sets.New[string](), + }, + { + name: "As alternative, update Advertisements, LocalASN, ListenPort and BGPPeers", + asAlternative: true, + updatedBP: generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 1179, + 65000, + true, + true, + true, + true, + true, + []v1alpha1.BGPPeer{updatedIPv4Peer2, + ipv4Peer3, + updatedIPv6Peer2, + ipv6Peer3, + }), + expectedEffectiveBP: bgpPolicyName2, + expectedAlternativeBPs: sets.New[string](bgpPolicyName1), + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + c := newFakeController(t, objects, crdObjects, true, true) + defer mockNewBGPServer(c.mockBGPServer)() + + stopCh := make(chan struct{}) + defer close(stopCh) + c.startInformers(stopCh) + + // Ignore the test BGPPolicy ADD event. + waitEvents(t, 1, c) + item, _ := c.queue.Get() + c.queue.Done(item) + + // Fake the BGPPolicy state, effective BGPPolicy and alternative BGPPolicies. + if !tt.asAlternative { + c.bgpPolicyStates[bp.Name] = generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{ipStrToPrefix(clusterIPv4), + ipStrToPrefix(clusterIPv6), + ipStrToPrefix(loadBalancerIPv4), + ipStrToPrefix(loadBalancerIPv6), + podIPv4CIDR.String(), + podIPv6CIDR.String(), + }, + []*engine.PeerConfig{ipv4Peer1Config, + ipv6Peer1Config, + ipv4Peer2Config, + ipv6Peer2Config, + }, + ) + c.bgpPolicyStates[bp.Name].bgpServer = c.mockBGPServer + c.bgpPolicyBinding.effectiveBP = bp.Name + c.bgpPolicyBinding.alternativeBPs = sets.New[string]() + } else { + c.bgpPolicyStates[bgpPolicyName2] = &bgpPolicyState{bgpServer: c.mockBGPServer} + c.bgpPolicyBinding.effectiveBP = bgpPolicyName2 + c.bgpPolicyBinding.alternativeBPs = sets.New[string](bp.Name) + } + // Fake the passwords of BGP peers. + c.bgpPeerPasswords = bgpPeerPasswords + + tt.updatedBP.Generation += 1 + _, err := c.crdClient.CrdV1alpha1().BGPPolicies().Update(context.TODO(), tt.updatedBP, metav1.UpdateOptions{}) + require.NoError(t, err) + waitEvents(t, 1, c) + + if tt.expectedCalls != nil { + tt.expectedCalls(c.mockBGPServer.EXPECT()) + } + assert.NoError(t, c.syncBGPPolicy(tt.updatedBP.Name)) + + assert.Equal(t, tt.expectedEffectiveBP, c.bgpPolicyBinding.effectiveBP) + assert.Equal(t, tt.expectedAlternativeBPs, c.bgpPolicyBinding.alternativeBPs) + checkBGPPolicyState(t, tt.expectedBGPPolicyState, c.bgpPolicyStates[tt.updatedBP.GetName()]) + }) + } +} + +func TestBGPPolicyDelete(t *testing.T) { + testCases := []struct { + name string + asAlternative bool + bpName string + existingEffectiveBP string + existingAlternativeBPs sets.Set[string] + expectedEffectiveBP string + expectedAlternativeBPs sets.Set[string] + expectedCalls func(mockBGPServer *bgptest.MockInterfaceMockRecorder) + }{ + { + name: "As effective", + bpName: bgpPolicyName1, + existingEffectiveBP: bgpPolicyName1, + existingAlternativeBPs: sets.New[string](bgpPolicyName2), + expectedEffectiveBP: bgpPolicyName2, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Stop().Times(1) + }, + }, + { + name: "As alternative", + bpName: bgpPolicyName1, + existingEffectiveBP: bgpPolicyName2, + existingAlternativeBPs: sets.New[string](bgpPolicyName1, bgpPolicyName3), + expectedEffectiveBP: bgpPolicyName2, + expectedAlternativeBPs: sets.New[string](bgpPolicyName3), + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + bp := generateBGPPolicy(tt.bpName, + nodeLabels1, + 179, + 65000, + true, + false, + true, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, + ipv4Peer2, + ipv6Peer1, + ipv6Peer2, + }) + c := newFakeController(t, []runtime.Object{node}, []runtime.Object{bp}, true, true) + defer mockNewBGPServer(c.mockBGPServer)() + + stopCh := make(chan struct{}) + defer close(stopCh) + + c.startInformers(stopCh) + + // Ignore the BGPPolicy ADD events for the test BGPPolicy. + waitEvents(t, 1, c) + + // Fake the BGPPolicy state, effective BGPPolicy and alternative BGPPolicies. + c.bgpPolicyBinding.effectiveBP = tt.existingEffectiveBP + c.bgpPolicyBinding.alternativeBPs = tt.existingAlternativeBPs + c.bgpPolicyStates[tt.existingEffectiveBP] = &bgpPolicyState{ + bgpServer: c.mockBGPServer, + } + // Fake the passwords of BGP peers. + c.bgpPeerPasswords = bgpPeerPasswords + + err := c.crdClient.CrdV1alpha1().BGPPolicies().Delete(context.TODO(), tt.bpName, metav1.DeleteOptions{}) + require.NoError(t, err) + waitEvents(t, 1, c) + + if tt.expectedCalls != nil { + tt.expectedCalls(c.mockBGPServer.EXPECT()) + } + assert.NoError(t, c.syncBGPPolicy(tt.bpName)) + + assert.Equal(t, tt.expectedEffectiveBP, c.bgpPolicyBinding.effectiveBP) + assert.Equal(t, tt.expectedAlternativeBPs, c.bgpPolicyBinding.alternativeBPs) + checkBGPPolicyState(t, nil, c.bgpPolicyStates[tt.bpName]) + }) + } +} + +func TestNodeUpdate(t *testing.T) { + bp1 := generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + false, + false, + false, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, ipv6Peer1}) + bp2 := generateBGPPolicy(bgpPolicyName2, + nodeLabels2, + 179, + 65000, + false, + false, + false, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, ipv6Peer1}) + bp3 := generateBGPPolicy(bgpPolicyName3, + nodeLabels3, + 179, + 65000, + false, + false, + false, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, ipv6Peer1}) + crdObjects := []runtime.Object{bp1, + bp2, + bp3, + } + testCases := []struct { + name string + ipv6Only bool + node *corev1.Node + updatedNode *corev1.Node + eventsTriggeredByStart int + eventsTriggeredByUpdate int + existingBGPPolicyState *bgpPolicyState + existingEffectiveBP string + existingAlternativeBPs sets.Set[string] + expectedEffectiveBP string + expectedAlternativeBPs sets.Set[string] + expectedCalls func(mockBGPServer *bgptest.MockInterfaceMockRecorder) + }{ + { + name: "Update labels, a BGPPolicy is added to alternatives", + node: generateNode(localNodeName, nodeLabels1, nodeAnnotations1), + updatedNode: generateNode(localNodeName, nodeLabels3, nodeAnnotations1), + eventsTriggeredByStart: 1, + eventsTriggeredByUpdate: 3, + existingBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{podIPv4CIDR.String(), podIPv6CIDR.String()}, + []*engine.PeerConfig{ipv4Peer1Config, ipv6Peer1Config}, + ), + existingEffectiveBP: bgpPolicyName1, + existingAlternativeBPs: sets.New[string](), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](bgpPolicyName2, bgpPolicyName3), + }, + { + name: "Update labels, a BGPPolicy is removed from alternatives", + node: generateNode(localNodeName, nodeLabels3, nodeAnnotations1), + updatedNode: generateNode(localNodeName, nodeLabels1, nodeAnnotations1), + eventsTriggeredByStart: 3, + eventsTriggeredByUpdate: 3, + existingBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{podIPv4CIDR.String(), podIPv6CIDR.String()}, + []*engine.PeerConfig{ipv4Peer1Config, ipv6Peer1Config}, + ), + existingEffectiveBP: bgpPolicyName1, + existingAlternativeBPs: sets.New[string](bgpPolicyName2, bgpPolicyName3), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + }, + { + name: "Update labels, effective BGPPolicy is updated to another one", + node: generateNode(localNodeName, nodeLabels1, nodeAnnotations1), + updatedNode: generateNode(localNodeName, nodeLabels2, nodeAnnotations1), + eventsTriggeredByStart: 1, + eventsTriggeredByUpdate: 2, + existingBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{podIPv4CIDR.String(), podIPv6CIDR.String()}, + []*engine.PeerConfig{ipv4Peer1Config, ipv6Peer1Config}, + ), + existingEffectiveBP: bgpPolicyName1, + existingAlternativeBPs: sets.New[string](), + expectedEffectiveBP: bgpPolicyName2, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.Stop().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv4Peer1Config}, protocolIPv4).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: podIPv4CIDR.String()}}, protocolIPv4).Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv6Peer1Config}, protocolIPv6).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: podIPv6CIDR.String()}}, protocolIPv6).Times(1) + }, + }, + { + name: "Update labels, effective BGPPolicy is updated to empty", + node: generateNode(localNodeName, nodeLabels1, nodeAnnotations1), + updatedNode: generateNode(localNodeName, nil, nodeAnnotations1), + eventsTriggeredByStart: 1, + eventsTriggeredByUpdate: 1, + existingBGPPolicyState: generateBGPPolicyState(true, + true, + 179, + 65000, + nodeIPv4Addr.IP.String(), + []string{podIPv4CIDR.String(), podIPv6CIDR.String()}, + []*engine.PeerConfig{ipv4Peer1Config, ipv6Peer1Config}, + ), + existingEffectiveBP: bgpPolicyName1, + existingAlternativeBPs: sets.New[string](), + expectedEffectiveBP: "", + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Stop().Times(1) + }, + }, + { + name: "IPv6 only, update annotations, effective BGPPolicy router ID is updated", + ipv6Only: true, + node: generateNode(localNodeName, nodeLabels1, nodeAnnotations1), + updatedNode: generateNode(localNodeName, nodeLabels1, nodeAnnotations2), + eventsTriggeredByStart: 1, + eventsTriggeredByUpdate: 1, + existingBGPPolicyState: generateBGPPolicyState(false, + true, + 179, + 65000, + "192.168.77.100", + []string{podIPv6CIDR.String()}, + []*engine.PeerConfig{ipv6Peer1Config}, + ), + existingEffectiveBP: bgpPolicyName1, + existingAlternativeBPs: sets.New[string](), + expectedEffectiveBP: bgpPolicyName1, + expectedAlternativeBPs: sets.New[string](), + expectedCalls: func(mockBGPServer *bgptest.MockInterfaceMockRecorder) { + mockBGPServer.Start().Times(1) + mockBGPServer.Stop().Times(1) + mockBGPServer.AddPeers([]engine.PeerConfig{*ipv6Peer1Config}, protocolIPv6).Times(1) + mockBGPServer.AdvertiseRoutes([]engine.Route{{Prefix: podIPv6CIDR.String()}}, protocolIPv6).Times(1) + }, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + c := newFakeController(t, []runtime.Object{tt.node}, crdObjects, !tt.ipv6Only, true) + defer mockNewBGPServer(c.mockBGPServer)() + + stopCh := make(chan struct{}) + defer close(stopCh) + + c.startInformers(stopCh) + + // Ignore the BGPPolicy ADD events for the test BGPPolicies. + waitEvents(t, tt.eventsTriggeredByStart, c) + + // Fake the BGPPolicy state, effective BGPPolicy and alternative BGPPolicies. + c.bgpPolicyBinding.effectiveBP = tt.existingEffectiveBP + c.bgpPolicyBinding.alternativeBPs = tt.existingAlternativeBPs + c.bgpPolicyStates[tt.existingEffectiveBP] = tt.existingBGPPolicyState + c.bgpPolicyStates[tt.existingEffectiveBP].bgpServer = c.mockBGPServer + // Fake the passwords of BGP peers. + c.bgpPeerPasswords = bgpPeerPasswords + + _, err := c.client.CoreV1().Nodes().Update(context.TODO(), tt.updatedNode, metav1.UpdateOptions{}) + require.NoError(t, err) + + if tt.expectedCalls != nil { + tt.expectedCalls(c.mockBGPServer.EXPECT()) + } + if tt.eventsTriggeredByUpdate > 0 { + waitEvents(t, tt.eventsTriggeredByUpdate, c) + for i := 0; i < tt.eventsTriggeredByUpdate; i++ { + item, _ := c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + } + } + + assert.Equal(t, tt.expectedEffectiveBP, c.bgpPolicyBinding.effectiveBP) + assert.Equal(t, tt.expectedAlternativeBPs, c.bgpPolicyBinding.alternativeBPs) + }) + } +} + +func TestServiceLifecycle(t *testing.T) { + bp := generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + true, + true, + true, + false, + false, + []v1alpha1.BGPPeer{ipv4Peer1}) + c := newFakeController(t, []runtime.Object{node}, []runtime.Object{bp}, true, false) + defer mockNewBGPServer(c.mockBGPServer)() + mockBGPServer := c.mockBGPServer + + stopCh := make(chan struct{}) + defer close(stopCh) + + c.startInformers(stopCh) + + // Fake the passwords of BGP peers. + c.bgpPeerPasswords = bgpPeerPasswords + + // Initialize the test BGPPolicy. + waitEvents(t, 1, c) + mockBGPServer.EXPECT().Start().Times(1) + mockBGPServer.EXPECT().AddPeers([]engine.PeerConfig{*ipv4Peer1Config}, protocolIPv4).Times(1) + item, _ := c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + + // Create a Service. + loadBalancer := generateService(ipv4LoadBalancerName, corev1.ServiceTypeLoadBalancer, "10.96.10.10", "192.168.77.100", "192.168.77.150", false) + _, err := c.client.CoreV1().Services("default").Create(context.TODO(), loadBalancer, metav1.CreateOptions{}) + require.NoError(t, err) + + waitEvents(t, 1, c) + mockBGPServer.EXPECT().AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "10.96.10.10/32"}, {Prefix: "192.168.77.100/32"}, {Prefix: "192.168.77.150/32"}}), protocolIPv4).Times(1) + item, _ = c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + + // Update the Service. + updatedLoadBalancer := generateService(ipv4LoadBalancerName, corev1.ServiceTypeLoadBalancer, "10.96.10.10", "192.168.77.101", "192.168.77.151", false) + _, err = c.client.CoreV1().Services("default").Update(context.TODO(), updatedLoadBalancer, metav1.UpdateOptions{}) + require.NoError(t, err) + + waitEvents(t, 1, c) + mockBGPServer.EXPECT().AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "192.168.77.101/32"}, {Prefix: "192.168.77.151/32"}}), protocolIPv4).Times(1) + mockBGPServer.EXPECT().WithdrawRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "192.168.77.100/32"}, {Prefix: "192.168.77.150/32"}}), protocolIPv4).Times(1) + item, _ = c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + + // Delete the Service. + err = c.client.CoreV1().Services("default").Delete(context.TODO(), updatedLoadBalancer.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + + waitEvents(t, 1, c) + mockBGPServer.EXPECT().WithdrawRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "10.96.10.10/32"}, {Prefix: "192.168.77.101/32"}, {Prefix: "192.168.77.151/32"}}), protocolIPv4).Times(1) + item, _ = c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) +} + +func TestEgressLifecycle(t *testing.T) { + bp := generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + false, + false, + false, + true, + false, + []v1alpha1.BGPPeer{ipv4Peer1}) + c := newFakeController(t, []runtime.Object{node}, []runtime.Object{bp}, true, false) + defer mockNewBGPServer(c.mockBGPServer)() + mockBGPServer := c.mockBGPServer + + stopCh := make(chan struct{}) + defer close(stopCh) + + c.startInformers(stopCh) + + // Fake the passwords of BGP peers. + c.bgpPeerPasswords = bgpPeerPasswords + + // Initialize the test BGPPolicy. + waitEvents(t, 1, c) + mockBGPServer.EXPECT().Start().Times(1) + mockBGPServer.EXPECT().AddPeers([]engine.PeerConfig{*ipv4Peer1Config}, protocolIPv4).Times(1) + item, _ := c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + + // Create an Egress. + egress := generateEgress("eg1-4", "192.168.77.200", localNodeName) + _, err := c.crdClient.CrdV1beta1().Egresses().Create(context.TODO(), egress, metav1.CreateOptions{}) + require.NoError(t, err) + + waitEvents(t, 1, c) + mockBGPServer.EXPECT().AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "192.168.77.200/32"}}), protocolIPv4).Times(1) + item, _ = c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + + // Update the Egress. + updatedEgress := generateEgress("eg1-4", "192.168.77.201", localNodeName) + _, err = c.crdClient.CrdV1beta1().Egresses().Update(context.TODO(), updatedEgress, metav1.UpdateOptions{}) + require.NoError(t, err) + + waitEvents(t, 1, c) + mockBGPServer.EXPECT().AdvertiseRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "192.168.77.201/32"}}), protocolIPv4).Times(1) + mockBGPServer.EXPECT().WithdrawRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "192.168.77.200/32"}}), protocolIPv4).Times(1) + item, _ = c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + + // Delete the Egress. + err = c.crdClient.CrdV1beta1().Egresses().Delete(context.TODO(), updatedEgress.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + + waitEvents(t, 1, c) + mockBGPServer.EXPECT().WithdrawRoutes(gomock.InAnyOrder([]engine.Route{{Prefix: "192.168.77.201/32"}}), protocolIPv4).Times(1) + item, _ = c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) +} + +func TestBGPSecretUpdate(t *testing.T) { + bp := generateBGPPolicy(bgpPolicyName1, + nodeLabels1, + 179, + 65000, + false, + false, + false, + false, + true, + []v1alpha1.BGPPeer{ipv4Peer1, ipv4Peer2, ipv4Peer3}) + c := newFakeController(t, []runtime.Object{node}, []runtime.Object{bp}, true, false) + defer mockNewBGPServer(c.mockBGPServer)() + mockBGPServer := c.mockBGPServer + + stopCh := make(chan struct{}) + defer close(stopCh) + c.startInformers(stopCh) + go c.watchSecretChanges(stopCh) + + // Wait the Secret watcher to be ready. + time.Sleep(time.Second) + + // Create the Secret. + secret := generateSecret(bgpPeerPasswords) + _, err := c.client.CoreV1().Secrets("kube-system").Create(context.TODO(), secret, metav1.CreateOptions{}) + require.NoError(t, err) + + require.Eventually(t, func() bool { + c.bgpPeerPasswordsMutex.RLock() + defer c.bgpPeerPasswordsMutex.RUnlock() + if reflect.DeepEqual(c.bgpPeerPasswords, bgpPeerPasswords) { + return true + } + return false + }, 5*time.Second, 10*time.Millisecond) + + // Initialize the test BGPPolicy. + waitEvents(t, 1, c) + mockBGPServer.EXPECT().Start().Times(1) + mockBGPServer.EXPECT().AddPeers(gomock.InAnyOrder([]engine.PeerConfig{*ipv4Peer1Config, *ipv4Peer2Config, *ipv4Peer3Config}), protocolIPv4).Times(1) + mockBGPServer.EXPECT().AdvertiseRoutes([]engine.Route{{Prefix: podIPv4CIDR.String()}}, protocolIPv4).Times(1) + item, _ := c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) + + // Update the Secret. + updatedBGPPeerPasswords := map[string]string{ + generateBGPPeerKey(ipv4Peer1Addr, peer1ASN): "updated-" + peer1AuthPassword, + generateBGPPeerKey(ipv4Peer2Addr, peer2ASN): peer2AuthPassword, + generateBGPPeerKey(ipv4Peer3Addr, peer3ASN): "updated-" + peer3AuthPassword, + } + updatedSecret := generateSecret(updatedBGPPeerPasswords) + _, err = c.client.CoreV1().Secrets("kube-system").Update(context.TODO(), updatedSecret, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Eventually(t, func() bool { + c.bgpPeerPasswordsMutex.RLock() + defer c.bgpPeerPasswordsMutex.RUnlock() + if reflect.DeepEqual(c.bgpPeerPasswords, updatedBGPPeerPasswords) { + return true + } + return false + }, 5*time.Second, 10*time.Millisecond) + + // Process the event triggered by the update of the Secret. + waitEvents(t, 1, c) + updatedIPv4Peer1Config := *ipv4Peer1Config + updatedIPv4Peer3Config := *ipv4Peer3Config + updatedIPv4Peer1Config.AuthPassword = "updated-" + peer1AuthPassword + updatedIPv4Peer3Config.AuthPassword = "updated-" + peer3AuthPassword + mockBGPServer.EXPECT().UpdatePeers(gomock.InAnyOrder([]engine.PeerConfig{updatedIPv4Peer1Config, updatedIPv4Peer3Config}), protocolIPv4).Times(1) + item, _ = c.queue.Get() + require.NoError(t, c.syncBGPPolicy(item.(string))) + c.queue.Done(item) +} + +func generateBGPPolicyState(ipv4Enabled bool, + ipv6Enabled bool, + listenPort int32, + localASN int32, + routerID string, + prefixes []string, + peerConfigs []*engine.PeerConfig) *bgpPolicyState { + routesMap := make(map[netutils.IPFamily]sets.Set[engine.Route]) + peersMap := make(map[netutils.IPFamily]sets.Set[string]) + peerConfigsMap := make(map[string]*engine.PeerConfig) + if ipv4Enabled { + routesMap[protocolIPv4] = sets.New[engine.Route]() + peersMap[protocolIPv4] = sets.New[string]() + } + if ipv6Enabled { + routesMap[protocolIPv6] = sets.New[engine.Route]() + peersMap[protocolIPv6] = sets.New[string]() + } + + for _, prefix := range prefixes { + if netutils.IsIPv4CIDRString(prefix) { + routesMap[protocolIPv4].Insert(engine.Route{Prefix: prefix}) + } + if netutils.IsIPv6CIDRString(prefix) { + routesMap[protocolIPv6].Insert(engine.Route{Prefix: prefix}) + } + } + for _, peerConfig := range peerConfigs { + peerKey := generateBGPPeerKey(peerConfig.Address, peerConfig.ASN) + if netutils.IsIPv4String(peerConfig.Address) { + peersMap[protocolIPv4].Insert(peerKey) + } + if netutils.IsIPv6String(peerConfig.Address) { + peersMap[protocolIPv6].Insert(peerKey) + } + peerConfigsMap[peerKey] = peerConfig + } + + return &bgpPolicyState{ + listenPort: listenPort, + localASN: localASN, + routerID: routerID, + routes: routesMap, + peers: peersMap, + peerConfigs: peerConfigsMap, + } +} + +func checkBGPPolicyState(t *testing.T, expected, got *bgpPolicyState) { + require.Equal(t, expected != nil, got != nil) + if expected != nil { + assert.Equal(t, expected.listenPort, got.listenPort) + assert.Equal(t, expected.localASN, got.localASN) + assert.Equal(t, expected.routerID, got.routerID) + assert.Equal(t, expected.routes, got.routes) + assert.Equal(t, expected.peers, got.peers) + assert.Equal(t, expected.peerConfigs, got.peerConfigs) + } +} + +func generateBGPPolicy(name string, + nodeSelector map[string]string, + listenPort int32, + localASN int32, + advertiseClusterIP bool, + advertiseExternalIP bool, + advertiseLoadBalancerIP bool, + advertiseEgressIP bool, + advertisePodCIDR bool, + externalPeers []v1alpha1.BGPPeer) *v1alpha1.BGPPolicy { + var advertisement v1alpha1.Advertisements + advertisement.Service = &v1alpha1.ServiceAdvertisement{} + if advertiseClusterIP { + advertisement.Service.IPTypes = append(advertisement.Service.IPTypes, v1alpha1.ServiceIPTypeClusterIP) + } + if advertiseExternalIP { + advertisement.Service.IPTypes = append(advertisement.Service.IPTypes, v1alpha1.ServiceIPTypeExternalIP) + } + if advertiseLoadBalancerIP { + advertisement.Service.IPTypes = append(advertisement.Service.IPTypes, v1alpha1.ServiceIPTypeLoadBalancerIP) + } + if advertiseEgressIP { + advertisement.Egress = &v1alpha1.EgressAdvertisement{} + } + + if advertisePodCIDR { + advertisement.Pod = &v1alpha1.PodAdvertisement{} + } + return &v1alpha1.BGPPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: "test-uid"}, + Spec: v1alpha1.BGPPolicySpec{ + NodeSelector: metav1.LabelSelector{MatchLabels: nodeSelector}, + LocalASN: localASN, + ListenPort: &listenPort, + Advertisements: advertisement, + BGPPeers: externalPeers, + }, + } +} + +func generateService(name string, + svcType corev1.ServiceType, + clusterIP string, + externalIP string, + LoadBalancerIP string, + trafficPolicyLocal bool) *corev1.Service { + itp := corev1.ServiceInternalTrafficPolicyCluster + if trafficPolicyLocal { + itp = corev1.ServiceInternalTrafficPolicyLocal + } + etp := corev1.ServiceExternalTrafficPolicyCluster + if trafficPolicyLocal { + etp = corev1.ServiceExternalTrafficPolicyLocal + } + var externalIPs []string + if externalIP != "" { + externalIPs = append(externalIPs, externalIP) + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + UID: "test-uid", + }, + Spec: corev1.ServiceSpec{ + Type: svcType, + ClusterIP: clusterIP, + Ports: []corev1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: corev1.ProtocolTCP, + }}, + ClusterIPs: []string{clusterIP}, + ExternalIPs: externalIPs, + InternalTrafficPolicy: &itp, + ExternalTrafficPolicy: etp, + }, + } + if LoadBalancerIP != "" { + ingress := []corev1.LoadBalancerIngress{{IP: LoadBalancerIP}} + svc.Status.LoadBalancer.Ingress = ingress + } + return svc +} + +func generateEgress(name string, ip string, nodeName string) *crdv1b1.Egress { + return &crdv1b1.Egress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: "test-uid", + }, + Spec: crdv1b1.EgressSpec{ + EgressIP: ip, + }, + Status: crdv1b1.EgressStatus{ + EgressIP: ip, + EgressNode: nodeName, + }, + } +} + +func generateNode(name string, labels, annotations map[string]string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: "test-uid", + Labels: labels, + Annotations: annotations, + }, + } +} + +func generateEndpointSlice(svcName string, + isLocal bool, + isIPv6 bool, + endpointIP string) *discovery.EndpointSlice { + addrType := discovery.AddressTypeIPv4 + if isIPv6 { + addrType = discovery.AddressTypeIPv6 + } + var nodeName *string + if isLocal { + nodeName = &localNodeName + } + protocol := corev1.ProtocolTCP + endpointSlice := &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", svcName, rand.String(5)), + Namespace: "default", + UID: "test-uid", + Labels: map[string]string{ + discovery.LabelServiceName: svcName, + }, + }, + AddressType: addrType, + Endpoints: []discovery.Endpoint{{ + Addresses: []string{ + endpointIP, + }, + Conditions: discovery.EndpointConditions{ + Ready: ptr.To(true), + }, + Hostname: nodeName, + NodeName: nodeName, + }}, + Ports: []discovery.EndpointPort{{ + Name: ptr.To("p80"), + Port: ptr.To(int32(80)), + Protocol: &protocol, + }}, + } + + return endpointSlice +} + +func generateBGPPeer(ip string, asn, port, gracefulRestartTimeSeconds int32) v1alpha1.BGPPeer { + return v1alpha1.BGPPeer{ + Address: ip, + Port: &port, + ASN: asn, + MultihopTTL: ptr.To(int32(1)), + GracefulRestartTimeSeconds: &gracefulRestartTimeSeconds, + } +} + +func generateBGPPeerConfig(ip string, asn, port, gracefulRestartTime int32, authPassword string) *engine.PeerConfig { + return &engine.PeerConfig{ + Address: ip, + Port: port, + ASN: asn, + MultihopTTL: int32(1), + GracefulRestartTimeSeconds: gracefulRestartTime, + AuthPassword: authPassword, + } +} + +func generateSecret(rawData map[string]string) *corev1.Secret { + data := make(map[string][]byte) + for k, v := range rawData { + data[k] = []byte(v) + } + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: testAntreaBGPPasswordSecret, + Namespace: "kube-system", + UID: "test-uid", + }, + Type: corev1.SecretTypeOpaque, + Data: data, + } +} + +func ipStrToPrefix(ipStr string) string { + if netutils.IsIPv4String(ipStr) { + return ipStr + ipv4Suffix + } else if netutils.IsIPv6String(ipStr) { + return ipStr + ipv6Suffix + } + return "" +} + +func waitEvents(t *testing.T, expectedEvents int, c *fakeController) { + require.Eventually(t, func() bool { + return c.queue.Len() == expectedEvents + }, 5*time.Second, 10*time.Millisecond) +} diff --git a/pkg/agent/bgp/engine/gobgp.go b/pkg/agent/bgp/engine/gobgp.go new file mode 100644 index 00000000000..a44479b673a --- /dev/null +++ b/pkg/agent/bgp/engine/gobgp.go @@ -0,0 +1,295 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import ( + "context" + "fmt" + "net/netip" + "time" + + gobgpapi "github.com/osrg/gobgp/v3/api" + "github.com/osrg/gobgp/v3/pkg/server" + "google.golang.org/protobuf/types/known/anypb" + "k8s.io/utils/net" +) + +const ( + ipv4AllZero = "0.0.0.0" + ipv6AllZero = "::" +) + +type GoBGPServer struct { + server *server.BgpServer + globalConfig *gobgpapi.Global +} + +func NewGoBGPServer(globalConfig *GlobalConfig, extraLogFields map[string]interface{}) (Interface, error) { + logger, err := newGoBGPServerLogger(extraLogFields) + if err != nil { + return nil, err + } + return &GoBGPServer{ + server: server.NewBgpServer(server.LoggerOption(logger)), + globalConfig: &gobgpapi.Global{ + Asn: globalConfig.ASN, + RouterId: globalConfig.RouterID, + ListenPort: globalConfig.ListenPort, + }, + }, nil +} + +func (g *GoBGPServer) Start() error { + go g.server.Serve() + if err := g.server.StartBgp(context.TODO(), &gobgpapi.StartBgpRequest{Global: g.globalConfig}); err != nil { + return fmt.Errorf("failed to start BGP server: %w", err) + } + return nil +} + +func (g *GoBGPServer) Stop() error { + if err := g.server.StopBgp(context.TODO(), &gobgpapi.StopBgpRequest{}); err != nil { + return fmt.Errorf("failed to stop BGP server: %w", err) + } + return nil +} + +func (g *GoBGPServer) AddPeers(peerConfigs []PeerConfig, ipFamily net.IPFamily) error { + for i := range peerConfigs { + request := &gobgpapi.AddPeerRequest{Peer: toGoBGPPeer(&peerConfigs[i], ipFamily)} + if err := g.server.AddPeer(context.TODO(), request); err != nil { + return err + } + } + return nil +} + +func (g *GoBGPServer) UpdatePeers(peerConfigs []PeerConfig, ipFamily net.IPFamily) error { + for i := range peerConfigs { + request := &gobgpapi.UpdatePeerRequest{Peer: toGoBGPPeer(&peerConfigs[i], ipFamily)} + if _, err := g.server.UpdatePeer(context.TODO(), request); err != nil { + return err + } + } + return nil +} + +func (g *GoBGPServer) RemovePeers(peerConfigs []PeerConfig) error { + for _, peerConfig := range peerConfigs { + request := &gobgpapi.DeletePeerRequest{Address: peerConfig.Address} + if err := g.server.DeletePeer(context.TODO(), request); err != nil { + return err + } + } + return nil +} + +func (g *GoBGPServer) GetPeers() ([]PeerStatus, error) { + var peerStatuses []PeerStatus + + fn := func(peer *gobgpapi.Peer) { + if peer == nil { + return + } + peerStatus := PeerStatus{} + if peer.Transport != nil { + peerStatus.Port = int32(peer.GetTransport().GetRemotePort()) + } + if peer.Conf != nil { + peerStatus.Address = peer.GetConf().GetNeighborAddress() + peerStatus.ASN = int32(peer.GetConf().GetPeerAsn()) + } + if peer.GetEbgpMultihop() != nil && peer.GetEbgpMultihop().GetEnabled() { + peerStatus.MultihopTTL = int32(peer.GetEbgpMultihop().GetMultihopTtl()) + } else { + peerStatus.MultihopTTL = DefaultBGPMultihopTTL + } + if peer.GetGracefulRestart() != nil { + peerStatus.GracefulRestartTimeSeconds = int32(peer.GetGracefulRestart().GetRestartTime()) + } + if peer.State != nil { + peerStatus.SessionState = toSessionState(peer.GetState().GetSessionState()) + if peerStatus.SessionState == SessionEstablished && peer.GetTimers() != nil && peer.GetTimers().GetState() != nil { + peerStatus.UptimeSeconds = int(time.Since(peer.GetTimers().GetState().GetUptime().AsTime()).Seconds()) + } + } + peerStatuses = append(peerStatuses, peerStatus) + } + + request := &gobgpapi.ListPeerRequest{EnableAdvertised: true} + err := g.server.ListPeer(context.TODO(), request, fn) + if err != nil { + return peerStatuses, err + } + return peerStatuses, nil +} + +func (g *GoBGPServer) AdvertiseRoutes(routes []Route, ipFamily net.IPFamily) error { + for i := range routes { + request := &gobgpapi.AddPathRequest{Path: toGoBGPPath(&routes[i], ipFamily)} + if _, err := g.server.AddPath(context.TODO(), request); err != nil { + return err + } + } + return nil +} + +func (g *GoBGPServer) WithdrawRoutes(routes []Route, ipFamily net.IPFamily) error { + for i := range routes { + request := &gobgpapi.DeletePathRequest{Path: toGoBGPPath(&routes[i], ipFamily)} + if err := g.server.DeletePath(context.TODO(), request); err != nil { + return err + } + } + return nil +} + +func (g *GoBGPServer) GetRoutes(routeType RouteType, peerAddress string, ipFamily net.IPFamily) ([]Route, error) { + var gobgpTableType gobgpapi.TableType + if routeType == RouteAdvertised { + gobgpTableType = gobgpapi.TableType_ADJ_OUT + } else if routeType == RouteReceived { + gobgpTableType = gobgpapi.TableType_ADJ_IN + } else { + return nil, fmt.Errorf("invalid route type: %v", routeType) + } + if peerAddress == "" { + return nil, fmt.Errorf("BGP peer IP address must be specified") + } + + var routes []Route + fn := func(destination *gobgpapi.Destination) { + if destination == nil { + return + } + routes = append(routes, Route{Prefix: destination.GetPrefix()}) + } + + request := &gobgpapi.ListPathRequest{ + TableType: gobgpTableType, + Family: &gobgpapi.Family{Afi: toGoBGPFamilyAfi(ipFamily), Safi: gobgpapi.Family_SAFI_UNICAST}, + Name: peerAddress, + } + + if err := g.server.ListPath(context.TODO(), request, fn); err != nil { + return nil, err + } + return routes, nil +} + +func toGoBGPPath(route *Route, ipFamily net.IPFamily) *gobgpapi.Path { + goBGPIPFamily := toGoBGPFamilyAfi(ipFamily) + prefix, _ := netip.ParsePrefix(route.Prefix) + nlri, _ := anypb.New(&gobgpapi.IPAddressPrefix{ + Prefix: prefix.Addr().String(), + PrefixLen: uint32(prefix.Bits()), + }) + + var attrs []*anypb.Any + a1, _ := anypb.New(&gobgpapi.OriginAttribute{ + Origin: 0, + }) + attrs = append(attrs, a1) + + if ipFamily == net.IPv4 { + a2, _ := anypb.New(&gobgpapi.NextHopAttribute{ + NextHop: ipv4AllZero, + }) + attrs = append(attrs, a2) + } else if ipFamily == net.IPv6 { + a2, _ := anypb.New(&gobgpapi.MpReachNLRIAttribute{ + Family: &gobgpapi.Family{Afi: goBGPIPFamily, Safi: gobgpapi.Family_SAFI_UNICAST}, + NextHops: []string{ipv6AllZero}, + Nlris: []*anypb.Any{nlri}, + }) + attrs = append(attrs, a2) + } + return &gobgpapi.Path{ + Family: &gobgpapi.Family{Afi: goBGPIPFamily, Safi: gobgpapi.Family_SAFI_UNICAST}, + Nlri: nlri, + Pattrs: attrs, + } +} + +func toGoBGPFamilyAfi(ipFamily net.IPFamily) gobgpapi.Family_Afi { + switch ipFamily { + case net.IPv4: + return gobgpapi.Family_AFI_IP + case net.IPv6: + return gobgpapi.Family_AFI_IP6 + default: + return gobgpapi.Family_AFI_UNKNOWN + } +} + +func toGoBGPPeer(peerConfig *PeerConfig, ipFamily net.IPFamily) *gobgpapi.Peer { + peer := &gobgpapi.Peer{ + Conf: &gobgpapi.PeerConf{ + NeighborAddress: peerConfig.Address, + PeerAsn: uint32(peerConfig.ASN), + AuthPassword: peerConfig.AuthPassword, + }, + Transport: &gobgpapi.Transport{ + RemotePort: uint32(peerConfig.Port), + }, + AfiSafis: []*gobgpapi.AfiSafi{ + { + Config: &gobgpapi.AfiSafiConfig{ + Family: &gobgpapi.Family{Afi: toGoBGPFamilyAfi(ipFamily), Safi: gobgpapi.Family_SAFI_UNICAST}, + Enabled: true, + }, + MpGracefulRestart: &gobgpapi.MpGracefulRestart{ + Config: &gobgpapi.MpGracefulRestartConfig{ + Enabled: true, + }, + }, + }, + }, + } + if peerConfig.MultihopTTL != DefaultBGPMultihopTTL { + peer.EbgpMultihop = &gobgpapi.EbgpMultihop{ + Enabled: true, + MultihopTtl: uint32(peerConfig.MultihopTTL), + } + } + if peerConfig.GracefulRestartTimeSeconds != 0 { + peer.GracefulRestart = &gobgpapi.GracefulRestart{ + Enabled: true, + RestartTime: uint32(peerConfig.GracefulRestartTimeSeconds), + } + } + return peer +} + +func toSessionState(s gobgpapi.PeerState_SessionState) SessionState { + switch s { + case gobgpapi.PeerState_UNKNOWN: + return SessionUnknown + case gobgpapi.PeerState_IDLE: + return SessionIdle + case gobgpapi.PeerState_CONNECT: + return SessionConnect + case gobgpapi.PeerState_ACTIVE: + return SessionActive + case gobgpapi.PeerState_OPENSENT: + return SessionOpenSent + case gobgpapi.PeerState_OPENCONFIRM: + return SessionOpenConfirm + case gobgpapi.PeerState_ESTABLISHED: + return SessionEstablished + default: + return SessionUnknown + } +} diff --git a/pkg/agent/bgp/engine/gobgp_test.go b/pkg/agent/bgp/engine/gobgp_test.go new file mode 100644 index 00000000000..4877a02754c --- /dev/null +++ b/pkg/agent/bgp/engine/gobgp_test.go @@ -0,0 +1,82 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import ( + "testing" + + gobgpapi "github.com/osrg/gobgp/v3/api" + "github.com/stretchr/testify/assert" + "k8s.io/utils/net" +) + +func TestToGoBGPPath(t *testing.T) { + route4 := &Route{Prefix: "192.168.0.0/24"} + path4 := toGoBGPPath(route4, net.IPv4) + + ipAddressPrefix4 := &gobgpapi.IPAddressPrefix{} + assert.NoError(t, path4.GetNlri().UnmarshalTo(ipAddressPrefix4)) + assert.Equal(t, "192.168.0.0", ipAddressPrefix4.Prefix) + assert.Equal(t, uint32(24), ipAddressPrefix4.PrefixLen) + assert.Equal(t, gobgpapi.Family_AFI_IP, path4.GetFamily().Afi) + + route6 := &Route{Prefix: "2001:db8::/64"} + path6 := toGoBGPPath(route6, net.IPv6) + + ipAddressPrefix6 := &gobgpapi.IPAddressPrefix{} + assert.NoError(t, path6.GetNlri().UnmarshalTo(ipAddressPrefix6)) + assert.Equal(t, "2001:db8::", ipAddressPrefix6.Prefix) + assert.Equal(t, uint32(64), ipAddressPrefix6.PrefixLen) + assert.Equal(t, gobgpapi.Family_AFI_IP6, path6.GetFamily().Afi) +} + +func TestToGoBGPPeer(t *testing.T) { + peerConfig := &PeerConfig{ + Address: "192.168.0.1", + ASN: 65000, + AuthPassword: "password", + Port: 179, + MultihopTTL: 2, + GracefulRestartTimeSeconds: 120, + } + peer := toGoBGPPeer(peerConfig, net.IPv4) + assert.Equal(t, "192.168.0.1", peer.GetConf().GetNeighborAddress()) + assert.Equal(t, uint32(65000), peer.GetConf().GetPeerAsn()) + assert.Equal(t, "password", peer.GetConf().GetAuthPassword()) + assert.Equal(t, uint32(179), peer.GetTransport().GetRemotePort()) + assert.Equal(t, uint32(2), peer.GetEbgpMultihop().GetMultihopTtl()) + assert.Equal(t, uint32(120), peer.GetGracefulRestart().GetRestartTime()) +} + +func TestToSessionState(t *testing.T) { + tests := []struct { + input gobgpapi.PeerState_SessionState + expected SessionState + }{ + {gobgpapi.PeerState_UNKNOWN, SessionUnknown}, + {gobgpapi.PeerState_IDLE, SessionIdle}, + {gobgpapi.PeerState_CONNECT, SessionConnect}, + {gobgpapi.PeerState_ACTIVE, SessionActive}, + {gobgpapi.PeerState_OPENSENT, SessionOpenSent}, + {gobgpapi.PeerState_OPENCONFIRM, SessionOpenConfirm}, + {gobgpapi.PeerState_ESTABLISHED, SessionEstablished}, + {gobgpapi.PeerState_SessionState(999), SessionUnknown}, + } + + for _, test := range tests { + output := toSessionState(test.input) + assert.Equal(t, test.expected, output) + } +} diff --git a/pkg/agent/bgp/engine/interface.go b/pkg/agent/bgp/engine/interface.go new file mode 100644 index 00000000000..00cf004aa34 --- /dev/null +++ b/pkg/agent/bgp/engine/interface.go @@ -0,0 +1,114 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import ( + "k8s.io/utils/net" +) + +// Interface defines the methods for managing a BGP (Border Gateway Protocol) process. +// Currently, only the goBGP implementation is available. +// More implementations might be added later. +type Interface interface { + // Start initiates the BGP process. + Start() error + + // Stop terminates the BGP process. + Stop() error + + // AddPeers adds new BGP peers based on the provided configurations and IP family. + AddPeers(peerConfigs []PeerConfig, ipFamily net.IPFamily) error + + // UpdatePeers updates the configurations of existing BGP peers based on the provided configurations and IP family. + UpdatePeers(peerConfigs []PeerConfig, ipFamily net.IPFamily) error + + // RemovePeers removes the specified BGP peers. + RemovePeers(peerConfigs []PeerConfig) error + + // GetPeers retrieves the current status of all BGP peers. + GetPeers() ([]PeerStatus, error) + + // AdvertiseRoutes announces the specified routes to BGP peers for the given IP family. + AdvertiseRoutes(routes []Route, ipFamily net.IPFamily) error + + // WithdrawRoutes withdraws the specified routes from BGP peers for the given IP family. + WithdrawRoutes(routes []Route, ipFamily net.IPFamily) error + + // GetRoutes retrieves the advertised / received routes to / from the given peer for the given IP family. + GetRoutes(routeType RouteType, peerAddress string, ipFamily net.IPFamily) ([]Route, error) +} + +const ( + // DefaultBGPListenPort is the default port for BGP server. + DefaultBGPListenPort int32 = 179 + + // DefaultBGPGracefulRestartTimeSeconds is the default time for BGP graceful restart in seconds. + DefaultBGPGracefulRestartTimeSeconds int32 = 120 + + // DefaultBGPMultihopTTL is the default Time To Live (TTL) for BGP multihop sessions. + DefaultBGPMultihopTTL int32 = 1 +) + +// GlobalConfig contains the global configuration to start a BGP server. More attributes might be added latter. +type GlobalConfig struct { + ASN uint32 + RouterID string + ListenPort int32 +} + +type SessionState string + +const ( + // SessionUnknown indicates an unknown BGP session state. + SessionUnknown SessionState = "Unknown" + // The following are the states of the BGP Finite State Machine. + // For more details see https://datatracker.ietf.org/doc/html/rfc4271#section-8.2.2. + SessionIdle SessionState = "Idle" + SessionConnect SessionState = "Connect" + SessionActive SessionState = "Active" + SessionOpenSent SessionState = "OpenSent" + SessionOpenConfirm SessionState = "OpenConfirm" + SessionEstablished SessionState = "Established" +) + +type RouteType int + +const ( + RouteAdvertised RouteType = iota + RouteReceived +) + +// PeerConfig contains the configuration for a BGP peer. More attributes might be added latter. +type PeerConfig struct { + Address string + Port int32 + ASN int32 + AuthPassword string + MultihopTTL int32 + GracefulRestartTimeSeconds int32 +} + +// PeerStatus contains the status information for a BGP peer. More attributes related to status might be added latter. +type PeerStatus struct { + PeerConfig + SessionState SessionState + UptimeSeconds int +} + +// Route represents a BGP route. Currently only prefix (e.g., "192.168.0.0/24") is needed. More attributes might be +// added latter. +type Route struct { + Prefix string +} diff --git a/pkg/agent/bgp/engine/logs.go b/pkg/agent/bgp/engine/logs.go new file mode 100644 index 00000000000..dd7b7944c6f --- /dev/null +++ b/pkg/agent/bgp/engine/logs.go @@ -0,0 +1,116 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import ( + "fmt" + "os" + "path/filepath" + + gobgplog "github.com/osrg/gobgp/v3/pkg/log" + "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" + + "antrea.io/antrea/pkg/util/logdir" +) + +const ( + logfileSubdir string = "bgp" + logfileName string = "gobgp.log" +) + +// goBGPServerLogger implements github.com/osrg/gobgp/v3/pkg/log/Logger interface. +type goBGPServerLogger struct { + logger *logrus.Logger + extraFields map[string]interface{} +} + +func newGoBGPServerLogger(extraFields map[string]interface{}) (*goBGPServerLogger, error) { + logDir := filepath.Join(logdir.GetLogDir(), logfileSubdir) + logFile := filepath.Join(logDir, logfileName) + _, err := os.Stat(logDir) + if os.IsNotExist(err) { + os.Mkdir(logDir, 0755) + } else if err != nil { + return nil, fmt.Errorf("received error while accessing BGP log directory: %v", err) + } + + // Use lumberjack log file rotation. + logOutput := &lumberjack.Logger{ + Filename: logFile, + MaxSize: 28, + MaxBackups: 3, + MaxAge: 28, + Compress: true, + } + + logger := logrus.New() + logger.SetOutput(logOutput) + + return &goBGPServerLogger{ + logger: logger, + extraFields: extraFields, + }, nil +} + +func (g *goBGPServerLogger) Panic(msg string, fields gobgplog.Fields) { + for k, v := range g.extraFields { + fields[k] = v + } + g.logger.WithFields(logrus.Fields(fields)).Panic(msg) +} + +func (g *goBGPServerLogger) Fatal(msg string, fields gobgplog.Fields) { + for k, v := range g.extraFields { + fields[k] = v + } + g.logger.WithFields(logrus.Fields(fields)).Fatal(msg) +} + +func (g *goBGPServerLogger) Error(msg string, fields gobgplog.Fields) { + for k, v := range g.extraFields { + fields[k] = v + } + g.logger.WithFields(logrus.Fields(fields)).Error(msg) +} + +func (g *goBGPServerLogger) Warn(msg string, fields gobgplog.Fields) { + for k, v := range g.extraFields { + fields[k] = v + } + g.logger.WithFields(logrus.Fields(fields)).Warn(msg) +} + +func (g *goBGPServerLogger) Info(msg string, fields gobgplog.Fields) { + for k, v := range g.extraFields { + fields[k] = v + } + g.logger.WithFields(logrus.Fields(fields)).Info(msg) +} + +func (g *goBGPServerLogger) Debug(msg string, fields gobgplog.Fields) { + for k, v := range g.extraFields { + fields[k] = v + } + g.logger.WithFields(logrus.Fields(fields)).Debug(msg) +} + +func (g *goBGPServerLogger) SetLevel(level gobgplog.LogLevel) { + g.logger.SetLevel(logrus.Level(level)) +} + +func (g *goBGPServerLogger) GetLevel() gobgplog.LogLevel { + return gobgplog.LogLevel(g.logger.GetLevel()) +} diff --git a/pkg/agent/bgp/engine/testing/mock_engine.go b/pkg/agent/bgp/engine/testing/mock_engine.go new file mode 100644 index 00000000000..123ed083b91 --- /dev/null +++ b/pkg/agent/bgp/engine/testing/mock_engine.go @@ -0,0 +1,183 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: antrea.io/antrea/pkg/agent/bgp/engine (interfaces: Interface) +// +// Generated by this command: +// +// mockgen -copyright_file hack/boilerplate/license_header.raw.txt -destination pkg/agent/bgp/engine/testing/mock_engine.go -package testing antrea.io/antrea/pkg/agent/bgp/engine Interface +// +// Package testing is a generated GoMock package. +package testing + +import ( + reflect "reflect" + + engine "antrea.io/antrea/pkg/agent/bgp/engine" + gomock "go.uber.org/mock/gomock" + net "k8s.io/utils/net" +) + +// MockInterface is a mock of Interface interface. +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface. +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance. +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// AddPeers mocks base method. +func (m *MockInterface) AddPeers(arg0 []engine.PeerConfig, arg1 net.IPFamily) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddPeers", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddPeers indicates an expected call of AddPeers. +func (mr *MockInterfaceMockRecorder) AddPeers(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPeers", reflect.TypeOf((*MockInterface)(nil).AddPeers), arg0, arg1) +} + +// AdvertiseRoutes mocks base method. +func (m *MockInterface) AdvertiseRoutes(arg0 []engine.Route, arg1 net.IPFamily) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdvertiseRoutes", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AdvertiseRoutes indicates an expected call of AdvertiseRoutes. +func (mr *MockInterfaceMockRecorder) AdvertiseRoutes(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdvertiseRoutes", reflect.TypeOf((*MockInterface)(nil).AdvertiseRoutes), arg0, arg1) +} + +// GetPeers mocks base method. +func (m *MockInterface) GetPeers() ([]engine.PeerStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPeers") + ret0, _ := ret[0].([]engine.PeerStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPeers indicates an expected call of GetPeers. +func (mr *MockInterfaceMockRecorder) GetPeers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPeers", reflect.TypeOf((*MockInterface)(nil).GetPeers)) +} + +// GetRoutes mocks base method. +func (m *MockInterface) GetRoutes(arg0 engine.RouteType, arg1 string, arg2 net.IPFamily) ([]engine.Route, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRoutes", arg0, arg1, arg2) + ret0, _ := ret[0].([]engine.Route) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRoutes indicates an expected call of GetRoutes. +func (mr *MockInterfaceMockRecorder) GetRoutes(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoutes", reflect.TypeOf((*MockInterface)(nil).GetRoutes), arg0, arg1, arg2) +} + +// RemovePeers mocks base method. +func (m *MockInterface) RemovePeers(arg0 []engine.PeerConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemovePeers", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemovePeers indicates an expected call of RemovePeers. +func (mr *MockInterfaceMockRecorder) RemovePeers(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePeers", reflect.TypeOf((*MockInterface)(nil).RemovePeers), arg0) +} + +// Start mocks base method. +func (m *MockInterface) Start() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start") + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *MockInterfaceMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockInterface)(nil).Start)) +} + +// Stop mocks base method. +func (m *MockInterface) Stop() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop") + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *MockInterfaceMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockInterface)(nil).Stop)) +} + +// UpdatePeers mocks base method. +func (m *MockInterface) UpdatePeers(arg0 []engine.PeerConfig, arg1 net.IPFamily) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdatePeers", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdatePeers indicates an expected call of UpdatePeers. +func (mr *MockInterfaceMockRecorder) UpdatePeers(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePeers", reflect.TypeOf((*MockInterface)(nil).UpdatePeers), arg0, arg1) +} + +// WithdrawRoutes mocks base method. +func (m *MockInterface) WithdrawRoutes(arg0 []engine.Route, arg1 net.IPFamily) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WithdrawRoutes", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// WithdrawRoutes indicates an expected call of WithdrawRoutes. +func (mr *MockInterfaceMockRecorder) WithdrawRoutes(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithdrawRoutes", reflect.TypeOf((*MockInterface)(nil).WithdrawRoutes), arg0, arg1) +} diff --git a/pkg/agent/controller/networkpolicy/l7engine/reconciler.go b/pkg/agent/controller/networkpolicy/l7engine/reconciler.go index 8f693782b7a..5ba7fed704f 100644 --- a/pkg/agent/controller/networkpolicy/l7engine/reconciler.go +++ b/pkg/agent/controller/networkpolicy/l7engine/reconciler.go @@ -518,7 +518,7 @@ func startSuricata() { if err := cmd.Run(); err != nil { klog.ErrorS(err, "Failed to start Suricata instance") } -} +} // suricata -c /etc/suricata/suricata.yaml --af-packet -D -l /var/log/antrea/networkpolicy/l7engine/ func suricataSc(scCmd string) (*scCmdRet, error) { cmd := exec.Command("suricatasc", "-c", scCmd) diff --git a/pkg/apiserver/handlers/featuregates/handler_test.go b/pkg/apiserver/handlers/featuregates/handler_test.go index adb630a1324..fe1318252d1 100644 --- a/pkg/apiserver/handlers/featuregates/handler_test.go +++ b/pkg/apiserver/handlers/featuregates/handler_test.go @@ -55,6 +55,7 @@ func Test_getGatesResponse(t *testing.T) { {Component: "agent", Name: "AntreaIPAM", Status: "Disabled", Version: "ALPHA"}, {Component: "agent", Name: "AntreaPolicy", Status: "Disabled", Version: "BETA"}, {Component: "agent", Name: "AntreaProxy", Status: "Enabled", Version: "GA"}, + {Component: "agent", Name: "BGPPolicy", Status: "Disabled", Version: "ALPHA"}, {Component: "agent", Name: "CleanupStaleUDPSvcConntrack", Status: cleanupStaleUDPSvcConntrackStatus, Version: "BETA"}, {Component: "agent", Name: "Egress", Status: egressStatus, Version: "BETA"}, {Component: "agent", Name: "EgressSeparateSubnet", Status: "Disabled", Version: "ALPHA"}, @@ -105,6 +106,7 @@ func Test_getGatesWindowsResponse(t *testing.T) { want: []apis.FeatureGateResponse{ {Component: "agent-windows", Name: "AntreaPolicy", Status: "Disabled", Version: "BETA"}, {Component: "agent-windows", Name: "AntreaProxy", Status: "Enabled", Version: "GA"}, + {Component: "agent-windows", Name: "BGPPolicy", Status: "Disabled", Version: "ALPHA"}, {Component: "agent-windows", Name: "EndpointSlice", Status: "Enabled", Version: "GA"}, {Component: "agent-windows", Name: "ExternalNode", Status: "Disabled", Version: "ALPHA"}, {Component: "agent-windows", Name: "FlowExporter", Status: "Disabled", Version: "ALPHA"}, diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go index bd298a778c6..4a952961519 100644 --- a/pkg/config/agent/config.go +++ b/pkg/config/agent/config.go @@ -197,6 +197,8 @@ type AgentConfig struct { // second(pps) and the burst size will be automatically set to twice the rate. // When the rate and burst size are exceeded, new packets will be dropped. PacketInRate int `yaml:"packetInRate,omitempty"` + // BGPPolicy related configurations. + BGPPolicy BGPPolicyConfig `yaml:"bgpPolicy,omitempty"` } type AntreaProxyConfig struct { @@ -392,3 +394,8 @@ type OVSBridgeConfig struct { // Names of physical interfaces to be connected to the bridge. PhysicalInterfaces []string `yaml:"physicalInterfaces,omitempty"` } + +type BGPPolicyConfig struct { + // Name of the Secret storing passwords of BGP peers. + SecretName string `yaml:"secretName,omitempty"` +} diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 05cb51a9df5..c12243e92a1 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -163,6 +163,10 @@ const ( // alpha: v2.1 // Enable the NodeLatencyMonitor feature. NodeLatencyMonitor featuregate.Feature = "NodeLatencyMonitor" + + // alpha: v2.1 + // Allow users to advertise Service IPs, Pod IPs, and Egress IPs to external BGP peers. + BGPPolicy featuregate.Feature = "BGPPolicy" ) var ( @@ -179,6 +183,7 @@ var ( DefaultAntreaFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ AntreaPolicy: {Default: true, PreRelease: featuregate.Beta}, AntreaProxy: {Default: true, PreRelease: featuregate.GA}, + BGPPolicy: {Default: false, PreRelease: featuregate.Alpha}, Egress: {Default: true, PreRelease: featuregate.Beta}, EndpointSlice: {Default: true, PreRelease: featuregate.GA}, TopologyAwareHints: {Default: true, PreRelease: featuregate.Beta}, @@ -213,6 +218,7 @@ var ( AntreaIPAM, AntreaPolicy, AntreaProxy, + BGPPolicy, CleanupStaleUDPSvcConntrack, Egress, EndpointSlice, diff --git a/test/integration/agent/gobgp_test.go b/test/integration/agent/gobgp_test.go new file mode 100644 index 00000000000..f8dd7c91be5 --- /dev/null +++ b/test/integration/agent/gobgp_test.go @@ -0,0 +1,415 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/net" + + "antrea.io/antrea/pkg/agent/bgp/engine" +) + +func TestGoBGPLifecycle(t *testing.T) { + asn1 := int32(61179) + asn2 := int32(62179) + asn3 := int32(63179) + routerID1 := "192.168.1.1" + routerID2 := "192.168.1.2" + routerID3 := "192.168.1.3" + listenPort1 := int32(1179) + listenPort2 := int32(2179) + listenPort3 := int32(3179) + server1GlobalConfig := &engine.GlobalConfig{ + ASN: uint32(asn1), + RouterID: routerID1, + ListenPort: listenPort1, + } + server2GlobalConfig := &engine.GlobalConfig{ + ASN: uint32(asn2), + RouterID: routerID2, + ListenPort: listenPort2, + } + server3GlobalConfig := &engine.GlobalConfig{ + ASN: uint32(asn3), + RouterID: routerID3, + ListenPort: listenPort3, + } + + server1, err := engine.NewGoBGPServer(server1GlobalConfig, nil) + require.NoError(t, err) + server2, err := engine.NewGoBGPServer(server2GlobalConfig, nil) + require.NoError(t, err) + server3, err := engine.NewGoBGPServer(server3GlobalConfig, nil) + require.NoError(t, err) + + t.Log("Starting all BGP servers") + require.NoError(t, server1.Start()) + require.NoError(t, server2.Start()) + require.NoError(t, server3.Start()) + t.Log("Started all BGP servers") + + ipv4Server1Config := engine.PeerConfig{ + Address: "127.0.0.1", + Port: 1179, + ASN: 61179, + MultihopTTL: 1, + GracefulRestartTimeSeconds: 120, + } + ipv6Server1Config := engine.PeerConfig{ + Address: "::1", + Port: 1179, + ASN: 61179, + MultihopTTL: 1, + GracefulRestartTimeSeconds: 120, + } + ipv4Server2Config := engine.PeerConfig{ + Address: "127.0.0.1", + Port: 2179, + ASN: 62179, + MultihopTTL: 1, + GracefulRestartTimeSeconds: 120, + } + ipv6Server3Config := engine.PeerConfig{ + Address: "::1", + Port: 3179, + ASN: 63179, + MultihopTTL: 1, + GracefulRestartTimeSeconds: 120, + } + + t.Log("Adding BGP peers for BGP server1") + require.NoError(t, server1.AddPeers([]engine.PeerConfig{ipv4Server2Config}, net.IPv4)) + require.NoError(t, server1.AddPeers([]engine.PeerConfig{ipv6Server3Config}, net.IPv6)) + t.Log("Added BGP peers for BGP server1") + + t.Log("Adding BGP peers for BGP server2") + require.NoError(t, server2.AddPeers([]engine.PeerConfig{ipv4Server1Config}, net.IPv4)) + t.Log("Added BGP peers for BGP server2") + + t.Log("Adding BGP peers for BGP server3") + require.NoError(t, server3.AddPeers([]engine.PeerConfig{ipv6Server1Config}, net.IPv6)) + t.Log("Added BGP peers for BGP server3") + + t.Log("Getting peers of BGP server1 and verifying them") + assert.Eventually(t, func() bool { + peers, err := server1.GetPeers() + if err != nil { + return false + } + expectedPeers := sets.New[string]("::1-63179", "127.0.0.1-62179") + gotPeers := sets.New[string]() + for _, peer := range peers { + if peer.SessionState != engine.SessionEstablished { + return false + } + gotPeers.Insert(fmt.Sprintf("%s-%d", peer.Address, peer.ASN)) + } + if !expectedPeers.Equal(gotPeers) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got peers of BGP server1 and verified them") + + t.Log("Getting peers of BGP server2 and verifying them") + assert.Eventually(t, func() bool { + peers, err := server2.GetPeers() + if err != nil { + return false + } + expectedPeers := sets.New[string]("127.0.0.1-61179") + gotPeers := sets.New[string]() + for _, peer := range peers { + if peer.SessionState != engine.SessionEstablished { + return false + } + gotPeers.Insert(fmt.Sprintf("%s-%d", peer.Address, peer.ASN)) + } + if !expectedPeers.Equal(gotPeers) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got peers of BGP server2 and verified them") + + t.Log("Getting peers of BGP server3 and verifying them") + assert.Eventually(t, func() bool { + peers, err := server3.GetPeers() + if err != nil { + return false + } + expectedPeers := sets.New[string]("::1-61179") + gotPeers := sets.New[string]() + for _, peer := range peers { + if peer.SessionState != engine.SessionEstablished { + return false + } + gotPeers.Insert(fmt.Sprintf("%s-%d", peer.Address, peer.ASN)) + } + if !expectedPeers.Equal(gotPeers) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got peers of BGP server3 and verified them") + + ipv4Server1Routes := []engine.Route{ + {Prefix: "1.1.0.0/24"}, + {Prefix: "1.2.0.0/24"}, + {Prefix: "1.3.0.0/24"}, + } + ipv6Server1Routes := []engine.Route{ + {Prefix: "1000:1::/64"}, + {Prefix: "1000:2::/64"}, + {Prefix: "1000:3::/64"}, + } + ipv4Server2Routes := []engine.Route{ + {Prefix: "2.1.0.0/24"}, + {Prefix: "2.2.0.0/24"}, + {Prefix: "2.3.0.0/24"}, + } + ipv6Server3Routes := []engine.Route{ + {Prefix: "3000:1::/64"}, + {Prefix: "3000:2::/64"}, + {Prefix: "3000:3::/64"}, + } + + t.Log("Advertising IPv4 and IPv6 routes on BGP server1") + require.NoError(t, server1.AdvertiseRoutes(ipv4Server1Routes, net.IPv4)) + require.NoError(t, server1.AdvertiseRoutes(ipv6Server1Routes, net.IPv6)) + t.Log("Advertised IPv4 and IPv6 routes on BGP server1") + + t.Log("Advertising IPv4 routes on BGP server2") + require.NoError(t, server2.AdvertiseRoutes(ipv4Server2Routes, net.IPv4)) + t.Log("Advertised IPv4 routes on BGP server2") + + t.Log("Advertising IPv6 routes on BGP server3") + require.NoError(t, server3.AdvertiseRoutes(ipv6Server3Routes, net.IPv6)) + t.Log("Advertised IPv6 routes on server3") + + t.Log("Getting received routes of BGP server1 and verifying them") + assert.Eventually(t, func() bool { + // Get the routes advertised by server2 and verify them. + routes, err := server1.GetRoutes(engine.RouteReceived, "127.0.0.1", net.IPv4) + if err != nil { + return false + } + if !assert.ElementsMatch(t, ipv4Server2Routes, routes) { + return false + } + // Get the routes advertised by server3 and verify them. + routes, err = server1.GetRoutes(engine.RouteReceived, "::1", net.IPv6) + if err != nil { + return false + } + if !assert.ElementsMatch(t, ipv6Server3Routes, routes) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got received routes of BGP server1 and verified them") + + t.Log("Getting received routes of BGP server2 and verifying them") + assert.Eventually(t, func() bool { + // Get the routes advertised by server1 and verify them. + routes, err := server2.GetRoutes(engine.RouteReceived, "127.0.0.1", net.IPv4) + if err != nil { + return false + } + if !assert.ElementsMatch(t, ipv4Server1Routes, routes) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got received routes of BGP server2 and verified them") + + t.Log("Getting received routes of BGP server3 and verifying them") + assert.Eventually(t, func() bool { + // Get the routes advertised by server1 and verify them. + routes, err := server3.GetRoutes(engine.RouteReceived, "::1", net.IPv6) + if err != nil { + return false + } + if !assert.ElementsMatch(t, ipv6Server1Routes, routes) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got received routes of BGP server3 and verified them") + + updatedIPv4Server1Routes := []engine.Route{ + {Prefix: "1.1.0.0/24"}, + {Prefix: "1.2.0.0/24"}, + } + ipv4Server1RoutesToWithdraw := []engine.Route{ + {Prefix: "1.3.0.0/24"}, + } + updatedIPv6Server1Routes := []engine.Route{ + {Prefix: "1000:1::/64"}, + {Prefix: "1000:2::/64"}, + } + ipv6Server1RoutesToWithdraw := []engine.Route{ + {Prefix: "1000:3::/64"}, + } + updatedIPv4Server2Routes := []engine.Route{ + {Prefix: "2.1.0.0/24"}, + {Prefix: "2.2.0.0/24"}, + } + ipv4Server2RoutesToWithdraw := []engine.Route{ + {Prefix: "2.3.0.0/24"}, + } + updatedIPv6Server3Routes := []engine.Route{ + {Prefix: "3000:1::/64"}, + {Prefix: "3000:2::/64"}, + } + ipv6Server3RoutesToWithdraw := []engine.Route{ + {Prefix: "3000:3::/64"}, + } + + t.Log("Withdrawing IPv4 and IPv6 routes on BGP server1") + require.NoError(t, server1.WithdrawRoutes(ipv4Server1RoutesToWithdraw, net.IPv4)) + require.NoError(t, server1.WithdrawRoutes(ipv6Server1RoutesToWithdraw, net.IPv6)) + t.Log("Withdrawn IPv4 and IPv6 routes on BGP server1") + + t.Log("Withdrawing IPv4 routes on BGP server2") + require.NoError(t, server2.WithdrawRoutes(ipv4Server2RoutesToWithdraw, net.IPv4)) + t.Log("Withdrawn IPv4 a routes on BGP server2") + + t.Log("Withdrawing IPv6 routes on BGP server3") + require.NoError(t, server3.WithdrawRoutes(ipv6Server3RoutesToWithdraw, net.IPv6)) + t.Log("Withdrawn IPv6 a routes on BGP server3") + + t.Log("Getting received routes of BGP server1 and verifying them") + assert.Eventually(t, func() bool { + // Get the routes advertised by server2 and verify them. + routes, err := server1.GetRoutes(engine.RouteReceived, "127.0.0.1", net.IPv4) + if err != nil { + return false + } + if !assert.ElementsMatch(t, updatedIPv4Server2Routes, routes) { + return false + } + // Get the routes advertised by server3 and verify them. + routes, err = server1.GetRoutes(engine.RouteReceived, "::1", net.IPv6) + if err != nil { + return false + } + if !assert.ElementsMatch(t, updatedIPv6Server3Routes, routes) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got received routes of BGP server1 and verified them") + + t.Log("Getting received routes of BGP server2 and verifying them") + assert.Eventually(t, func() bool { + // Get the routes advertised by server1 and verify them. + routes, err := server2.GetRoutes(engine.RouteReceived, "127.0.0.1", net.IPv4) + if err != nil { + return false + } + if !assert.ElementsMatch(t, updatedIPv4Server1Routes, routes) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got received routes of BGP server2 and verified them") + + t.Log("Getting received routes of BGP server3 and verifying them") + assert.Eventually(t, func() bool { + // Get the routes advertised by server1 and verify them. + routes, err := server3.GetRoutes(engine.RouteReceived, "::1", net.IPv6) + if err != nil { + return false + } + if !assert.ElementsMatch(t, updatedIPv6Server1Routes, routes) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got received routes of BGP server3 and verified them") + + updatedIPv4Server2Config := engine.PeerConfig{ + Address: "127.0.0.1", + Port: 2179, + ASN: 62179, + MultihopTTL: 1, + GracefulRestartTimeSeconds: 180, + } + updatedIPv6Server3Config := engine.PeerConfig{ + Address: "::1", + Port: 3179, + ASN: 63179, + MultihopTTL: 1, + GracefulRestartTimeSeconds: 180, + } + t.Log("Updating peers of BGP server1") + require.NoError(t, server1.UpdatePeers([]engine.PeerConfig{updatedIPv4Server2Config}, net.IPv4)) + require.NoError(t, server1.UpdatePeers([]engine.PeerConfig{updatedIPv6Server3Config}, net.IPv6)) + t.Log("Updated peers of server1") + + t.Log("Getting peers of BGP server1 and verifying them") + assert.Eventually(t, func() bool { + peers, err := server1.GetPeers() + if err != nil { + return false + } + expectedPeers := sets.New[string]("::1-63179", "127.0.0.1-62179") + gotPeers := sets.New[string]() + for _, peer := range peers { + if peer.SessionState != engine.SessionEstablished { + return false + } + if peer.GracefulRestartTimeSeconds != 180 { + return false + } + gotPeers.Insert(fmt.Sprintf("%s-%d", peer.Address, peer.ASN)) + } + if !expectedPeers.Equal(gotPeers) { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got peers of BGP server1 and verified them") + + t.Log("Deleting peers of BGP server1") + require.NoError(t, server1.RemovePeers([]engine.PeerConfig{updatedIPv4Server2Config, updatedIPv6Server3Config})) + t.Log("Deleted peers of BGP server1") + + t.Log("Getting peers of BGP server1 and verifying them") + assert.Eventually(t, func() bool { + peers, err := server1.GetPeers() + if err != nil { + return false + } + if len(peers) != 0 { + return false + } + return true + }, 30*time.Second, time.Second) + t.Log("Got peers of BGP server1 and verified them") + + t.Log("Stopping all BGP servers") + require.NoError(t, server1.Stop()) + require.NoError(t, server2.Stop()) + require.NoError(t, server3.Stop()) + t.Log("Stopped all BGP servers") +}