From 07ba67020b55a95d2e9ca9874ce37c147fae711d Mon Sep 17 00:00:00 2001 From: ykim-1 Date: Wed, 12 Jun 2024 16:19:49 -0700 Subject: [PATCH 1/3] Add calico rules script and update workflow files --- .github/workflows/ci.yml | 14 ++++ .github/workflows/integration_tests_pr.yml | 14 ++++ scripts/lke-policy.yaml | 78 ++++++++++++++++++++++ scripts/lke_calico_rules_e2e.sh | 60 +++++++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 scripts/lke-policy.yaml create mode 100755 scripts/lke_calico_rules_e2e.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2bdb68c0..5f5b53da6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,14 @@ jobs: - name: Install Python deps run: pip3 install requests wheel boto3 + - name: Download kubectl and calicoctl for LKE clusters + run: | + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" + curl -LO "https://github.com/projectcalico/calico/releases/download/v3.25.0/calicoctl-linux-amd64" + chmod +x calicoctl-linux-amd64 kubectl + mv calicoctl-linux-amd64 /usr/local/bin/calicoctl + mv kubectl /usr/local/bin/kubectl + - name: Install go-junit-report run: go install github.com/jstemmer/go-junit-report/v2@latest @@ -64,6 +72,12 @@ jobs: env: SKIP_LINT: 1 + - name: Apply Calico Rules to LKE + run: | + cd scripts && ./lke_calico_rules_e2e.sh + env: + LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }} + - name: Upload test results to bucket if: github.ref == 'refs/heads/main' && github.event_name == 'push' && always() run: | diff --git a/.github/workflows/integration_tests_pr.yml b/.github/workflows/integration_tests_pr.yml index aaee9ccd2..093f48dc7 100644 --- a/.github/workflows/integration_tests_pr.yml +++ b/.github/workflows/integration_tests_pr.yml @@ -36,6 +36,14 @@ jobs: with: ref: ${{ inputs.sha }} + - name: Download kubectl and calicoctl for LKE clusters + run: | + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" + curl -LO "https://github.com/projectcalico/calico/releases/download/v3.25.0/calicoctl-linux-amd64" + chmod +x calicoctl-linux-amd64 kubectl + mv calicoctl-linux-amd64 /usr/local/bin/calicoctl + mv kubectl /usr/local/bin/kubectl + - run: make ARGS="-run ${{ inputs.module }}" fixtures if: ${{ inputs.module != '' && steps.disallowed-char-check.outputs.match == '' }} env: @@ -44,6 +52,12 @@ jobs: if: ${{ inputs.module == '' }} env: LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }} + + - name: Apply Calico Rules to LKE + run: | + cd scripts && ./lke_calico_rules_e2e.sh + env: + LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }} - name: Get the hash value of the latest commit from the PR branch uses: octokit/graphql-action@v2.x diff --git a/scripts/lke-policy.yaml b/scripts/lke-policy.yaml new file mode 100644 index 000000000..d1a8436e8 --- /dev/null +++ b/scripts/lke-policy.yaml @@ -0,0 +1,78 @@ +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: lke-rules +spec: + preDNAT: true + applyOnForward: true + order: 100 + # Remember to run calicoctl patch command for this to work + selector: "" + ingress: + # Allow ICMP + - action: Allow + protocol: ICMP + - action: Allow + protocol: ICMPv6 + + # Allow LKE-required ports + - action: Allow + protocol: TCP + destination: + nets: + - 192.168.128.0/17 + - 10.0.0.0/8 + ports: + - 10250 + - 10256 + - 179 + - action: Allow + protocol: UDP + destination: + nets: + - 192.168.128.0/17 + - 10.2.0.0/16 + ports: + - 51820 + + # Allow NodeBalancer ingress to the Node Ports & Allow DNS + - action: Allow + protocol: TCP + source: + nets: + - 192.168.255.0/24 + - 10.0.0.0/8 + destination: + ports: + - 53 + - 30000:32767 + - action: Allow + protocol: UDP + source: + nets: + - 192.168.255.0/24 + - 10.0.0.0/8 + destination: + ports: + - 53 + - 30000:32767 + + # Allow cluster internal communication + - action: Allow + destination: + nets: + - 10.0.0.0/8 + - action: Allow + source: + nets: + - 10.0.0.0/8 + + # 127.0.0.1/32 is needed for kubectl exec and node-shell + - action: Allow + destination: + nets: + - 127.0.0.1/32 + + # Block everything else + - action: Deny + - action: Log diff --git a/scripts/lke_calico_rules_e2e.sh b/scripts/lke_calico_rules_e2e.sh new file mode 100755 index 000000000..48ad5caec --- /dev/null +++ b/scripts/lke_calico_rules_e2e.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +RETRIES=3 +DELAY=30 + +# Function to retry a command with exponential backoff +retry_command() { + local retries=$1 + local wait_time=60 + shift + until "$@"; do + if ((retries == 0)); then + echo "Command failed after multiple retries. Exiting." + exit 1 + fi + echo "Command failed. Retrying in $wait_time seconds..." + sleep $wait_time + ((retries--)) + wait_time=$((wait_time * 2)) + done +} + +# Fetch the list of LKE cluster IDs +CLUSTER_IDS=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" \ + "https://api.linode.com/v4/lke/clusters" | jq -r '.data[].id') + +# Check if CLUSTER_IDS is empty +if [ -z "$CLUSTER_IDS" ]; then + echo "All clusters have been cleaned and properly destroyed. No need to apply inbound or outbound rules" + exit 0 +fi + +for ID in $CLUSTER_IDS; do + echo "Applying Calico rules to nodes in Cluster ID: $ID" + + # Download cluster configuration file with retry + for ((i=1; i<=RETRIES; i++)); do + config_response=$(curl -sH "Authorization: Bearer $LINODE_TOKEN" "https://api.linode.com/v4/lke/clusters/$ID/kubeconfig") + if [[ $config_response != *"kubeconfig is not yet available"* ]]; then + echo $config_response | jq -r '.[] | @base64d' > "/tmp/${ID}_config.yaml" + break + fi + echo "Attempt $i to download kubeconfig for cluster $ID failed. Retrying in $DELAY seconds..." + sleep $DELAY + done + + if [[ $config_response == *"kubeconfig is not yet available"* ]]; then + echo "kubeconfig for cluster id:$ID not available after $RETRIES attempts, mostly likely it is an empty cluster. Skipping..." + else + # Export downloaded config file + export KUBECONFIG="/tmp/${ID}_config.yaml" + + retry_command $RETRIES kubectl get nodes + + retry_command $RETRIES calicoctl patch kubecontrollersconfiguration default --allow-version-mismatch --patch='{"spec": {"controllers": {"node": {"hostEndpoint": {"autoCreate": "Enabled"}}}}}' + + retry_command $RETRIES calicoctl apply --allow-version-mismatch -f "$(pwd)/lke-policy.yaml" + fi +done From 2a55eade3a0be3b10e72d0590876cf4be415b3df Mon Sep 17 00:00:00 2001 From: ykim-1 Date: Wed, 12 Jun 2024 16:27:28 -0700 Subject: [PATCH 2/3] add condition always --- .github/workflows/ci.yml | 1 + .github/workflows/integration_tests_pr.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f5b53da6..9cbdb0b08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,7 @@ jobs: SKIP_LINT: 1 - name: Apply Calico Rules to LKE + if: always() run: | cd scripts && ./lke_calico_rules_e2e.sh env: diff --git a/.github/workflows/integration_tests_pr.yml b/.github/workflows/integration_tests_pr.yml index 093f48dc7..785a8f2be 100644 --- a/.github/workflows/integration_tests_pr.yml +++ b/.github/workflows/integration_tests_pr.yml @@ -54,6 +54,7 @@ jobs: LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }} - name: Apply Calico Rules to LKE + if: always() run: | cd scripts && ./lke_calico_rules_e2e.sh env: From 720253577de364d2c7553b7a5b2496e0f116824a Mon Sep 17 00:00:00 2001 From: ykim-1 Date: Mon, 17 Jun 2024 08:52:04 -0700 Subject: [PATCH 3/3] revert ci.yml --- .github/workflows/ci.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cbdb0b08..900126459 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,14 +52,6 @@ jobs: - name: Install Python deps run: pip3 install requests wheel boto3 - - name: Download kubectl and calicoctl for LKE clusters - run: | - curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" - curl -LO "https://github.com/projectcalico/calico/releases/download/v3.25.0/calicoctl-linux-amd64" - chmod +x calicoctl-linux-amd64 kubectl - mv calicoctl-linux-amd64 /usr/local/bin/calicoctl - mv kubectl /usr/local/bin/kubectl - - name: Install go-junit-report run: go install github.com/jstemmer/go-junit-report/v2@latest @@ -72,13 +64,6 @@ jobs: env: SKIP_LINT: 1 - - name: Apply Calico Rules to LKE - if: always() - run: | - cd scripts && ./lke_calico_rules_e2e.sh - env: - LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }} - - name: Upload test results to bucket if: github.ref == 'refs/heads/main' && github.event_name == 'push' && always() run: | @@ -91,4 +76,4 @@ jobs: python tod_scripts/test_report_upload_script.py "$REPORT_FILENAME" env: LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }} - LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} + LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} \ No newline at end of file