From 64a9b9d74ff57d840622379243b2da647eb700aa Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 27 Mar 2024 18:43:36 -0700 Subject: [PATCH 001/110] Seed repo --- .gitignore | 65 ++++++++++++++++++++++++++++++++++- .golangci.yml | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 34 +++++------------- docker-user | 1 + version | 1 + 5 files changed, 170 insertions(+), 26 deletions(-) create mode 100644 .golangci.yml create mode 100644 docker-user create mode 100644 version diff --git a/.gitignore b/.gitignore index cb5c33a6..4e1c0fd2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,67 @@ cover* tmp/ docs/tls/DESIGN.md :q -qqq \ No newline at end of file +qqq.env +.env* +!.env.example +!.allowed_clients.json +!.env.example.auth +*.db +priv/certs +priv/nginx-agent/* +!priv/nginx-agent/nginx-agent.conf.example +key-data.json +nginx-instance-manager.tar.gz +vendor/ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Output from debugger +__debug_bin + +code-quality.json +coverage/* + +# vim +*~ +*.swp + +### VisualStudioCode (from https://gitignore.io/api/VisualStudioCode) ### +.vscode/* +!.vscode/tasks.example.json +!.vscode/launch.example.json +!.vscode/extensions.json +!.vscode/KubernetesLocalProcessConfig*.yaml +*.code-workspace + +### Goland +.idea/* + +# bridge to kubernetes artifact +/KubernetesLocalProcessConfig.yaml + + +# output directory for build artifacts +build + +# output directory for test artifacts (eg. coverage report, junit xml) +results + +# devops-utils repo +.devops-utils/ + +# Ignore golang cache in CI +.go/pkg/mod + +.go-build diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..d89f3f5f --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,95 @@ +# GolangCI-Lint settings + +# Disable all linters and enable the required ones +linters: + disable-all: true + + # Supported linters: https://golangci-lint.run/usage/linters/ + enable: + - errcheck + - exhaustruct + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + - bodyclose + - dupl + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - gosec + - lll + - misspell + - nakedret + - prealloc + - exportloopref + - stylecheck + - unconvert + - unparam + - paralleltest + - forbidigo + fast: false + +# Run options +run: + # 10 minute timeout for analysis + timeout: 10m + skip-dirs-use-default: true + skip-dirs: + - .go/pkg/mod + - pkg/spec/api # Generated code + - vendor + - vendor-fork +# Specific linter settings +linters-settings: + gocyclo: + # Minimal code complexity to report + min-complexity: 16 + govet: + # Report shadowed variables + check-shadowing: true + misspell: + # Correct spellings using locale preferences for US + locale: US + goimports: + # Put imports beginning with prefix after 3rd-party packages + local-prefixes: gitswarm.f5net.com/indigo,gitlab.com/f5 + exhaustruct: + # List of regular expressions to match struct packages and names. + # If this list is empty, all structs are tested. + # Default: [] + include: + - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/pkg/token.TokenID' + - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest' + +issues: + # Exclude configuration + exclude-rules: + # Exclude gochecknoinits and gosec from running on tests files + - path: _test\.go + linters: + - gochecknoinits + - gosec + - path: test/* + linters: + - gochecknoinits + - gosec + # Exclude lll issues for long lines with go:generate + - linters: + - lll + source: "^//go:generate " + # Exclude false positive paralleltest error : Range statement for test case does not use range value in test Run + - linters: + - paralleltest + text: "does not use range value in test Run" + + # Disable maximum issues count per one linter + max-issues-per-linter: 0 + + # Disable maximum count of issues with the same text + max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile index 9aa6b8cb..0a12779f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,11 @@ -# Copyright 2023 f5 Inc. All rights reserved. -# Use of this source code is governed by the Apache -# license that can be found in the LICENSE file. +FROM alpine:3.14.1 AS base-certs +RUN apk update && apk add --no-cache ca-certificates -FROM golang:1.19.5-alpine3.16 AS builder +FROM scratch AS base +COPY docker-user /etc/passwd +USER 101 +COPY --from=base-certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -WORKDIR /app - -COPY go.mod go.sum ./ - -RUN go mod download - -COPY . . - -RUN go build -o nginx-loadbalancer-kubernetes ./cmd/nginx-loadbalancer-kubernetes/main.go - -FROM alpine:3.16 - -WORKDIR /opt/nginx-loadbalancer-kubernetes - -RUN adduser -u 11115 -D -H nlk - -USER nlk - -COPY --from=builder /app/nginx-loadbalancer-kubernetes . - -ENTRYPOINT ["/opt/nginx-loadbalancer-kubernetes/nginx-loadbalancer-kubernetes"] +FROM base as nlk +ENTRYPOINT ["/nlk"] +COPY build/nlk / diff --git a/docker-user b/docker-user new file mode 100644 index 00000000..65be48a5 --- /dev/null +++ b/docker-user @@ -0,0 +1 @@ +nginx:x:101:101:nginx:/var/cache/nginx:/sbin/nologin \ No newline at end of file diff --git a/version b/version new file mode 100644 index 00000000..e0212c0d --- /dev/null +++ b/version @@ -0,0 +1 @@ +export VERSION="1.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" From 5200cca180009084f0a30bfb604735992f82f7f1 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 27 Mar 2024 18:44:57 -0700 Subject: [PATCH 002/110] Remove Github-y stuff --- .github/CODEOWNERS | 3 - .github/ISSUE_TEMPLATE/bug_report.md | 32 ------- .github/ISSUE_TEMPLATE/feature_request.md | 22 ----- .github/dependabot.yml | 9 -- .github/pull_request_template.md | 12 --- .github/workflows/build-and-sign-image.yml | 98 ---------------------- .github/workflows/run-scorecard.yml | 72 ---------------- .github/workflows/run-tests.yml | 32 ------- 8 files changed, 280 deletions(-) delete mode 100644 .github/CODEOWNERS delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 .github/dependabot.yml delete mode 100644 .github/pull_request_template.md delete mode 100644 .github/workflows/build-and-sign-image.yml delete mode 100644 .github/workflows/run-scorecard.yml delete mode 100644 .github/workflows/run-tests.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index f330be63..00000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,3 +0,0 @@ -# Main global owner # -##################### -* @ciroque @chrisakker diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index cc6c1d26..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' ---- -### Describe the bug - -A clear and concise description of what the bug is. - -### To reproduce - -Steps to reproduce the behavior: - -1. Deploy nginx_loadbalancer_kubernetes using -2. View output/logs/configuration on '...' -3. See error - -### Expected behavior - -A clear and concise description of what you expected to happen. - -### Your environment - -- Version of the nginx_loadbalancer_kubernetes or specific commit - -- Target deployment platform - -### Additional context - -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index d27aba8e..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' ---- -### Is your feature request related to a problem? Please describe - -A clear and concise description of what the problem is. Ex. I'm always frustrated when ... - -### Describe the solution you'd like - -A clear and concise description of what you want to happen. - -### Describe alternatives you've considered - -A clear and concise description of any alternative solutions or features you've considered. - -### Additional context - -Add any other context or screenshots about the feature request here. diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 4450376b..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -version: 2 -updates: - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - day: monday - time: "00:00" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index fad5aa1e..00000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,12 +0,0 @@ -### Proposed changes - -Describe the use case and detail of the change. If this PR addresses an issue on GitHub, make sure to include a link to that issue using one of the [supported keywords](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue) here in this description (not in the title of the PR). - -### Checklist - -Before creating a PR, run through this checklist and mark each as complete. - -- [ ] I have read the [`CONTRIBUTING`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/CONTRIBUTING.md) document -- [ ] If applicable, I have added tests that prove my fix is effective or that my feature works -- [ ] If applicable, I have checked that any relevant tests pass after adding my changes -- [ ] I have updated any relevant documentation ([`README.md`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/README.md) and [`CHANGELOG.md`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/CHANGELOG.md)) diff --git a/.github/workflows/build-and-sign-image.yml b/.github/workflows/build-and-sign-image.yml deleted file mode 100644 index 2fbf227c..00000000 --- a/.github/workflows/build-and-sign-image.yml +++ /dev/null @@ -1,98 +0,0 @@ -# This workflow will build and push a signed Docker image - -name: Build and sign image - -on: - push: - tags: - - "v[0-9]+.[0-9]+.[0-9]+" -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build_and_sign_image: - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - id-token: write - security-events: write - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: anchore/sbom-action@v0 - with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - output-file: ./nginx-loadbalancer-kubernetes-${{env.GITHUB_REF_NAME}}.spdx.json - registry-username: ${{ github.actor }} - registry-password: ${{ secrets.GITHUB_TOKEN }} - - - name: Install cosign - uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 #v3.0.2 - with: - cosign-release: 'v1.13.1' - - - name: Log into registry ${{ env.REGISTRY }} for ${{ github.actor }} - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build Docker Image - id: docker-build-and-push - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{github.run_number}} - - - name: Sign the published Docker images - env: - COSIGN_EXPERIMENTAL: "true" - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: cosign sign "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.docker-build-and-push.outputs.digest }}" - - # NOTE: This runs statically against the latest tag in Docker Hub which was not produced by this workflow - # This should be updated once this workflow is fully implemented - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@91713af97dc80187565512baba96e4364e983601 # 0.16.0 - continue-on-error: true - with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - format: 'sarif' - output: 'trivy-results-${{ inputs.image }}.sarif' - ignore-unfixed: 'true' - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v2.2.11 - continue-on-error: true - with: - sarif_file: 'trivy-results-${{ inputs.image }}.sarif' - sha: ${{ github.sha }} - ref: ${{ github.ref }} - - - name: Generate Release - uses: ncipollo/release-action@v1 - with: - artifacts: | - trivy-results-${{ inputs.image }}.sarif - ./nginx-loadbalancer-kubernetes-${{env.GITHUB_REF_NAME}}.spdx.json - body: | - # Release ${{env.GITHUB_REF_NAME}} - ## Changelog - ${{ steps.meta.outputs.changelog }} - generateReleaseNotes: true - makeLatest: false - name: "${{env.GITHUB_REF_NAME}}" diff --git a/.github/workflows/run-scorecard.yml b/.github/workflows/run-scorecard.yml deleted file mode 100644 index 3bbad843..00000000 --- a/.github/workflows/run-scorecard.yml +++ /dev/null @@ -1,72 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: '15 14 * * 3' - push: - branches: [ "main" ] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@v4 # v3.1.0 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecard on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 - with: - sarif_file: results.sarif diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index 454c7169..00000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,32 +0,0 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - -name: Run tests - -on: - branch_protection_rule: - types: - - created - - push: - branches: - - main - - * - -jobs: - - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.19 - - - name: Build - run: go build -v ./... - - - name: Test - run: go test -v ./... From 9f8312898311c05ccfcfea3689160f4dc8e7219d Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 29 Mar 2024 12:12:35 -0700 Subject: [PATCH 003/110] Format code --- cmd/certificates-test-harness/main.go | 3 ++- cmd/configuration-test-harness/main.go | 3 ++- cmd/nginx-loadbalancer-kubernetes/main.go | 1 + cmd/tls-config-factory-test-harness/main.go | 3 ++- internal/application/application_common_test.go | 1 + internal/application/border_client.go | 1 + internal/application/border_client_test.go | 3 ++- internal/application/nginx_http_border_client.go | 1 + internal/application/nginx_stream_border_client.go | 1 + internal/authentication/factory.go | 1 + internal/certification/certificates_test.go | 5 +++-- internal/communication/factory.go | 5 +++-- internal/communication/factory_test.go | 3 ++- internal/communication/roundtripper_test.go | 5 +++-- internal/configuration/settings.go | 5 +++-- internal/core/event_test.go | 3 ++- internal/observation/handler.go | 1 + internal/observation/handler_test.go | 3 ++- internal/observation/watcher.go | 3 ++- internal/observation/watcher_test.go | 3 ++- internal/probation/server.go | 3 ++- internal/probation/server_test.go | 5 +++-- internal/synchronization/synchronizer.go | 1 + internal/synchronization/synchronizer_test.go | 3 ++- internal/translation/translator.go | 3 ++- internal/translation/translator_test.go | 7 ++++--- 26 files changed, 51 insertions(+), 25 deletions(-) diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index 44d4a4e6..d2b746bc 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -4,13 +4,14 @@ import ( "context" "errors" "fmt" + "path/filepath" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" - "path/filepath" ) func main() { diff --git a/cmd/configuration-test-harness/main.go b/cmd/configuration-test-harness/main.go index 56e8b5dd..5079a9d0 100644 --- a/cmd/configuration-test-harness/main.go +++ b/cmd/configuration-test-harness/main.go @@ -4,13 +4,14 @@ import ( "context" "errors" "fmt" + "path/filepath" + configuration2 "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" - "path/filepath" ) func main() { diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 69365579..07319653 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/observation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/probation" diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 3f46d4fd..1813bd93 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -3,12 +3,13 @@ package main import ( "bufio" "fmt" + "os" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" - "os" ) const ( diff --git a/internal/application/application_common_test.go b/internal/application/application_common_test.go index e963d03d..f7e95615 100644 --- a/internal/application/application_common_test.go +++ b/internal/application/application_common_test.go @@ -7,6 +7,7 @@ package application import ( "errors" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" ) diff --git a/internal/application/border_client.go b/internal/application/border_client.go index a5cc93e0..0ab6be63 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -7,6 +7,7 @@ package application import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" ) diff --git a/internal/application/border_client_test.go b/internal/application/border_client_test.go index 0b8105ec..60ac41f5 100644 --- a/internal/application/border_client_test.go +++ b/internal/application/border_client_test.go @@ -6,8 +6,9 @@ package application import ( - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" ) func TestBorderClient_CreatesHttpBorderClient(t *testing.T) { diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index b7657c5a..78db7738 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -7,6 +7,7 @@ package application import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index 46cd4985..a0adff08 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -7,6 +7,7 @@ package application import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index 8b8d06ec..69c5ee71 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -12,6 +12,7 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/sirupsen/logrus" diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index c8edf14e..d9d61ac9 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -7,12 +7,13 @@ package certification import ( "context" + "testing" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" - "testing" - "time" ) const ( diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 9a3d4113..8b194908 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -7,11 +7,12 @@ package communication import ( "crypto/tls" + netHttp "net/http" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/sirupsen/logrus" - netHttp "net/http" - "time" ) // NewHttpClient is a factory method to create a new Http Client with a default configuration. diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index f25abefb..f95722da 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -7,9 +7,10 @@ package communication import ( "context" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "k8s.io/client-go/kubernetes/fake" - "testing" ) func TestNewHttpClient(t *testing.T) { diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index f46fb710..0a549d61 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -8,10 +8,11 @@ package communication import ( "bytes" "context" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" netHttp "net/http" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "k8s.io/client-go/kubernetes/fake" ) func TestNewRoundTripper(t *testing.T) { diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index d15ad845..e6bd30b5 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -8,6 +8,9 @@ package configuration import ( "context" "fmt" + "strings" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" @@ -16,8 +19,6 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "strings" - "time" ) const ( diff --git a/internal/core/event_test.go b/internal/core/event_test.go index b3b89261..7f7448d4 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -1,8 +1,9 @@ package core import ( - v1 "k8s.io/api/core/v1" "testing" + + v1 "k8s.io/api/core/v1" ) func TestNewEvent(t *testing.T) { diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 83601b0f..8e897452 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -7,6 +7,7 @@ package observation import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index f4a617f8..e5bef3d2 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -8,12 +8,13 @@ package observation import ( "context" "fmt" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" v1 "k8s.io/api/core/v1" "k8s.io/client-go/util/workqueue" - "testing" ) func TestHandler_AddsEventToSynchronizer(t *testing.T) { diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 3ee9d3fe..74e2d05e 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -8,6 +8,8 @@ package observation import ( "errors" "fmt" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" @@ -16,7 +18,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" "k8s.io/client-go/tools/cache" - "time" ) // Watcher is responsible for watching for changes to Kubernetes resources. diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 36d64ee2..7c820bf6 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -7,10 +7,11 @@ package observation import ( "context" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" "k8s.io/client-go/kubernetes" - "testing" ) func TestWatcher_MustInitialize(t *testing.T) { diff --git a/internal/probation/server.go b/internal/probation/server.go index 12b16993..ff23e339 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -7,8 +7,9 @@ package probation import ( "fmt" - "github.com/sirupsen/logrus" "net/http" + + "github.com/sirupsen/logrus" ) const ( diff --git a/internal/probation/server_test.go b/internal/probation/server_test.go index f594bffe..4ea51e7a 100644 --- a/internal/probation/server_test.go +++ b/internal/probation/server_test.go @@ -6,10 +6,11 @@ package probation import ( - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "github.com/sirupsen/logrus" "net/http" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" + "github.com/sirupsen/logrus" ) func TestHealthServer_HandleLive(t *testing.T) { diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 1061b014..5fc07f55 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -7,6 +7,7 @@ package synchronization import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index 315def73..ef510b85 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -8,10 +8,11 @@ package synchronization import ( "context" "fmt" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "testing" ) func TestSynchronizer_NewSynchronizer(t *testing.T) { diff --git a/internal/translation/translator.go b/internal/translation/translator.go index b2d0e87c..98ff3684 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -7,12 +7,13 @@ package translation import ( "fmt" + "strings" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" - "strings" ) // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 2acfd34d..a393f642 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -7,12 +7,13 @@ package translation import ( "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - v1 "k8s.io/api/core/v1" "math/rand" "testing" "time" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" + v1 "k8s.io/api/core/v1" ) const ( From 1a95954830e5d31348ad8cca22ea2e34b28fe42c Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 29 Mar 2024 12:33:13 -0700 Subject: [PATCH 004/110] Lint code --- cmd/certificates-test-harness/doc.go | 3 +- cmd/certificates-test-harness/main.go | 2 +- cmd/nginx-loadbalancer-kubernetes/main.go | 14 ++---- cmd/tls-config-factory-test-harness/main.go | 45 ++++++++--------- doc.go | 2 +- .../application/application_common_test.go | 4 +- internal/application/application_constants.go | 7 +-- internal/application/border_client.go | 4 +- internal/application/border_client_test.go | 7 ++- internal/application/doc.go | 5 +- .../application/nginx_client_interface.go | 17 +++++-- .../application/nginx_http_border_client.go | 27 ++++++----- .../nginx_http_border_client_test.go | 26 ++++++---- .../application/nginx_stream_border_client.go | 5 +- .../nginx_stream_border_client_test.go | 12 +++-- internal/application/null_border_client.go | 3 +- .../application/null_border_client_test.go | 2 + internal/authentication/factory.go | 31 ++++++------ internal/authentication/factory_test.go | 48 +++++++++++-------- internal/certification/certificates.go | 12 ++--- internal/certification/certificates_test.go | 6 +++ internal/communication/factory.go | 14 +++--- internal/communication/factory_test.go | 12 +++-- internal/communication/roundtripper.go | 3 +- internal/communication/roundtripper_test.go | 10 +++- internal/configuration/settings.go | 38 ++++++++------- internal/configuration/tlsmodes_test.go | 2 + internal/core/event_test.go | 1 + internal/core/secret_bytes_test.go | 2 + internal/core/server_update_event.go | 15 ++++-- internal/core/server_update_event_test.go | 23 +++++---- internal/core/upstream_server.go | 3 +- internal/core/upstream_server_test.go | 1 + internal/observation/handler.go | 6 ++- internal/observation/handler_test.go | 7 ++- internal/observation/watcher.go | 28 ++++++----- internal/observation/watcher_test.go | 1 + internal/probation/check_test.go | 3 ++ internal/probation/server.go | 3 +- internal/probation/server_test.go | 6 +++ internal/synchronization/rand.go | 5 +- internal/synchronization/synchronizer.go | 31 +++++++----- internal/synchronization/synchronizer_test.go | 32 ++++++++++++- internal/translation/translator.go | 24 ++++++---- internal/translation/translator_test.go | 34 ++++++++++++- test/mocks/mock_nginx_plus_client.go | 10 +++- 46 files changed, 388 insertions(+), 208 deletions(-) diff --git a/cmd/certificates-test-harness/doc.go b/cmd/certificates-test-harness/doc.go index 538bed98..2d76fd59 100644 --- a/cmd/certificates-test-harness/doc.go +++ b/cmd/certificates-test-harness/doc.go @@ -4,7 +4,8 @@ */ /* -Package certificates_test_harness includes functionality boostrap and test the certification.Certificates implplementation. +Package certificates_test_harness includes functionality boostrap +and test the certification.Certificates implplementation. */ package main diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index d2b746bc..056d3033 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -40,7 +40,7 @@ func run() error { return fmt.Errorf(`error occurred initializing certificates: %w`, err) } - go certificates.Run() + go certificates.Run() //nolint:errcheck <-ctx.Done() return nil diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 07319653..89f764f4 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -47,20 +47,14 @@ func run() error { go settings.Run() - synchronizerWorkqueue, err := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - if err != nil { - return fmt.Errorf(`error occurred building a workqueue: %w`, err) - } + synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) synchronizer, err := synchronization.NewSynchronizer(settings, synchronizerWorkqueue) if err != nil { return fmt.Errorf(`error initializing synchronizer: %w`, err) } - handlerWorkqueue, err := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - if err != nil { - return fmt.Errorf(`error occurred building a workqueue: %w`, err) - } + handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue) @@ -106,9 +100,9 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { return client, nil } -func buildWorkQueue(settings configuration.WorkQueueSettings) (workqueue.RateLimitingInterface, error) { +func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { logrus.Debug("Watcher::buildSynchronizerWorkQueue") rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) - return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name), nil + return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) } diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 1813bd93..51f1d1d9 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "fmt" "os" "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" @@ -17,7 +16,7 @@ const ( ClientCertificateSecretKey = "nlk-tls-client-secret" ) -type TlsConfiguration struct { +type TLSConfiguration struct { Description string Settings configuration.Settings } @@ -28,11 +27,11 @@ func main() { configurations := buildConfigMap() for name, settings := range configurations { - fmt.Print("\033[H\033[2J") + logrus.Infof("\033[H\033[2J") logrus.Infof("\n\n\t*** Building TLS config for <<< %s >>>\n\n", name) - tlsConfig, err := authentication.NewTlsConfig(&settings.Settings) + tlsConfig, err := authentication.NewTLSConfig(&settings.Settings) if err != nil { panic(err) } @@ -41,41 +40,43 @@ func main() { certificateCount := 0 if tlsConfig.RootCAs != nil { - rootCaCount = len(tlsConfig.RootCAs.Subjects()) + rootCaCount = len(tlsConfig.RootCAs.Subjects()) //nolint:staticcheck } if tlsConfig.Certificates != nil { certificateCount = len(tlsConfig.Certificates) } - logrus.Infof("Successfully built TLS config: \n\tDescription: %s \n\tRootCA count: %v\n\tCertificate count: %v", settings.Description, rootCaCount, certificateCount) + logrus.Infof("Successfully built TLS config: \n\tDescription: %s \n\tRootCA count: %v\n\tCertificate count: %v", + settings.Description, rootCaCount, certificateCount, + ) - bufio.NewReader(os.Stdin).ReadBytes('\n') + _, _ = bufio.NewReader(os.Stdin).ReadBytes('\n') } - fmt.Print("\033[H\033[2J") + logrus.Infof("\033[H\033[2J") logrus.Infof("\n\n\t*** All done! ***\n\n") } -func buildConfigMap() map[string]TlsConfiguration { - configurations := make(map[string]TlsConfiguration) +func buildConfigMap() map[string]TLSConfiguration { + configurations := make(map[string]TLSConfiguration) - configurations["ss-tls"] = TlsConfiguration{ + configurations["ss-tls"] = TLSConfiguration{ Description: "Self-signed TLS requires just a CA certificate", - Settings: ssTlsConfig(), + Settings: ssTLSConfig(), } - configurations["ss-mtls"] = TlsConfiguration{ + configurations["ss-mtls"] = TLSConfiguration{ Description: "Self-signed mTLS requires a CA certificate and a client certificate", Settings: ssMtlsConfig(), } - configurations["ca-tls"] = TlsConfiguration{ + configurations["ca-tls"] = TLSConfiguration{ Description: "CA TLS requires no certificates", - Settings: caTlsConfig(), + Settings: caTLSConfig(), } - configurations["ca-mtls"] = TlsConfiguration{ + configurations["ca-mtls"] = TLSConfiguration{ Description: "CA mTLS requires a client certificate", Settings: caMtlsConfig(), } @@ -83,13 +84,13 @@ func buildConfigMap() map[string]TlsConfiguration { return configurations } -func ssTlsConfig() configuration.Settings { +func ssTLSConfig() configuration.Settings { certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) return configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, @@ -102,16 +103,16 @@ func ssMtlsConfig() configuration.Settings { certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) return configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } } -func caTlsConfig() configuration.Settings { +func caTLSConfig() configuration.Settings { return configuration.Settings{ - TlsMode: configuration.CertificateAuthorityTLS, + TLSMode: configuration.CertificateAuthorityTLS, } } @@ -120,7 +121,7 @@ func caMtlsConfig() configuration.Settings { certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) return configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, + TLSMode: configuration.CertificateAuthorityMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, diff --git a/doc.go b/doc.go index 7c97bd20..1034f144 100644 --- a/doc.go +++ b/doc.go @@ -3,4 +3,4 @@ * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ -package kubernetes_nginx_ingress +package kubernetesnginxingress diff --git a/internal/application/application_common_test.go b/internal/application/application_common_test.go index f7e95615..c42bc04c 100644 --- a/internal/application/application_common_test.go +++ b/internal/application/application_common_test.go @@ -19,11 +19,11 @@ const ( server = "server" ) -func buildTerrorizingBorderClient(clientType string) (Interface, *mocks.MockNginxClient, error) { +func buildTerrorizingBorderClient(clientType string) (Interface, error) { nginxClient := mocks.NewErroringMockClient(errors.New(`something went horribly horribly wrong`)) bc, err := NewBorderClient(clientType, nginxClient) - return bc, nginxClient, err + return bc, err } func buildBorderClient(clientType string) (Interface, *mocks.MockNginxClient, error) { diff --git a/internal/application/application_constants.go b/internal/application/application_constants.go index 0ec18264..4cb23a54 100644 --- a/internal/application/application_constants.go +++ b/internal/application/application_constants.go @@ -12,7 +12,8 @@ package application // annotations: // nginxinc.io/nlk-: // -// where is the name of the upstream in the NGINX Plus configuration and is one of the constants below. +// where is the name of the upstream in the NGINX Plus configuration +// and is one of the constants below. // // Note, this is an extensibility point. To add a Border Server client... // 1. Create a module that implements the BorderClient interface; @@ -23,6 +24,6 @@ const ( // ClientTypeNginxStream creates a NginxStreamBorderClient that uses the Stream* methods of the NGINX Plus client. ClientTypeNginxStream = "stream" - // ClientTypeNginxHttp creates an NginxHttpBorderClient that uses the HTTP* methods of the NGINX Plus client. - ClientTypeNginxHttp = "http" + // ClientTypeNginxHTTP creates an NginxHTTPBorderClient that uses the HTTP* methods of the NGINX Plus client. + ClientTypeNginxHTTP = "http" ) diff --git a/internal/application/border_client.go b/internal/application/border_client.go index 0ab6be63..a5109532 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -35,8 +35,8 @@ func NewBorderClient(clientType string, borderClient interface{}) (Interface, er case ClientTypeNginxStream: return NewNginxStreamBorderClient(borderClient) - case ClientTypeNginxHttp: - return NewNginxHttpBorderClient(borderClient) + case ClientTypeNginxHTTP: + return NewNginxHTTPBorderClient(borderClient) default: borderClient, _ := NewNullBorderClient() diff --git a/internal/application/border_client_test.go b/internal/application/border_client_test.go index 60ac41f5..8960eee7 100644 --- a/internal/application/border_client_test.go +++ b/internal/application/border_client_test.go @@ -12,18 +12,20 @@ import ( ) func TestBorderClient_CreatesHttpBorderClient(t *testing.T) { + t.Parallel() borderClient := mocks.MockNginxClient{} client, err := NewBorderClient("http", borderClient) if err != nil { t.Errorf(`error creating border client: %v`, err) } - if _, ok := client.(*NginxHttpBorderClient); !ok { - t.Errorf(`expected client to be of type NginxHttpBorderClient`) + if _, ok := client.(*NginxHTTPBorderClient); !ok { + t.Errorf(`expected client to be of type NginxHTTPBorderClient`) } } func TestBorderClient_CreatesTcpBorderClient(t *testing.T) { + t.Parallel() borderClient := mocks.MockNginxClient{} client, err := NewBorderClient("stream", borderClient) if err != nil { @@ -36,6 +38,7 @@ func TestBorderClient_CreatesTcpBorderClient(t *testing.T) { } func TestBorderClient_UnknownClientType(t *testing.T) { + t.Parallel() unknownClientType := "unknown" borderClient := mocks.MockNginxClient{} client, err := NewBorderClient(unknownClientType, borderClient) diff --git a/internal/application/doc.go b/internal/application/doc.go index 34c27d0e..296cb67c 100644 --- a/internal/application/doc.go +++ b/internal/application/doc.go @@ -17,7 +17,7 @@ To add a Border Server client... At this time the only supported Border Servers are NGINX Plus servers. The two Border Server clients for NGINX Plus are: -- NginxHttpBorderClient: updates NGINX Plus servers using HTTP Upstream methods on the NGINX Plus API. +- NginxHTTPBorderClient: updates NGINX Plus servers using HTTP Upstream methods on the NGINX Plus API. - NginxStreamBorderClient: updates NGINX Plus servers using Stream Upstream methods on the NGINX Plus API. Both of these implementations use the NGINX Plus client module to communicate with the NGINX Plus server. @@ -27,7 +27,8 @@ Selection of the appropriate client is based on the Annotations present on the S annotations: nginxinc.io/nlk-: -where is the name of the upstream in the NGINX Plus configuration and is one of the constants in application_constants.go. +where is the name of the upstream in the NGINX Plus configuration +and is one of the constants in application_constants.go. */ package application diff --git a/internal/application/nginx_client_interface.go b/internal/application/nginx_client_interface.go index 728db1e3..31a7b946 100644 --- a/internal/application/nginx_client_interface.go +++ b/internal/application/nginx_client_interface.go @@ -7,17 +7,24 @@ package application import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" -// NginxClientInterface defines the functions used on the NGINX Plus client, abstracting away the full details of that client. +// NginxClientInterface defines the functions used on the NGINX Plus client, +// abstracting away the full details of that client. type NginxClientInterface interface { // DeleteStreamServer is used by the NginxStreamBorderClient. DeleteStreamServer(upstream string, server string) error // UpdateStreamServers is used by the NginxStreamBorderClient. - UpdateStreamServers(upstream string, servers []nginxClient.StreamUpstreamServer) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) + UpdateStreamServers( + upstream string, + servers []nginxClient.StreamUpstreamServer, + ) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) - // DeleteHTTPServer is used by the NginxHttpBorderClient. + // DeleteHTTPServer is used by the NginxHTTPBorderClient. DeleteHTTPServer(upstream string, server string) error - // UpdateHTTPServers is used by the NginxHttpBorderClient. - UpdateHTTPServers(upstream string, servers []nginxClient.UpstreamServer) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) + // UpdateHTTPServers is used by the NginxHTTPBorderClient. + UpdateHTTPServers( + upstream string, + servers []nginxClient.UpstreamServer, + ) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) } diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index 78db7738..e9e754a6 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_stream_border_client.go +//nolint:dupl package application import ( @@ -13,26 +14,26 @@ import ( ) // NginxHttpBorderClient implements the BorderClient interface for HTTP upstreams. -type NginxHttpBorderClient struct { +type NginxHTTPBorderClient struct { BorderClient nginxClient NginxClientInterface } -// NewNginxHttpBorderClient is the Factory function for creating an NginxHttpBorderClient. -func NewNginxHttpBorderClient(client interface{}) (Interface, error) { +// NewNginxHTTPBorderClient is the Factory function for creating an NewNginxHTTPBorderClient. +func NewNginxHTTPBorderClient(client interface{}) (Interface, error) { ngxClient, ok := client.(NginxClientInterface) if !ok { return nil, fmt.Errorf(`expected a NginxClientInterface, got a %v`, client) } - return &NginxHttpBorderClient{ + return &NginxHTTPBorderClient{ nginxClient: ngxClient, }, nil } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHttpBorderClient) Update(event *core.ServerUpdateEvent) error { - httpUpstreamServers := asNginxHttpUpstreamServers(event.UpstreamServers) +func (hbc *NginxHTTPBorderClient) Update(event *core.ServerUpdateEvent) error { + httpUpstreamServers := asNginxHTTPUpstreamServers(event.UpstreamServers) _, _, _, err := hbc.nginxClient.UpdateHTTPServers(event.UpstreamName, httpUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) @@ -42,7 +43,7 @@ func (hbc *NginxHttpBorderClient) Update(event *core.ServerUpdateEvent) error { } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHttpBorderClient) Delete(event *core.ServerUpdateEvent) error { +func (hbc *NginxHTTPBorderClient) Delete(event *core.ServerUpdateEvent) error { err := hbc.nginxClient.DeleteHTTPServer(event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) @@ -52,18 +53,18 @@ func (hbc *NginxHttpBorderClient) Delete(event *core.ServerUpdateEvent) error { } // asNginxHttpUpstreamServer converts a core.UpstreamServer to a nginxClient.UpstreamServer. -func asNginxHttpUpstreamServer(server *core.UpstreamServer) nginxClient.UpstreamServer { +func asNginxHTTPUpstreamServer(server *core.UpstreamServer) nginxClient.UpstreamServer { return nginxClient.UpstreamServer{ Server: server.Host, } } -// asNginxHttpUpstreamServers converts a core.UpstreamServers to a []nginxClient.UpstreamServer. -func asNginxHttpUpstreamServers(servers core.UpstreamServers) []nginxClient.UpstreamServer { - var upstreamServers []nginxClient.UpstreamServer +// asNginxHTTPUpstreamServers converts a core.UpstreamServers to a []nginxClient.UpstreamServer. +func asNginxHTTPUpstreamServers(servers core.UpstreamServers) []nginxClient.UpstreamServer { + upstreamServers := []nginxClient.UpstreamServer{} for _, server := range servers { - upstreamServers = append(upstreamServers, asNginxHttpUpstreamServer(server)) + upstreamServers = append(upstreamServers, asNginxHTTPUpstreamServer(server)) } return upstreamServers diff --git a/internal/application/nginx_http_border_client_test.go b/internal/application/nginx_http_border_client_test.go index defc2ef8..039b4ecd 100644 --- a/internal/application/nginx_http_border_client_test.go +++ b/internal/application/nginx_http_border_client_test.go @@ -3,6 +3,9 @@ * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ +// dupl complains about duplicates with nginx_stream_border_client_test.go +// +//nolint:dupl package application import ( @@ -10,8 +13,9 @@ import ( ) func TestHttpBorderClient_Delete(t *testing.T) { - event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHttp) - borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHTTP) + borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -27,8 +31,9 @@ func TestHttpBorderClient_Delete(t *testing.T) { } func TestHttpBorderClient_Update(t *testing.T) { - event := buildServerUpdateEvent(createEventType, ClientTypeNginxHttp) - borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(createEventType, ClientTypeNginxHTTP) + borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -44,16 +49,18 @@ func TestHttpBorderClient_Update(t *testing.T) { } func TestHttpBorderClient_BadNginxClient(t *testing.T) { + t.Parallel() var emptyInterface interface{} - _, err := NewBorderClient(ClientTypeNginxHttp, emptyInterface) + _, err := NewBorderClient(ClientTypeNginxHTTP, emptyInterface) if err == nil { t.Fatalf(`expected an error to occur when creating a new border client`) } } func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { - event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHttp) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHTTP) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -66,8 +73,9 @@ func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { } func TestHttpBorderClient_UpdateReturnsError(t *testing.T) { - event := buildServerUpdateEvent(createEventType, ClientTypeNginxHttp) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(createEventType, ClientTypeNginxHTTP) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index a0adff08..19982b45 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_http_border_client.go +//nolint:dupl package application import ( @@ -58,7 +59,7 @@ func asNginxStreamUpstreamServer(server *core.UpstreamServer) nginxClient.Stream } func asNginxStreamUpstreamServers(servers core.UpstreamServers) []nginxClient.StreamUpstreamServer { - var upstreamServers []nginxClient.StreamUpstreamServer + upstreamServers := []nginxClient.StreamUpstreamServer{} for _, server := range servers { upstreamServers = append(upstreamServers, asNginxStreamUpstreamServer(server)) diff --git a/internal/application/nginx_stream_border_client_test.go b/internal/application/nginx_stream_border_client_test.go index ddcb3466..c86a7767 100644 --- a/internal/application/nginx_stream_border_client_test.go +++ b/internal/application/nginx_stream_border_client_test.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_http_border_client_test.go +//nolint:dupl package application import ( @@ -10,6 +11,7 @@ import ( ) func TestTcpBorderClient_Delete(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxStream) borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxStream) if err != nil { @@ -27,6 +29,7 @@ func TestTcpBorderClient_Delete(t *testing.T) { } func TestTcpBorderClient_Update(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(createEventType, ClientTypeNginxStream) borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxStream) if err != nil { @@ -44,6 +47,7 @@ func TestTcpBorderClient_Update(t *testing.T) { } func TestTcpBorderClient_BadNginxClient(t *testing.T) { + t.Parallel() var emptyInterface interface{} _, err := NewBorderClient(ClientTypeNginxStream, emptyInterface) if err == nil { @@ -52,8 +56,9 @@ func TestTcpBorderClient_BadNginxClient(t *testing.T) { } func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxStream) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxStream) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -66,8 +71,9 @@ func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { } func TestTcpBorderClient_UpdateReturnsError(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(createEventType, ClientTypeNginxStream) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxStream) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } diff --git a/internal/application/null_border_client.go b/internal/application/null_border_client.go index 8370fe02..b59ca229 100644 --- a/internal/application/null_border_client.go +++ b/internal/application/null_border_client.go @@ -11,7 +11,8 @@ import ( ) // NullBorderClient is a BorderClient that does nothing. -// / It serves only to prevent a panic if the BorderClient is not set correctly and errors from the factory methods are ignored. +// It serves only to prevent a panic if the BorderClient +// is not set correctly and errors from the factory methods are ignored. type NullBorderClient struct { } diff --git a/internal/application/null_border_client_test.go b/internal/application/null_border_client_test.go index 42e9dfb0..f973949f 100644 --- a/internal/application/null_border_client_test.go +++ b/internal/application/null_border_client_test.go @@ -8,6 +8,7 @@ package application import "testing" func TestNullBorderClient_Delete(t *testing.T) { + t.Parallel() client := NullBorderClient{} err := client.Delete(nil) if err != nil { @@ -16,6 +17,7 @@ func TestNullBorderClient_Delete(t *testing.T) { } func TestNullBorderClient_Update(t *testing.T) { + t.Parallel() client := NullBorderClient{} err := client.Update(nil) if err != nil { diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index 69c5ee71..32add620 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -18,37 +18,38 @@ import ( "github.com/sirupsen/logrus" ) -func NewTlsConfig(settings *configuration.Settings) (*tls.Config, error) { - logrus.Debugf("authentication::NewTlsConfig Creating TLS config for mode: '%s'", settings.TlsMode) - switch settings.TlsMode { +func NewTLSConfig(settings *configuration.Settings) (*tls.Config, error) { + logrus.Debugf("authentication::NewTLSConfig Creating TLS config for mode: '%s'", settings.TLSMode) + switch settings.TLSMode { case configuration.NoTLS: - return buildBasicTlsConfig(true), nil + return buildBasicTLSConfig(true), nil case configuration.SelfSignedTLS: // needs ca cert - return buildSelfSignedTlsConfig(settings.Certificates) + return buildSelfSignedTLSConfig(settings.Certificates) case configuration.SelfSignedMutualTLS: // needs ca cert and client cert return buildSelfSignedMtlsConfig(settings.Certificates) case configuration.CertificateAuthorityTLS: // needs nothing - return buildBasicTlsConfig(false), nil + return buildBasicTLSConfig(false), nil case configuration.CertificateAuthorityMutualTLS: // needs client cert - return buildCaTlsConfig(settings.Certificates) + return buildCATLSConfig(settings.Certificates) default: - return nil, fmt.Errorf("unknown TLS mode: %s", settings.TlsMode) + return nil, fmt.Errorf("unknown TLS mode: %s", settings.TLSMode) } } -func buildSelfSignedTlsConfig(certificates *certification.Certificates) (*tls.Config, error) { +func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Config, error) { logrus.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) if err != nil { return nil, err } + //nolint:gosec return &tls.Config{ InsecureSkipVerify: false, RootCAs: certPool, @@ -68,6 +69,7 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C } logrus.Debugf("buildSelfSignedMtlsConfig Certificate: %v", certificate) + //nolint:gosec return &tls.Config{ InsecureSkipVerify: false, RootCAs: certPool, @@ -76,20 +78,21 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C }, nil } -func buildBasicTlsConfig(skipVerify bool) *tls.Config { - logrus.Debugf("authentication::buildBasicTlsConfig skipVerify(%v)", skipVerify) +func buildBasicTLSConfig(skipVerify bool) *tls.Config { + logrus.Debugf("authentication::buildBasicTLSConfig skipVerify(%v)", skipVerify) return &tls.Config{ - InsecureSkipVerify: skipVerify, + InsecureSkipVerify: skipVerify, //nolint:gosec } } -func buildCaTlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildCaTlsConfig") +func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, error) { + logrus.Debug("authentication::buildCATLSConfig") certificate, err := buildCertificates(certificates.GetClientCertificate()) if err != nil { return nil, err } + //nolint:gosec return &tls.Config{ InsecureSkipVerify: false, Certificates: []tls.Certificate{certificate}, diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go index a5352007..e9015c07 100644 --- a/internal/authentication/factory_test.go +++ b/internal/authentication/factory_test.go @@ -19,9 +19,10 @@ const ( ) func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { + t.Parallel() settings := configuration.Settings{} - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -36,11 +37,12 @@ func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { } func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -48,7 +50,7 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { }, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -71,17 +73,18 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { } func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -92,11 +95,12 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { } func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -104,7 +108,7 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -115,12 +119,13 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) } func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -128,7 +133,7 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -151,18 +156,19 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { } func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -173,12 +179,13 @@ func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { } func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -186,7 +193,7 @@ func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -197,11 +204,12 @@ func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { } func TestTlsFactory_CaTlsMode(t *testing.T) { + t.Parallel() settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityTLS, + TLSMode: configuration.CertificateAuthorityTLS, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -224,11 +232,12 @@ func TestTlsFactory_CaTlsMode(t *testing.T) { } func TestTlsFactory_CaMtlsMode(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, + TLSMode: configuration.CertificateAuthorityMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -236,7 +245,7 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -259,18 +268,19 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { } func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, + TLSMode: configuration.CertificateAuthorityMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go index 53bd8435..3ecbc46c 100644 --- a/internal/certification/certificates.go +++ b/internal/certification/certificates.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. * - * Establishes a Watcher for the Kubernetes Secrets that contain the various certificates and keys used to generate a tls.Config object; + * Establishes a Watcher for the Kubernetes Secrets that contain the various certificates + * and keys used to generate a tls.Config object; * exposes the certificates and keys. */ @@ -86,10 +87,7 @@ func (c *Certificates) Initialize() error { c.Certificates = make(map[string]map[string]core.SecretBytes) - informer, err := c.buildInformer() - if err != nil { - return fmt.Errorf(`error occurred building an informer: %w`, err) - } + informer := c.buildInformer() c.informer = informer @@ -116,14 +114,14 @@ func (c *Certificates) Run() error { return nil } -func (c *Certificates) buildInformer() (cache.SharedInformer, error) { +func (c *Certificates) buildInformer() cache.SharedInformer { logrus.Debug("Certificates::buildInformer") options := informers.WithNamespace(SecretsNamespace) factory := informers.NewSharedInformerFactoryWithOptions(c.k8sClient, 0, options) informer := factory.Core().V1().Secrets().Informer() - return informer, nil + return informer } func (c *Certificates) initializeEventHandlers() error { diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index d9d61ac9..89b21bf9 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -21,6 +21,7 @@ const ( ) func TestNewCertificate(t *testing.T) { + t.Parallel() ctx := context.Background() certificates := NewCertificates(ctx, nil) @@ -31,6 +32,7 @@ func TestNewCertificate(t *testing.T) { } func TestCertificates_Initialize(t *testing.T) { + t.Parallel() certificates := NewCertificates(context.Background(), nil) err := certificates.Initialize() @@ -40,6 +42,7 @@ func TestCertificates_Initialize(t *testing.T) { } func TestCertificates_RunWithoutInitialize(t *testing.T) { + t.Parallel() certificates := NewCertificates(context.Background(), nil) err := certificates.Run() @@ -53,6 +56,7 @@ func TestCertificates_RunWithoutInitialize(t *testing.T) { } func TestCertificates_EmptyCertificates(t *testing.T) { + t.Parallel() certificates := NewCertificates(context.Background(), nil) err := certificates.Initialize() @@ -75,6 +79,7 @@ func TestCertificates_EmptyCertificates(t *testing.T) { } func TestCertificates_ExerciseHandlers(t *testing.T) { + t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -86,6 +91,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { certificates.CaCertificateSecretKey = CaCertificateSecretKey + //nolint:govet,staticcheck go func() { err := certificates.Run() if err != nil { diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 8b194908..40bf02a7 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -15,12 +15,12 @@ import ( "github.com/sirupsen/logrus" ) -// NewHttpClient is a factory method to create a new Http Client with a default configuration. +// NewHTTPClient is a factory method to create a new Http Client with a default configuration. // RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, // the Headers are configured for JSON. -func NewHttpClient(settings *configuration.Settings) (*netHttp.Client, error) { +func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { headers := NewHeaders() - tlsConfig := NewTlsConfig(settings) + tlsConfig := NewTLSConfig(settings) transport := NewTransport(tlsConfig) roundTripper := NewRoundTripper(headers, transport) @@ -40,13 +40,13 @@ func NewHeaders() []string { } } -// NewTlsConfig is a factory method to create a new basic Tls Config. +// NewTLSConfig is a factory method to create a new basic Tls Config. // More attention should be given to the use of `InsecureSkipVerify: true`, as it is not recommended for production use. -func NewTlsConfig(settings *configuration.Settings) *tls.Config { - tlsConfig, err := authentication.NewTlsConfig(settings) +func NewTLSConfig(settings *configuration.Settings) *tls.Config { + tlsConfig, err := authentication.NewTLSConfig(settings) if err != nil { logrus.Warnf("Failed to create TLS config: %v", err) - return &tls.Config{InsecureSkipVerify: true} + return &tls.Config{InsecureSkipVerify: true} //nolint:gosec } return tlsConfig diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index f95722da..375da3b2 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -13,10 +13,14 @@ import ( "k8s.io/client-go/kubernetes/fake" ) -func TestNewHttpClient(t *testing.T) { +func TestNewHTTPClient(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, err := configuration.NewSettings(context.Background(), k8sClient) - client, err := NewHttpClient(settings) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } + client, err := NewHTTPClient(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) @@ -28,6 +32,7 @@ func TestNewHttpClient(t *testing.T) { } func TestNewHeaders(t *testing.T) { + t.Parallel() headers := NewHeaders() if headers == nil { @@ -48,9 +53,10 @@ func TestNewHeaders(t *testing.T) { } func TestNewTransport(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, _ := configuration.NewSettings(context.Background(), k8sClient) - config := NewTlsConfig(settings) + config := NewTLSConfig(settings) transport := NewTransport(config) if transport == nil { diff --git a/internal/communication/roundtripper.go b/internal/communication/roundtripper.go index 3781c62d..58de6f03 100644 --- a/internal/communication/roundtripper.go +++ b/internal/communication/roundtripper.go @@ -7,7 +7,6 @@ package communication import ( "net/http" - netHttp "net/http" "strings" ) @@ -18,7 +17,7 @@ type RoundTripper struct { } // NewRoundTripper is a factory method to create a new RoundTripper. -func NewRoundTripper(headers []string, transport *netHttp.Transport) *RoundTripper { +func NewRoundTripper(headers []string, transport *http.Transport) *RoundTripper { return &RoundTripper{ Headers: headers, RoundTripper: transport, diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index 0a549d61..ff6d5c4f 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -16,10 +16,11 @@ import ( ) func TestNewRoundTripper(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, _ := configuration.NewSettings(context.Background(), k8sClient) headers := NewHeaders() - transport := NewTransport(NewTlsConfig(settings)) + transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) if roundTripper == nil { @@ -48,10 +49,14 @@ func TestNewRoundTripper(t *testing.T) { } func TestRoundTripperRoundTrip(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, err := configuration.NewSettings(context.Background(), k8sClient) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } headers := NewHeaders() - transport := NewTransport(NewTlsConfig(settings)) + transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) request, err := NewRequest("GET", "http://example.com", nil) @@ -70,6 +75,7 @@ func TestRoundTripperRoundTrip(t *testing.T) { if response == nil { t.Fatalf(`response should not be nil`) } + defer response.Body.Close() headerLen := len(response.Header) if headerLen <= 2 { diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index e6bd30b5..05c3690d 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -46,7 +46,8 @@ const ( // There are two work queues in the application: // 1. nlk-handler queue, used to move messages between the Watcher and the Handler. // 2. nlk-synchronizer queue, used to move message between the Handler and the Synchronizer. -// The queues are NamedDelayingQueue objects that use an ItemExponentialFailureRateLimiter as the underlying rate limiter. +// The queues are NamedDelayingQueue objects that use an ItemExponentialFailureRateLimiter +// as the underlying rate limiter. type WorkQueueSettings struct { // Name is the name of the queue. Name string @@ -110,8 +111,9 @@ type Settings struct { // NginxPlusHosts is a list of Nginx Plus hosts that will be used to update the Border Servers. NginxPlusHosts []string - // TlsMode is the value used to determine which of the five TLS modes will be used to communicate with the Border Servers (see: ../../docs/tls/README.md). - TlsMode TLSMode + // TlsMode is the value used to determine which of the five TLS modes will be used to communicate + // with the Border Servers (see: ../../docs/tls/README.md). + TLSMode TLSMode // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. Certificates *certification.Certificates @@ -140,7 +142,7 @@ func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings settings := &Settings{ Context: ctx, K8sClient: k8sClient, - TlsMode: NoTLS, + TLSMode: NoTLS, Certificates: nil, Handler: HandlerSettings{ RetryCount: 5, @@ -187,10 +189,12 @@ func (s *Settings) Initialize() error { s.Certificates = certificates - go certificates.Run() + certificates.Run() //nolint:errcheck logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieving nlk-config ConfigMap") - configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get(s.Context, "nlk-config", metav1.GetOptions{}) + configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get( + s.Context, "nlk-config", metav1.GetOptions{}, + ) if err != nil { return err } @@ -198,10 +202,7 @@ func (s *Settings) Initialize() error { s.handleUpdateEvent(nil, configMap) logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieved nlk-config ConfigMap") - informer, err := s.buildInformer() - if err != nil { - return fmt.Errorf(`error occurred building ConfigMap informer: %w`, err) - } + informer := s.buildInformer() s.informer = informer @@ -213,7 +214,7 @@ func (s *Settings) Initialize() error { return nil } -// Run starts the SharedInformer and waits for the Context to be cancelled. +// Run starts the SharedInformer and waits for the Context to be canceled. func (s *Settings) Run() { logrus.Debug("Settings::Run") @@ -224,12 +225,12 @@ func (s *Settings) Run() { <-s.Context.Done() } -func (s *Settings) buildInformer() (cache.SharedInformer, error) { +func (s *Settings) buildInformer() cache.SharedInformer { options := informers.WithNamespace(ConfigMapsNamespace) factory := informers.NewSharedInformerFactoryWithOptions(s.K8sClient, ResyncPeriod, options) informer := factory.Core().V1().ConfigMaps().Informer() - return informer, nil + return informer } func (s *Settings) initializeEventListeners() error { @@ -283,12 +284,15 @@ func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { logrus.Warnf("Settings::handleUpdateEvent: nginx-hosts key not found in ConfigMap") } - tlsMode, err := validateTlsMode(configMap) + tlsMode, err := validateTLSMode(configMap) if err != nil { // NOTE: the TLSMode defaults to NoTLS on startup, or the last known good value if previously set. - logrus.Errorf("There was an error with the configured TLS Mode. TLS Mode has NOT been changed. The current mode is: '%v'. Error: %v. ", s.TlsMode, err) + logrus.Errorf( + "Error with configured TLS Mode. TLS Mode has NOT been changed. The current mode is: '%v'. Error: %v. ", + s.TLSMode, err, + ) } else { - s.TlsMode = tlsMode + s.TLSMode = tlsMode } caCertificateSecretKey, found := configMap.Data["ca-certificate"] @@ -314,7 +318,7 @@ func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { logrus.Debugf("Settings::handleUpdateEvent: \n\tHosts: %v,\n\tSettings: %v ", s.NginxPlusHosts, configMap) } -func validateTlsMode(configMap *corev1.ConfigMap) (TLSMode, error) { +func validateTLSMode(configMap *corev1.ConfigMap) (TLSMode, error) { tlsConfigMode, tlsConfigModeFound := configMap.Data["tls-mode"] if !tlsConfigModeFound { return NoTLS, fmt.Errorf(`tls-mode key not found in ConfigMap`) diff --git a/internal/configuration/tlsmodes_test.go b/internal/configuration/tlsmodes_test.go index 62abf962..d849cd97 100644 --- a/internal/configuration/tlsmodes_test.go +++ b/internal/configuration/tlsmodes_test.go @@ -10,6 +10,7 @@ import ( ) func Test_String(t *testing.T) { + t.Parallel() mode := NoTLS.String() if mode != "no-tls" { t.Errorf("Expected TLSModeNoTLS to be 'no-tls', got '%s'", mode) @@ -42,6 +43,7 @@ func Test_String(t *testing.T) { } func Test_TLSModeMap(t *testing.T) { + t.Parallel() mode := TLSModeMap["no-tls"] if mode != NoTLS { t.Errorf("Expected TLSModeMap['no-tls'] to be TLSModeNoTLS, got '%d'", mode) diff --git a/internal/core/event_test.go b/internal/core/event_test.go index 7f7448d4..662eb8f1 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -7,6 +7,7 @@ import ( ) func TestNewEvent(t *testing.T) { + t.Parallel() expectedType := Created expectedService := &v1.Service{} expectedPreviousService := &v1.Service{} diff --git a/internal/core/secret_bytes_test.go b/internal/core/secret_bytes_test.go index 8dd80247..5e6bc3ff 100644 --- a/internal/core/secret_bytes_test.go +++ b/internal/core/secret_bytes_test.go @@ -12,6 +12,7 @@ import ( ) func TestSecretBytesToString(t *testing.T) { + t.Parallel() sensitive := SecretBytes([]byte("If you can see this we have a problem")) expected := "foo [REDACTED] bar" @@ -22,6 +23,7 @@ func TestSecretBytesToString(t *testing.T) { } func TestSecretBytesToJSON(t *testing.T) { + t.Parallel() sensitive, _ := json.Marshal(SecretBytes([]byte("If you can see this we have a problem"))) expected := `foo "[REDACTED]" bar` result := fmt.Sprintf("foo %v bar", string(sensitive)) diff --git a/internal/core/server_update_event.go b/internal/core/server_update_event.go index f3961eae..0bc88680 100644 --- a/internal/core/server_update_event.go +++ b/internal/core/server_update_event.go @@ -15,7 +15,7 @@ type ServerUpdateEvent struct { ClientType string // Id is the unique identifier for this event. - Id string + ID string // NginxHost is the host name of the NGINX Plus instance that should handle this event. NginxHost string @@ -34,7 +34,12 @@ type ServerUpdateEvent struct { type ServerUpdateEvents = []*ServerUpdateEvent // NewServerUpdateEvent creates a new ServerUpdateEvent. -func NewServerUpdateEvent(eventType EventType, upstreamName string, clientType string, upstreamServers UpstreamServers) *ServerUpdateEvent { +func NewServerUpdateEvent( + eventType EventType, + upstreamName string, + clientType string, + upstreamServers UpstreamServers, +) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: clientType, Type: eventType, @@ -43,11 +48,11 @@ func NewServerUpdateEvent(eventType EventType, upstreamName string, clientType s } } -// ServerUpdateEventWithIdAndHost creates a new ServerUpdateEvent with the specified Id and Host. -func ServerUpdateEventWithIdAndHost(event *ServerUpdateEvent, id string, nginxHost string) *ServerUpdateEvent { +// ServerUpdateEventWithIDAndHost creates a new ServerUpdateEvent with the specified Id and Host. +func ServerUpdateEventWithIDAndHost(event *ServerUpdateEvent, id string, nginxHost string) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: event.ClientType, - Id: id, + ID: id, NginxHost: nginxHost, Type: event.Type, UpstreamName: event.UpstreamName, diff --git a/internal/core/server_update_event_test.go b/internal/core/server_update_event_test.go index a891e237..9f36002c 100644 --- a/internal/core/server_update_event_test.go +++ b/internal/core/server_update_event_test.go @@ -14,32 +14,34 @@ const clientType = "clientType" var emptyUpstreamServers UpstreamServers func TestServerUpdateEventWithIdAndHost(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) - if event.Id != "" { - t.Errorf("expected empty Id, got %s", event.Id) + if event.ID != "" { + t.Errorf("expected empty ID, got %s", event.ID) } if event.NginxHost != "" { t.Errorf("expected empty NginxHost, got %s", event.NginxHost) } - eventWithIdAndHost := ServerUpdateEventWithIdAndHost(event, "id", "host") + eventWithIDAndHost := ServerUpdateEventWithIDAndHost(event, "id", "host") - if eventWithIdAndHost.Id != "id" { - t.Errorf("expected Id to be 'id', got %s", eventWithIdAndHost.Id) + if eventWithIDAndHost.ID != "id" { + t.Errorf("expected Id to be 'id', got %s", eventWithIDAndHost.ID) } - if eventWithIdAndHost.NginxHost != "host" { - t.Errorf("expected NginxHost to be 'host', got %s", eventWithIdAndHost.NginxHost) + if eventWithIDAndHost.NginxHost != "host" { + t.Errorf("expected NginxHost to be 'host', got %s", eventWithIDAndHost.NginxHost) } - if eventWithIdAndHost.ClientType != clientType { - t.Errorf("expected ClientType to be '%s', got %s", clientType, eventWithIdAndHost.ClientType) + if eventWithIDAndHost.ClientType != clientType { + t.Errorf("expected ClientType to be '%s', got %s", clientType, eventWithIDAndHost.ClientType) } } func TestTypeNameCreated(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Created" { @@ -48,6 +50,7 @@ func TestTypeNameCreated(t *testing.T) { } func TestTypeNameUpdated(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Updated, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Updated" { @@ -56,6 +59,7 @@ func TestTypeNameUpdated(t *testing.T) { } func TestTypeNameDeleted(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Deleted, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Deleted" { @@ -64,6 +68,7 @@ func TestTypeNameDeleted(t *testing.T) { } func TestTypeNameUnknown(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(EventType(100), "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Unknown" { diff --git a/internal/core/upstream_server.go b/internal/core/upstream_server.go index 7c89b1e2..eeb72ac0 100644 --- a/internal/core/upstream_server.go +++ b/internal/core/upstream_server.go @@ -5,7 +5,8 @@ package core -// UpstreamServer represents a single upstream server. This is an internal representation used to abstract the definition +// UpstreamServer represents a single upstream server. +// This is an internal representation used to abstract the definition // of an upstream server from any specific client. type UpstreamServer struct { diff --git a/internal/core/upstream_server_test.go b/internal/core/upstream_server_test.go index 7b0eed52..91592cd3 100644 --- a/internal/core/upstream_server_test.go +++ b/internal/core/upstream_server_test.go @@ -8,6 +8,7 @@ package core import "testing" func TestNewUpstreamServer(t *testing.T) { + t.Parallel() host := "localhost" us := NewUpstreamServer(host) if us.Host != host { diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 8e897452..55849391 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -47,7 +47,11 @@ type Handler struct { } // NewHandler creates a new event handler -func NewHandler(settings *configuration.Settings, synchronizer synchronization.Interface, eventQueue workqueue.RateLimitingInterface) *Handler { +func NewHandler( + settings *configuration.Settings, + synchronizer synchronization.Interface, + eventQueue workqueue.RateLimitingInterface, +) *Handler { return &Handler{ eventQueue: eventQueue, settings: settings, diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index e5bef3d2..b7a47919 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -18,6 +18,7 @@ import ( ) func TestHandler_AddsEventToSynchronizer(t *testing.T) { + t.Parallel() _, _, synchronizer, handler, err := buildHandler() if err != nil { t.Errorf(`should have been no error, %v`, err) @@ -45,7 +46,11 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { } } -func buildHandler() (*configuration.Settings, workqueue.RateLimitingInterface, *mocks.MockSynchronizer, *Handler, error) { +func buildHandler() ( + *configuration.Settings, + workqueue.RateLimitingInterface, + *mocks.MockSynchronizer, *Handler, error, +) { settings, err := configuration.NewSettings(context.Background(), nil) if err != nil { return nil, nil, nil, nil, fmt.Errorf(`should have been no error, %v`, err) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 74e2d05e..a022f2e8 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -51,10 +51,7 @@ func (w *Watcher) Initialize() error { logrus.Debug("Watcher::Initialize") var err error - w.informer, err = w.buildInformer() - if err != nil { - return fmt.Errorf(`initialization error: %w`, err) - } + w.informer = w.buildInformer() err = w.initializeEventListeners() if err != nil { @@ -78,7 +75,11 @@ func (w *Watcher) Watch() error { go w.informer.Run(w.settings.Context.Done()) - if !cache.WaitForNamedCacheSync(w.settings.Handler.WorkQueueSettings.Name, w.settings.Context.Done(), w.informer.HasSynced) { + if !cache.WaitForNamedCacheSync( + w.settings.Handler.WorkQueueSettings.Name, + w.settings.Context.Done(), + w.informer.HasSynced, + ) { return fmt.Errorf(`error occurred waiting for the cache to sync`) } @@ -86,7 +87,8 @@ func (w *Watcher) Watch() error { return nil } -// buildEventHandlerForAdd creates a function that is used as an event handler for the informer when Add events are raised. +// buildEventHandlerForAdd creates a function that is used as an event handler +// for the informer when Add events are raised. func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { @@ -102,7 +104,8 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { } } -// buildEventHandlerForDelete creates a function that is used as an event handler for the informer when Delete events are raised. +// buildEventHandlerForDelete creates a function that is used as an event handler +// for the informer when Delete events are raised. func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { @@ -118,7 +121,8 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { } } -// buildEventHandlerForUpdate creates a function that is used as an event handler for the informer when Update events are raised. +// buildEventHandlerForUpdate creates a function that is used as an event handler +// for the informer when Update events are raised. func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { logrus.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { @@ -135,14 +139,16 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { } // buildInformer creates the informer used to watch for changes to Kubernetes resources. -func (w *Watcher) buildInformer() (cache.SharedIndexInformer, error) { +func (w *Watcher) buildInformer() cache.SharedIndexInformer { logrus.Debug("Watcher::buildInformer") options := informers.WithNamespace(w.settings.Watcher.NginxIngressNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, options) + factory := informers.NewSharedInformerFactoryWithOptions( + w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, options, + ) informer := factory.Core().V1().Services().Informer() - return informer, nil + return informer } // initializeEventListeners initializes the event listeners for the informer. diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 7c820bf6..2a6d94b4 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -15,6 +15,7 @@ import ( ) func TestWatcher_MustInitialize(t *testing.T) { + t.Parallel() watcher, _ := buildWatcher() if err := watcher.Watch(); err == nil { t.Errorf("Expected error, got %s", err) diff --git a/internal/probation/check_test.go b/internal/probation/check_test.go index 208c9a41..95358e58 100644 --- a/internal/probation/check_test.go +++ b/internal/probation/check_test.go @@ -8,6 +8,7 @@ package probation import "testing" func TestCheck_LiveCheck(t *testing.T) { + t.Parallel() check := LiveCheck{} if !check.Check() { t.Errorf("LiveCheck should return true") @@ -15,6 +16,7 @@ func TestCheck_LiveCheck(t *testing.T) { } func TestCheck_ReadyCheck(t *testing.T) { + t.Parallel() check := ReadyCheck{} if !check.Check() { t.Errorf("ReadyCheck should return true") @@ -22,6 +24,7 @@ func TestCheck_ReadyCheck(t *testing.T) { } func TestCheck_StartupCheck(t *testing.T) { + t.Parallel() check := StartupCheck{} if !check.Check() { t.Errorf("StartupCheck should return true") diff --git a/internal/probation/server.go b/internal/probation/server.go index ff23e339..84b5b674 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -8,6 +8,7 @@ package probation import ( "fmt" "net/http" + "time" "github.com/sirupsen/logrus" ) @@ -59,7 +60,7 @@ func (hs *HealthServer) Start() { mux.HandleFunc("/livez", hs.HandleLive) mux.HandleFunc("/readyz", hs.HandleReady) mux.HandleFunc("/startupz", hs.HandleStartup) - hs.httpServer = &http.Server{Addr: address, Handler: mux} + hs.httpServer = &http.Server{Addr: address, Handler: mux, ReadTimeout: 2 * time.Second} go func() { if err := hs.httpServer.ListenAndServe(); err != nil { diff --git a/internal/probation/server_test.go b/internal/probation/server_test.go index 4ea51e7a..9c7d37db 100644 --- a/internal/probation/server_test.go +++ b/internal/probation/server_test.go @@ -14,6 +14,7 @@ import ( ) func TestHealthServer_HandleLive(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleLive(writer, nil) @@ -24,6 +25,7 @@ func TestHealthServer_HandleLive(t *testing.T) { } func TestHealthServer_HandleReady(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleReady(writer, nil) @@ -34,6 +36,7 @@ func TestHealthServer_HandleReady(t *testing.T) { } func TestHealthServer_HandleStartup(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleStartup(writer, nil) @@ -44,6 +47,7 @@ func TestHealthServer_HandleStartup(t *testing.T) { } func TestHealthServer_HandleFailCheck(t *testing.T) { + t.Parallel() failCheck := mocks.NewMockCheck(false) server := NewHealthServer() writer := mocks.NewMockResponseWriter() @@ -56,6 +60,7 @@ func TestHealthServer_HandleFailCheck(t *testing.T) { } func TestHealthServer_Start(t *testing.T) { + t.Parallel() server := NewHealthServer() server.Start() @@ -65,6 +70,7 @@ func TestHealthServer_Start(t *testing.T) { if err != nil { t.Error(err) } + defer response.Body.Close() if response.StatusCode != http.StatusOK { t.Errorf("Expected status code %v, got %v", http.StatusAccepted, response.StatusCode) diff --git a/internal/synchronization/rand.go b/internal/synchronization/rand.go index 425b99ad..6bf58d1d 100644 --- a/internal/synchronization/rand.go +++ b/internal/synchronization/rand.go @@ -6,6 +6,7 @@ package synchronization import ( + // Try using crpyto if needed. "math/rand" "time" ) @@ -24,14 +25,14 @@ func RandomString(n int) string { b := make([]byte, n) for i := range b { // randomly select 1 character from given charset - b[i] = alphaNumeric[rand.Intn(len(alphaNumeric))] + b[i] = alphaNumeric[rand.Intn(len(alphaNumeric))] //nolint:gosec } return string(b) } // RandomMilliseconds returns a random duration between min and max milliseconds func RandomMilliseconds(min, max int) time.Duration { - randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) + randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec random := randomizer.Intn(max-min) + min return time.Millisecond * time.Duration(random) diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 5fc07f55..7522f3a2 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -35,15 +35,19 @@ type Interface interface { } // Synchronizer is responsible for synchronizing the state of the Border Servers. -// Operating against the "nlk-synchronizer", it handles events by creating a Border Client as specified in the -// Service annotation for the Upstream. see application/border_client.go and application/application_constants.go for details. +// Operating against the "nlk-synchronizer", it handles events by creating +// a Border Client as specified in the Service annotation for the Upstream. +// See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { eventQueue workqueue.RateLimitingInterface settings *configuration.Settings } // NewSynchronizer creates a new Synchronizer. -func NewSynchronizer(settings *configuration.Settings, eventQueue workqueue.RateLimitingInterface) (*Synchronizer, error) { +func NewSynchronizer( + settings *configuration.Settings, + eventQueue workqueue.RateLimitingInterface, +) (*Synchronizer, error) { synchronizer := Synchronizer{ eventQueue: eventQueue, settings: settings, @@ -79,7 +83,10 @@ func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { return } - after := RandomMilliseconds(s.settings.Synchronizer.MinMillisecondsJitter, s.settings.Synchronizer.MaxMillisecondsJitter) + after := RandomMilliseconds( + s.settings.Synchronizer.MinMillisecondsJitter, + s.settings.Synchronizer.MaxMillisecondsJitter, + ) s.eventQueue.AddAfter(event, after) } @@ -108,7 +115,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica var err error - httpClient, err := communication.NewHttpClient(s.settings) + httpClient, err := communication.NewHTTPClient(s.settings) if err != nil { return nil, fmt.Errorf(`error creating HTTP client: %v`, err) } @@ -130,7 +137,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se for hidx, host := range s.settings.NginxPlusHosts { for eidx, event := range event { id := fmt.Sprintf(`[%d:%d]-[%s]-[%s]-[%s]`, hidx, eidx, RandomString(12), event.UpstreamName, host) - updatedEvent := core.ServerUpdateEventWithIdAndHost(event, id, host) + updatedEvent := core.ServerUpdateEventWithIDAndHost(event, id, host) events = append(events, updatedEvent) } @@ -141,7 +148,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se // handleEvent dispatches an event to the proper handler function. func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleEvent: Id: %s`, event.Id) + logrus.Debugf(`Synchronizer::handleEvent: Id: %s`, event.ID) var err error @@ -160,7 +167,9 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { } if err == nil { - logrus.Infof(`Synchronizer::handleEvent: successfully %s the nginx+ host(s) for Upstream: %s: Id(%s)`, event.TypeName(), event.UpstreamName, event.Id) + logrus.Infof( + `Synchronizer::handleEvent: successfully %s the nginx+ host(s) for Upstream: %s: Id(%s)`, + event.TypeName(), event.UpstreamName, event.ID) } return err @@ -168,7 +177,7 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { // handleCreatedUpdatedEvent handles events of type Created or Updated. func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleCreatedUpdatedEvent: Id: %s`, serverUpdateEvent.Id) + logrus.Debugf(`Synchronizer::handleCreatedUpdatedEvent: Id: %s`, serverUpdateEvent.ID) var err error @@ -186,7 +195,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU // handleDeletedEvent handles events of type Deleted. func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleDeletedEvent: Id: %s`, serverUpdateEvent.Id) + logrus.Debugf(`Synchronizer::handleDeletedEvent: Id: %s`, serverUpdateEvent.ID) var err error @@ -233,7 +242,7 @@ func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { // TODO: Add Telemetry if s.eventQueue.NumRequeues(event) < s.settings.Synchronizer.RetryCount { // TODO: Make this configurable s.eventQueue.AddRateLimited(event) - logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.Id, err) + logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) } else { s.eventQueue.Forget(event) logrus.Warnf(`Synchronizer::withRetry: event %#v has been dropped due to too many retries`, event) diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index ef510b85..36345139 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -16,7 +16,11 @@ import ( ) func TestSynchronizer_NewSynchronizer(t *testing.T) { + t.Parallel() settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } rateLimiter := &mocks.MockRateLimiter{} @@ -31,15 +35,19 @@ func TestSynchronizer_NewSynchronizer(t *testing.T) { } func TestSynchronizer_AddEventNoHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 event := &core.ServerUpdateEvent{ - Id: "", + ID: "", NginxHost: "", Type: 0, UpstreamName: "", UpstreamServers: nil, } settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } rateLimiter := &mocks.MockRateLimiter{} synchronizer, err := NewSynchronizer(settings, rateLimiter) @@ -61,9 +69,13 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { } func TestSynchronizer_AddEventOneHost(t *testing.T) { + t.Parallel() const expectedEventCount = 1 events := buildEvents(1) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{"https://localhost:8080"} rateLimiter := &mocks.MockRateLimiter{} @@ -84,9 +96,13 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { } func TestSynchronizer_AddEventManyHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 1 events := buildEvents(1) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{ "https://localhost:8080", "https://localhost:8081", @@ -111,9 +127,13 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { } func TestSynchronizer_AddEventsNoHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 events := buildEvents(4) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } rateLimiter := &mocks.MockRateLimiter{} synchronizer, err := NewSynchronizer(settings, rateLimiter) @@ -135,9 +155,13 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { } func TestSynchronizer_AddEventsOneHost(t *testing.T) { + t.Parallel() const expectedEventCount = 4 events := buildEvents(4) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{"https://localhost:8080"} rateLimiter := &mocks.MockRateLimiter{} @@ -158,10 +182,14 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { } func TestSynchronizer_AddEventsManyHosts(t *testing.T) { + t.Parallel() const eventCount = 4 events := buildEvents(eventCount) rateLimiter := &mocks.MockRateLimiter{} settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{ "https://localhost:8080", "https://localhost:8081", @@ -189,7 +217,7 @@ func buildEvents(count int) core.ServerUpdateEvents { events := make(core.ServerUpdateEvents, count) for i := 0; i < count; i++ { events[i] = &core.ServerUpdateEvent{ - Id: fmt.Sprintf("id-%v", i), + ID: fmt.Sprintf("id-%v", i), NginxHost: "https://localhost:8080", Type: 0, UpstreamName: "", diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 98ff3684..5dc49ecf 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -40,16 +40,18 @@ func filterPorts(ports []v1.ServicePort) []v1.ServicePort { } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type -// The NGINX+ Client uses a list of servers for Created and Updated events; the client performs reconciliation between -// the list of servers in the NGINX+ Client call and the list of servers in NGINX+. -// The NGINX+ Client uses a single server for Deleted events; so the list of servers is broken up into individual events. +// The NGINX+ Client uses a list of servers for Created and Updated events. +// The client performs reconciliation between the list of servers in the NGINX+ Client call +// and the list of servers in NGINX+. +// The NGINX+ Client uses a single server for Deleted events; +// so the list of servers is broken up into individual events. func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.ServerUpdateEvents, error) { logrus.Debugf("Translate::buildServerUpdateEvents(ports=%#v)", ports) events := core.ServerUpdateEvents{} for _, port := range ports { ingressName := fixIngressName(port.Name) - upstreamServers, _ := buildUpstreamServers(event.NodeIps, port) + upstreamServers := buildUpstreamServers(event.NodeIps, port) clientType := getClientType(port.Name, event.Service.Annotations) switch event.Type { @@ -61,7 +63,9 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se case core.Deleted: for _, server := range upstreamServers { - events = append(events, core.NewServerUpdateEvent(event.Type, ingressName, clientType, core.UpstreamServers{server})) + events = append(events, core.NewServerUpdateEvent( + event.Type, ingressName, clientType, core.UpstreamServers{server}, + )) } default: @@ -73,16 +77,16 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se return events, nil } -func buildUpstreamServers(nodeIps []string, port v1.ServicePort) (core.UpstreamServers, error) { +func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamServers { var servers core.UpstreamServers - for _, nodeIp := range nodeIps { - host := fmt.Sprintf("%s:%d", nodeIp, port.NodePort) + for _, nodeIP := range nodeIPs { + host := fmt.Sprintf("%s:%d", nodeIP, port.NodePort) server := core.NewUpstreamServer(host) servers = append(servers, server) } - return servers, nil + return servers } // fixIngressName removes the NlkPrefix from the port name @@ -100,5 +104,5 @@ func getClientType(portName string, annotations map[string]string) string { } } - return application.ClientTypeNginxHttp + return application.ClientTypeNginxHTTP } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index a393f642..b53abcc7 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -29,6 +29,7 @@ const ( */ func TestCreatedTranslateNoPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -46,6 +47,7 @@ func TestCreatedTranslateNoPorts(t *testing.T) { } func TestCreatedTranslateNoInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -65,6 +67,7 @@ func TestCreatedTranslateNoInterestingPorts(t *testing.T) { } func TestCreatedTranslateOneInterestingPort(t *testing.T) { + t.Parallel() const expectedEventCount = 1 const portCount = 1 @@ -86,6 +89,7 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { } func TestCreatedTranslateManyInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 4 const portCount = 4 @@ -107,6 +111,7 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { } func TestCreatedTranslateManyMixedPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -129,6 +134,7 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { } func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -155,6 +161,7 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { */ func TestUpdatedTranslateNoPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -172,6 +179,7 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { } func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -191,6 +199,7 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { } func TestUpdatedTranslateOneInterestingPort(t *testing.T) { + t.Parallel() const expectedEventCount = 1 const portCount = 1 @@ -212,6 +221,7 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { } func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 4 const portCount = 4 @@ -233,6 +243,7 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { } func TestUpdatedTranslateManyMixedPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -255,6 +266,7 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { } func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -281,6 +293,7 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { */ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -300,6 +313,7 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -321,6 +335,8 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 const portCount = 1 @@ -342,6 +358,7 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { } func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 4 @@ -363,6 +380,7 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 6 const updatablePortCount = 2 @@ -385,6 +403,7 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -404,6 +423,7 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -425,6 +445,7 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 1 const portCount = 1 @@ -446,6 +467,7 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { } func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 4 const portCount = 4 @@ -467,6 +489,7 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -489,6 +512,7 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -508,6 +532,7 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { } func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 1 const updatablePortCount = 0 const expectedEventCount = updatablePortCount * ManyNodes @@ -530,6 +555,7 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { } func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 1 const expectedEventCount = portCount * ManyNodes @@ -551,6 +577,7 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { } func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 4 const expectedEventCount = portCount * ManyNodes @@ -572,6 +599,7 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { } func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 6 const updatablePortCount = 2 const expectedEventCount = updatablePortCount * ManyNodes @@ -650,7 +678,7 @@ func generatePorts(portCount int) []v1.ServicePort { // This is probably A Little Bit of Too Muchâ„¢, but helps to ensure ordering is not a factor. func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort { - var ports []v1.ServicePort + ports := []v1.ServicePort{} updatable := make([]string, updatableCount) nonupdatable := make([]string, portCount-updatableCount) @@ -663,7 +691,9 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort nonupdatable[j] = "olm-" } - prefixes := append(updatable, nonupdatable...) + var prefixes []string + prefixes = append(prefixes, updatable...) + prefixes = append(prefixes, nonupdatable...) source := rand.NewSource(time.Now().UnixNano()) random := rand.New(source) diff --git a/test/mocks/mock_nginx_plus_client.go b/test/mocks/mock_nginx_plus_client.go index 2c16e12f..00b560ea 100644 --- a/test/mocks/mock_nginx_plus_client.go +++ b/test/mocks/mock_nginx_plus_client.go @@ -36,7 +36,10 @@ func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { return nil } -func (m MockNginxClient) UpdateStreamServers(_ string, _ []nginxClient.StreamUpstreamServer) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { +func (m MockNginxClient) UpdateStreamServers( + _ string, + _ []nginxClient.StreamUpstreamServer, +) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { m.CalledFunctions["UpdateStreamServers"] = true if m.Error != nil { @@ -56,7 +59,10 @@ func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { return nil } -func (m MockNginxClient) UpdateHTTPServers(_ string, _ []nginxClient.UpstreamServer) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { +func (m MockNginxClient) UpdateHTTPServers( + _ string, + _ []nginxClient.UpstreamServer, +) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { m.CalledFunctions["UpdateHTTPServers"] = true if m.Error != nil { From a5bcb38240d17055a63e915bcfb158164c7c1c08 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 29 Mar 2024 18:26:15 -0700 Subject: [PATCH 005/110] Update go version and deps --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 38f6adb0..7a95a390 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.19 +go 1.21 require ( github.com/nginxinc/nginx-plus-go-client v0.10.0 diff --git a/go.sum b/go.sum index 867f71f1..7d3d0e5e 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -160,7 +161,9 @@ github.com/nginxinc/nginx-plus-go-client v0.10.0/go.mod h1:0v3RsQCvRn/IyrMtW+DK6 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -179,6 +182,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From d58089f30dd85c219c21c2777628b23610e0b7bd Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 8 Jul 2024 15:47:57 -0700 Subject: [PATCH 006/110] Disable exhaustruct linter for now --- .golangci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index d89f3f5f..6cb35877 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,7 +7,6 @@ linters: # Supported linters: https://golangci-lint.run/usage/linters/ enable: - errcheck - - exhaustruct - gosimple - govet - ineffassign From 0f8dbc7be7c5dccc70c26878cf879afd2d2ad155 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 26 Jun 2024 13:39:35 -0600 Subject: [PATCH 007/110] NLB-4655 NLK will retry a work item to update upstreams indefinitely The primary intent behind this is to keep retrying updates which may be made before the controlplane has registered the existence of a named upstream in the customer's NGINX configuration. --- internal/synchronization/synchronizer.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 7522f3a2..9d1a6973 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -20,7 +20,6 @@ import ( // Interface defines the interface needed to implement a synchronizer. type Interface interface { - // AddEvents adds a list of events to the queue. AddEvents(events core.ServerUpdateEvents) @@ -240,13 +239,8 @@ func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { logrus.Debug("Synchronizer::withRetry") if err != nil { // TODO: Add Telemetry - if s.eventQueue.NumRequeues(event) < s.settings.Synchronizer.RetryCount { // TODO: Make this configurable - s.eventQueue.AddRateLimited(event) - logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) - } else { - s.eventQueue.Forget(event) - logrus.Warnf(`Synchronizer::withRetry: event %#v has been dropped due to too many retries`, event) - } + s.eventQueue.AddRateLimited(event) + logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) } else { s.eventQueue.Forget(event) } // TODO: Add error logging From a4dfe4f67d7064e6f6416f5b8b8fb625efa65812 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 23 Jul 2024 08:48:55 -0700 Subject: [PATCH 008/110] Update binary/docker img to nginxaas-operator --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0a12779f..d80281c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,6 @@ COPY docker-user /etc/passwd USER 101 COPY --from=base-certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -FROM base as nlk -ENTRYPOINT ["/nlk"] -COPY build/nlk / +FROM base as nginxaas-operator +ENTRYPOINT ["/nginxaas-operator"] +COPY build/nginxaas-operator / From 4605a7996e57d05d34e6b007b6e3d0665fcc7c14 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 7 Aug 2024 13:44:57 -0700 Subject: [PATCH 009/110] Run the informer in go routine NOTE: This commit was accidentally missed out in the first iteration of this fork and is present in upstream: https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/internal/configuration/settings.go#L189 The code needs to move forward to start the watchers and health server (side note: we should also think about the ordering of these at some point) and not running the infomer in a go routine prevents the program from further execution until the context is canceled. In the current iteration on main, the controller is stuck on the informer, and then k8s kills the service and restarts it since the health server is not up. --- internal/configuration/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 05c3690d..af7c2d87 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -189,7 +189,7 @@ func (s *Settings) Initialize() error { s.Certificates = certificates - certificates.Run() //nolint:errcheck + go certificates.Run() //nolint:errcheck logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieving nlk-config ConfigMap") configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get( From e2ad4d8242d85571209dcf27d5ac1f876a10112e Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 29 Jul 2024 16:22:01 -0600 Subject: [PATCH 010/110] NLB-4617 Watcher filters events by annotation on ingress service name The user specifies the ingress service whose events the application should watch through setting the "service-annotation-match" annotation on the application's config map. Only events with a matching service annotation will be passed by the watcher to the handlers. The informer now listens to events from all namespaces. This frees the end user from the restriction of only using the nginx ingress controller. --- internal/configuration/settings.go | 27 +++++++++++++++------- internal/observation/watcher.go | 37 ++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index af7c2d87..857bd433 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -40,6 +40,14 @@ const ( // The value of the annotation determines which BorderServer implementation will be used. // See the documentation in the `application/application_constants.go` file for details. PortAnnotationPrefix = "nginxinc.io" + + // ServiceAnnotationMatchKey is the key name of the annotation in the application's config map + // that identifies the ingress service whose events will be monitored. + ServiceAnnotationMatchKey = "service-annotation-match" + + // DefaultServiceAnnotation is the default name of the ingress service whose events will be + // monitored. + DefaultServiceAnnotation = "nginxaas" ) // WorkQueueSettings contains the configuration values needed by the Work Queues. @@ -62,7 +70,6 @@ type WorkQueueSettings struct { // HandlerSettings contains the configuration values needed by the Handler. type HandlerSettings struct { - // RetryCount is the number of times the Handler will attempt to process a message before giving up. RetryCount int @@ -75,9 +82,8 @@ type HandlerSettings struct { // WatcherSettings contains the configuration values needed by the Watcher. type WatcherSettings struct { - - // NginxIngressNamespace is the namespace used to filter Services in the Watcher. - NginxIngressNamespace string + // ServiceAnnotation is the annotation of the ingress service whose events the watcher should monitor. + ServiceAnnotation string // ResyncPeriod is the value used to set the resync period for the underlying SharedInformer. ResyncPeriod time.Duration @@ -85,7 +91,6 @@ type WatcherSettings struct { // SynchronizerSettings contains the configuration values needed by the Synchronizer. type SynchronizerSettings struct { - // MaxMillisecondsJitter is the maximum number of milliseconds that will be applied when adding an event to the queue. MaxMillisecondsJitter int @@ -104,7 +109,6 @@ type SynchronizerSettings struct { // Settings contains the configuration values needed by the application. type Settings struct { - // Context is the context used to control the application. Context context.Context @@ -165,8 +169,8 @@ func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings }, }, Watcher: WatcherSettings{ - NginxIngressNamespace: "nginx-ingress", - ResyncPeriod: 0, + ResyncPeriod: 0, + ServiceAnnotation: DefaultServiceAnnotation, }, } @@ -313,6 +317,13 @@ func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { logrus.Warnf("Settings::handleUpdateEvent: client-certificate key not found in ConfigMap") } + if serviceAnnotation, found := configMap.Data[ServiceAnnotationMatchKey]; found { + s.Watcher.ServiceAnnotation = serviceAnnotation + } else { + s.Watcher.ServiceAnnotation = DefaultServiceAnnotation + } + logrus.Debugf("Settings::handleUpdateEvent: %s: %s", ServiceAnnotationMatchKey, s.Watcher.ServiceAnnotation) + setLogLevel(configMap.Data["log-level"]) logrus.Debugf("Settings::handleUpdateEvent: \n\tHosts: %v,\n\tSettings: %v ", s.NginxPlusHosts, configMap) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index a022f2e8..a07ef125 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -24,7 +24,6 @@ import ( // Particularly, Services in the namespace defined in the WatcherSettings::NginxIngressNamespace setting. // When a change is detected, an Event is generated and added to the Handler's queue. type Watcher struct { - // eventHandlerRegistration is used to track the event handlers eventHandlerRegistration interface{} @@ -87,17 +86,32 @@ func (w *Watcher) Watch() error { return nil } +// isDesiredService returns whether the user has configured the given service for watching. +func (w *Watcher) isDesiredService(service *v1.Service) bool { + annotation, ok := service.Annotations["nginx.com/nginxaas"] + if !ok { + return false + } + + return annotation == w.settings.Watcher.ServiceAnnotation +} + // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { + service := obj.(*v1.Service) + if !w.isDesiredService(service) { + return + } + nodeIps, err := w.retrieveNodeIps() if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return } - service := obj.(*v1.Service) + var previousService *v1.Service e := core.NewEvent(core.Created, service, previousService, nodeIps) w.handler.AddRateLimitedEvent(&e) @@ -109,12 +123,17 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { + service := obj.(*v1.Service) + if !w.isDesiredService(service) { + return + } + nodeIps, err := w.retrieveNodeIps() if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return } - service := obj.(*v1.Service) + var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService, nodeIps) w.handler.AddRateLimitedEvent(&e) @@ -126,12 +145,18 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { logrus.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { + // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API + service := updated.(*v1.Service) + if !w.isDesiredService(service) { + return + } + nodeIps, err := w.retrieveNodeIps() if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return } - service := updated.(*v1.Service) + previousService := previous.(*v1.Service) e := core.NewEvent(core.Updated, service, previousService, nodeIps) w.handler.AddRateLimitedEvent(&e) @@ -142,9 +167,8 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { func (w *Watcher) buildInformer() cache.SharedIndexInformer { logrus.Debug("Watcher::buildInformer") - options := informers.WithNamespace(w.settings.Watcher.NginxIngressNamespace) factory := informers.NewSharedInformerFactoryWithOptions( - w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, options, + w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, ) informer := factory.Core().V1().Services().Informer() @@ -185,7 +209,6 @@ func (w *Watcher) retrieveNodeIps() ([]string, error) { } for _, node := range nodes.Items { - // this is kind of a broad assumption, should probably make this a configurable option if w.notMasterNode(node) { for _, address := range node.Status.Addresses { From c0e86a3e3a5f4459f397fda1edfbbf92db6561b0 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 26 Jul 2024 09:25:37 -0600 Subject: [PATCH 011/110] NLB-4823 Translator assumes that port names provide context and upstream name The port name should now be formatted like this: "http-tea", where the first part of the string is the context type (either "http" or "stream") and the second part of the string after the hyphen is the name of the upstream. --- internal/observation/handler_test.go | 2 +- internal/translation/translator.go | 55 +++++++++---------------- internal/translation/translator_test.go | 7 ++-- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index b7a47919..72b6d8f4 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -30,7 +30,7 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ { - Name: "nlk-back", + Name: "http-back", }, }, }, diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 5dc49ecf..9dad7f81 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -9,8 +9,6 @@ import ( "fmt" "strings" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" @@ -21,22 +19,7 @@ import ( func Translate(event *core.Event) (core.ServerUpdateEvents, error) { logrus.Debug("Translate::Translate") - portsOfInterest := filterPorts(event.Service.Spec.Ports) - - return buildServerUpdateEvents(portsOfInterest, event) -} - -// filterPorts returns a list of ports that have the NlkPrefix in the port name. -func filterPorts(ports []v1.ServicePort) []v1.ServicePort { - var portsOfInterest []v1.ServicePort - - for _, port := range ports { - if strings.HasPrefix(port.Name, configuration.NlkPrefix) { - portsOfInterest = append(portsOfInterest, port) - } - } - - return portsOfInterest + return buildServerUpdateEvents(event.Service.Spec.Ports, event) } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type @@ -50,21 +33,25 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se events := core.ServerUpdateEvents{} for _, port := range ports { - ingressName := fixIngressName(port.Name) + context, upstreamName, err := getContextAndUpstreamName(port) + if err != nil { + logrus.Info(err) + continue + } + upstreamServers := buildUpstreamServers(event.NodeIps, port) - clientType := getClientType(port.Name, event.Service.Annotations) switch event.Type { case core.Created: fallthrough case core.Updated: - events = append(events, core.NewServerUpdateEvent(event.Type, ingressName, clientType, upstreamServers)) + events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) case core.Deleted: for _, server := range upstreamServers { events = append(events, core.NewServerUpdateEvent( - event.Type, ingressName, clientType, core.UpstreamServers{server}, + event.Type, upstreamName, context, core.UpstreamServers{server}, )) } @@ -89,20 +76,18 @@ func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamSe return servers } -// fixIngressName removes the NlkPrefix from the port name -func fixIngressName(name string) string { - return name[4:] -} +// getContextAndUpstreamName returns the nginx context being supplied by the port (either "http" or "stream") +// and the upstream name. +func getContextAndUpstreamName(port v1.ServicePort) (clientType string, appName string, err error) { + parts := strings.Split(port.Name, "-") + if len(parts) != 2 { + return clientType, appName, + fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", port.Name) + } -// getClientType returns the client type for the port, defaults to ClientTypeNginxHttp if no Annotation is found. -func getClientType(portName string, annotations map[string]string) string { - key := fmt.Sprintf("%s/%s", configuration.PortAnnotationPrefix, portName) - logrus.Infof("getClientType: key=%s", key) - if annotations != nil { - if clientType, ok := annotations[key]; ok { - return clientType - } + if parts[0] != "http" && parts[0] != "stream" { + return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", port.Name) } - return application.ClientTypeNginxHTTP + return parts[0], parts[1], nil } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index b53abcc7..5b508c3b 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" ) @@ -682,9 +681,11 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort updatable := make([]string, updatableCount) nonupdatable := make([]string, portCount-updatableCount) + contexts := []string{"http-", "stream-"} for i := range updatable { - updatable[i] = configuration.NlkPrefix + randomIndex := int(rand.Float32() * 2.0) + updatable[i] = contexts[randomIndex] } for j := range nonupdatable { @@ -701,7 +702,7 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort for i, prefix := range prefixes { ports = append(ports, v1.ServicePort{ - Name: fmt.Sprintf("%sport-%d", prefix, i), + Name: fmt.Sprintf("%supstream%d", prefix, i), }) } From cb499888882fcb81f6cd987042484d5cf71b0009 Mon Sep 17 00:00:00 2001 From: sarna Date: Sat, 17 Aug 2024 16:27:27 -0700 Subject: [PATCH 012/110] NLB-5282: Allow images to be pushed to Dockerhub We need to be able to publish the operator to dockerhub in order to be publicly available for customers. Following what we have in ARP as a release strategy where a release tag action would publish the image to dockerhub. --- scripts/release.sh | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 scripts/release.sh diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 00000000..7c857927 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [[ "${CI}" != "true" ]]; then + echo "This script is meant to be run in the CI." + exit 1 +fi + +pttn="^release-[0-9]+\.[0-9]+\.[0-9]+" +if ! [[ "${CI_COMMIT_TAG}" =~ $pttn ]]; then + echo "CI_COMMIT_TAG needs to be set to valid semver format." + exit 1 +fi + +ROOT_DIR=$(git rev-parse --show-toplevel) +source ${ROOT_DIR}/.devops.sh + +DOCKERHUB_USERNAME=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".username") +if [[ -z "${DOCKERHUB_USERNAME}" ]]; then + echo "DOCKERHUB_USERNAME needs to be set." + exit 1 +fi + +DOCKERHUB_PASSWORD=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".password") +if [[ -z "${DOCKERHUB_PASSWORD}" ]]; then + echo "DOCKERHUB_PASSWORD needs to be set." + exit 1 +fi + +SRC_REGISTRY="${DEVOPS_DOCKER_URL}" +SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" +SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") +SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" + +DST_REGISTRY="docker.io" +DST_PATH="nginx/nginxaas-operator" +DST_TAG="${CI_COMMIT_TAG}" +DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" + +devops.docker.login +docker pull "${SRC_IMG}" +docker tag "${SRC_IMG}" "${DST_IMG}" + +# Login to Dockerhub and push release image to it. +docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_PASSWORD}" "${DST_REGISTRY}" +docker push "${DST_IMG}" From afd9aa3272a99a85209a6c22203251ae7769d4a0 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 31 Jul 2024 10:48:55 -0700 Subject: [PATCH 013/110] Remove unneeded file --- .tool-versions | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 09548d5e..00000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -golang 1.19.13 From 692ad94a53bdb5fdbffaadcfb1d29fe8b9ec8850 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 19 Aug 2024 16:21:40 -0700 Subject: [PATCH 014/110] Capture code coverage go test does not have a good way to capture unit-test coverage as part of test runs. This commit captures the error code of the unit test run, runs the coverage generation and then exits based on the test status. --- scripts/release.sh | 47 ---------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100755 scripts/release.sh diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index 7c857927..00000000 --- a/scripts/release.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -if [[ "${CI}" != "true" ]]; then - echo "This script is meant to be run in the CI." - exit 1 -fi - -pttn="^release-[0-9]+\.[0-9]+\.[0-9]+" -if ! [[ "${CI_COMMIT_TAG}" =~ $pttn ]]; then - echo "CI_COMMIT_TAG needs to be set to valid semver format." - exit 1 -fi - -ROOT_DIR=$(git rev-parse --show-toplevel) -source ${ROOT_DIR}/.devops.sh - -DOCKERHUB_USERNAME=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".username") -if [[ -z "${DOCKERHUB_USERNAME}" ]]; then - echo "DOCKERHUB_USERNAME needs to be set." - exit 1 -fi - -DOCKERHUB_PASSWORD=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".password") -if [[ -z "${DOCKERHUB_PASSWORD}" ]]; then - echo "DOCKERHUB_PASSWORD needs to be set." - exit 1 -fi - -SRC_REGISTRY="${DEVOPS_DOCKER_URL}" -SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" -SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") -SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" - -DST_REGISTRY="docker.io" -DST_PATH="nginx/nginxaas-operator" -DST_TAG="${CI_COMMIT_TAG}" -DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" - -devops.docker.login -docker pull "${SRC_IMG}" -docker tag "${SRC_IMG}" "${DST_IMG}" - -# Login to Dockerhub and push release image to it. -docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_PASSWORD}" "${DST_REGISTRY}" -docker push "${DST_IMG}" From a64e5c8cda541993a2181daa700b2f1a998144be Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 22 Aug 2024 13:15:39 -0600 Subject: [PATCH 015/110] NLB-5360 Upgraded nginx-plus client to v 1.2.2 --- go.mod | 6 ++++-- go.sum | 4 ++-- internal/synchronization/synchronizer.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 7a95a390..118de5f5 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.21 +go 1.21.2 + +toolchain go1.21.4 require ( - github.com/nginxinc/nginx-plus-go-client v0.10.0 + github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/sirupsen/logrus v1.9.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 diff --git a/go.sum b/go.sum index 7d3d0e5e..46a175a7 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginxinc/nginx-plus-go-client v0.10.0 h1:3zsMMkPvRDo8D7ZSprXtbAEW/SDmezZWzxdyS+6oAlc= -github.com/nginxinc/nginx-plus-go-client v0.10.0/go.mod h1:0v3RsQCvRn/IyrMtW+DK6CNkz+PxEsXDJPjQ3yUMBF0= +github.com/nginxinc/nginx-plus-go-client v1.2.2 h1:sl7HqNDDZq2EVu0eQQVoZ6PKYGa4h2dB/Qr5Ib0YKGw= +github.com/nginxinc/nginx-plus-go-client v1.2.2/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 9d1a6973..52807264 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -119,7 +119,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica return nil, fmt.Errorf(`error creating HTTP client: %v`, err) } - ngxClient, err := nginxClient.NewNginxClient(httpClient, event.NginxHost) + ngxClient, err := nginxClient.NewNginxClient(event.NginxHost, nginxClient.WithHTTPClient(httpClient)) if err != nil { return nil, fmt.Errorf(`error creating Nginx Plus client: %v`, err) } From cb78349f25031c69f44a80642b60c749be9032ef Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 31 Jul 2024 15:58:35 -0600 Subject: [PATCH 016/110] NLB-5065 Operator adds API Key to header --- internal/communication/factory.go | 13 ++++++--- internal/communication/factory_test.go | 29 +++++++++++++++++++-- internal/communication/roundtripper_test.go | 16 +++++++----- internal/configuration/settings.go | 9 +++++++ 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 40bf02a7..64ffe687 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -7,6 +7,7 @@ package communication import ( "crypto/tls" + "fmt" netHttp "net/http" "time" @@ -19,7 +20,7 @@ import ( // RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, // the Headers are configured for JSON. func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { - headers := NewHeaders() + headers := NewHeaders(settings.APIKey) tlsConfig := NewTLSConfig(settings) transport := NewTransport(tlsConfig) roundTripper := NewRoundTripper(headers, transport) @@ -33,11 +34,17 @@ func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { } // NewHeaders is a factory method to create a new basic Http Headers slice. -func NewHeaders() []string { - return []string{ +func NewHeaders(apiKey string) []string { + headers := []string{ "Content-Type: application/json", "Accept: application/json", } + + if apiKey != "" { + headers = append(headers, fmt.Sprintf("Authorization: ApiKey %s", apiKey)) + } + + return headers } // NewTLSConfig is a factory method to create a new basic Tls Config. diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 375da3b2..398bb6ca 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -21,7 +21,6 @@ func TestNewHTTPClient(t *testing.T) { t.Fatalf(`Unexpected error: %v`, err) } client, err := NewHTTPClient(settings) - if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -31,9 +30,35 @@ func TestNewHTTPClient(t *testing.T) { } } +//nolint:goconst func TestNewHeaders(t *testing.T) { t.Parallel() - headers := NewHeaders() + headers := NewHeaders("fakeKey") + + if headers == nil { + t.Fatalf(`headers should not be nil`) + } + + if len(headers) != 3 { + t.Fatalf(`headers should have 3 elements`) + } + + if headers[0] != "Content-Type: application/json" { + t.Fatalf(`headers[0] should be "Content-Type: application/json"`) + } + + if headers[1] != "Accept: application/json" { + t.Fatalf(`headers[1] should be "Accept: application/json"`) + } + + if headers[2] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + } +} + +func TestNewHeadersWithNoAPIKey(t *testing.T) { + t.Parallel() + headers := NewHeaders("") if headers == nil { t.Fatalf(`headers should not be nil`) diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index ff6d5c4f..abee4555 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -19,7 +19,7 @@ func TestNewRoundTripper(t *testing.T) { t.Parallel() k8sClient := fake.NewSimpleClientset() settings, _ := configuration.NewSettings(context.Background(), k8sClient) - headers := NewHeaders() + headers := NewHeaders("fakeKey") transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) @@ -31,8 +31,8 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers should not be nil`) } - if len(roundTripper.Headers) != 2 { - t.Fatalf(`roundTripper.Headers should have 2 elements`) + if len(roundTripper.Headers) != 3 { + t.Fatalf(`roundTripper.Headers should have 3 elements`) } if roundTripper.Headers[0] != "Content-Type: application/json" { @@ -43,6 +43,10 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers[1] should be "Accept: application/json"`) } + if roundTripper.Headers[2] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + } + if roundTripper.RoundTripper == nil { t.Fatalf(`roundTripper.RoundTripper should not be nil`) } @@ -55,7 +59,7 @@ func TestRoundTripperRoundTrip(t *testing.T) { if err != nil { t.Fatalf(`Unexpected error: %v`, err) } - headers := NewHeaders() + headers := NewHeaders("fakeKey") transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) @@ -78,8 +82,8 @@ func TestRoundTripperRoundTrip(t *testing.T) { defer response.Body.Close() headerLen := len(response.Header) - if headerLen <= 2 { - t.Fatalf(`response.Header should have at least 2 elements, found %d`, headerLen) + if headerLen <= 3 { + t.Fatalf(`response.Header should have at least 3 elements, found %d`, headerLen) } } diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 857bd433..400b3114 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -7,7 +7,9 @@ package configuration import ( "context" + "encoding/base64" "fmt" + "os" "strings" "time" @@ -119,6 +121,9 @@ type Settings struct { // with the Border Servers (see: ../../docs/tls/README.md). TLSMode TLSMode + // APIKey is the api key used to authenticate with the dataplane API. + APIKey string + // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. Certificates *certification.Certificates @@ -143,10 +148,14 @@ type Settings struct { // NewSettings creates a new Settings object with default values. func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings, error) { + // get base64 encoded version of raw api key set by user + apiKey := base64.StdEncoding.EncodeToString([]byte(os.Getenv("NGINXAAS_DATAPLANE_API_KEY"))) + settings := &Settings{ Context: ctx, K8sClient: k8sClient, TLSMode: NoTLS, + APIKey: apiKey, Certificates: nil, Handler: HandlerSettings{ RetryCount: 5, From d219d43791699b534613d83acd039a9fbd82b364 Mon Sep 17 00:00:00 2001 From: sarna Date: Sun, 1 Sep 2024 19:57:16 -0700 Subject: [PATCH 017/110] Publish helm charts for development Helm will be used as part of the user story to deploy the operator but it is also a good tool to deploy the operator while developing it. This commit adds the ability to publish helm charts: - to the dev registry for local iteration. - to the regular devops registry for CI iteration and testing. This will also help us test the helm chart itself. --- .gitignore | 2 ++ scripts/publish-helm.sh | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100755 scripts/publish-helm.sh diff --git a/.gitignore b/.gitignore index 4e1c0fd2..396ec9f7 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,5 @@ results .go/pkg/mod .go-build + +nginx-loadbalancer-kubernetes-* diff --git a/scripts/publish-helm.sh b/scripts/publish-helm.sh new file mode 100755 index 00000000..45e94aaf --- /dev/null +++ b/scripts/publish-helm.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) + +publish_helm() { + pkg="nginx-loadbalancer-kubernetes-${VERSION}.tgz" + helm package --version "${VERSION}" --app-version "${VERSION}" charts/nlk + helm push "${pkg}" "${repo}" +} + +init_ci_vars() { + if [ -z "$CI_PROJECT_NAME" ]; then + CI_PROJECT_NAME=$(basename "$ROOT_DIR") + fi + if [ -z "$CI_COMMIT_REF_SLUG" ]; then + CI_COMMIT_REF_SLUG=$( + git rev-parse --abbrev-ref HEAD | tr "[:upper:]" "[:lower:]" \ + | LANG=en_US.utf8 sed -E -e 's/[^a-zA-Z0-9]/-/g' -e 's/^-+|-+$$//g' \ + | cut -c 1-63 + ) + fi +} + +# MAIN +init_ci_vars + +# shellcheck source=/dev/null +source "${ROOT_DIR}/.devops.sh" +if [ "$CI" != "true" ]; then + devops.backend.docker.set "azure.container-registry-dev" +fi +repo="oci://${DEVOPS_DOCKER_URL}/nginx-azure-lb/${CI_PROJECT_NAME}/charts/${CI_COMMIT_REF_SLUG}" +# shellcheck source=/dev/null +# shellcheck disable=SC2153 +version=$(source "${ROOT_DIR}/version";echo "$VERSION") + +publish_helm From 438be427090782eac663ca5b54216df5705b69d3 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 22 Aug 2024 17:44:57 -0700 Subject: [PATCH 018/110] Update Chart version to be 0.1.0 0.0.1 is okay but it's not really a bug fix. --- charts/nlk/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index c11d8853..e82eaaba 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -16,4 +16,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.0.1 +version: 0.1.0 From ab3ffb92714164d81170fb764f1788ab21965303 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 26 Aug 2024 12:57:00 -0700 Subject: [PATCH 019/110] Update release script to handle dual artifacts We should keep a single release script that will publish docker images and helm charts for the official release. This commit just updates the current release script to handle both artifact types. Helm logic will follow. --- scripts/publish-helm.sh | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100755 scripts/publish-helm.sh diff --git a/scripts/publish-helm.sh b/scripts/publish-helm.sh deleted file mode 100755 index 45e94aaf..00000000 --- a/scripts/publish-helm.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -ROOT_DIR=$(git rev-parse --show-toplevel) - -publish_helm() { - pkg="nginx-loadbalancer-kubernetes-${VERSION}.tgz" - helm package --version "${VERSION}" --app-version "${VERSION}" charts/nlk - helm push "${pkg}" "${repo}" -} - -init_ci_vars() { - if [ -z "$CI_PROJECT_NAME" ]; then - CI_PROJECT_NAME=$(basename "$ROOT_DIR") - fi - if [ -z "$CI_COMMIT_REF_SLUG" ]; then - CI_COMMIT_REF_SLUG=$( - git rev-parse --abbrev-ref HEAD | tr "[:upper:]" "[:lower:]" \ - | LANG=en_US.utf8 sed -E -e 's/[^a-zA-Z0-9]/-/g' -e 's/^-+|-+$$//g' \ - | cut -c 1-63 - ) - fi -} - -# MAIN -init_ci_vars - -# shellcheck source=/dev/null -source "${ROOT_DIR}/.devops.sh" -if [ "$CI" != "true" ]; then - devops.backend.docker.set "azure.container-registry-dev" -fi -repo="oci://${DEVOPS_DOCKER_URL}/nginx-azure-lb/${CI_PROJECT_NAME}/charts/${CI_COMMIT_REF_SLUG}" -# shellcheck source=/dev/null -# shellcheck disable=SC2153 -version=$(source "${ROOT_DIR}/version";echo "$VERSION") - -publish_helm From 024ed4e3a2a473a5ca7a5f88cadd04cf29bc6ffc Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 16 Sep 2024 09:12:00 -0600 Subject: [PATCH 020/110] NLB-5549 Translator allows hyphens in upstream name Previously, owing to a bug, if the name of the upstream included hyphens it would be rejected by the operator. --- internal/translation/translator.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 9dad7f81..fe8532c8 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -79,15 +79,14 @@ func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamSe // getContextAndUpstreamName returns the nginx context being supplied by the port (either "http" or "stream") // and the upstream name. func getContextAndUpstreamName(port v1.ServicePort) (clientType string, appName string, err error) { - parts := strings.Split(port.Name, "-") - if len(parts) != 2 { + context, upstreamName, found := strings.Cut(port.Name, "-") + switch { + case !found: return clientType, appName, fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", port.Name) - } - - if parts[0] != "http" && parts[0] != "stream" { + case context != "http" && context != "stream": return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", port.Name) + default: + return context, upstreamName, nil } - - return parts[0], parts[1], nil } From a9f024e1cbf31023d1d797a2d1c9f6280c9ba2e3 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 12 Sep 2024 09:36:55 -0700 Subject: [PATCH 021/110] Keep major version to 0 We are going to release with `0.x.y` and keeping the internal version inline with what will get published out will reduce confusion. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index e0212c0d..a8eecae5 100644 --- a/version +++ b/version @@ -1 +1 @@ -export VERSION="1.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" +export VERSION="0.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" From cab489e447f138aacc758221cc3ca4e5bd1c7854 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 20 Sep 2024 16:26:55 -0600 Subject: [PATCH 022/110] Use semantic versioning This repository is going to produce artifacts that will be available publicly and the end users will care about semantic versioning. We need to be able to map a public facing version to internally produced artifacts easily and having semver internally eases that work. This commit does not enforce the versioning but adds a version file that has the semver, which will be used to version the product. We can follow a workflow where during release time, we cut a release, which creates a tag and we retag existing dev artifacts to be shipped as an official artifact. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index a8eecae5..6e8bf73a 100644 --- a/version +++ b/version @@ -1 +1 @@ -export VERSION="0.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" +0.1.0 From 8f6ca531fdf1478ea292c3e76314896ede0b3bf9 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 11:05:05 -0700 Subject: [PATCH 023/110] Rename the chart to nginxaas-operator While a cosmetic change, it does impact how dokerhub repo needs to be setup to publish the helm chart. In addition to that, it impacts what the user see on k8s itself and it should not be nginx-loadbalancer-kubernetes as that is confusing. --- charts/nlk/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index e82eaaba..1f986a3d 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.1.0 description: NGINX LoadBalancer for Kubernetes -name: nginx-loadbalancer-kubernetes +name: nginxaas-operator home: https://github.com/nginxinc/nginx-loadbalancer-kubernetes icon: https://raw.githubusercontent.com/nginxinc/nginx-loadbalancer-kubernetes/main/nlk-logo.svg keywords: From fa944bdde27dfed05c36261e74a64bd34c28371c Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 13:04:36 -0700 Subject: [PATCH 024/110] Update name to be operator While the chartname (which was renamed in the prior commit) gets used for naming stuff, it makes sense to also change the name value in the chart itself to use the new name. --- charts/nlk/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 394bc1fb..ab957a63 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -1,5 +1,5 @@ nlk: - name: nginx-loadbalancer-kubernetes + name: nginxaas-operator kind: deployment From 8f472233eea7699d1808bb7e58eb18abdc9a81be Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 21:05:34 -0700 Subject: [PATCH 025/110] Update registry paths to dockerhub Now that official images exist on docker hub, we should use those images in our charts. --- charts/nlk/values.yaml | 36 ++++++++++++++++++------------------ version | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index ab957a63..11d082bd 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -4,18 +4,18 @@ nlk: kind: deployment replicaCount: 1 - + image: - registry: ghcr.io - repository: nginxinc/nginx-loadbalancer-kubernetes + registry: registry-1.docker.io + repository: nginx/nginxaas-operator pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: latest - + tag: release-0.1.0 + imagePullSecrets: [] nameOverride: "" fullnameOverride: "" - + serviceAccount: # Specifies whether a service account should be created create: true @@ -23,13 +23,13 @@ nlk: automount: true # Annotations to add to the service account annotations: {} - + podAnnotations: {} podLabels: {} - + podSecurityContext: {} # fsGroup: 2000 - + securityContext: {} # capabilities: # drop: @@ -37,11 +37,11 @@ nlk: # readOnlyRootFilesystem: true # runAsNonRoot: true # runAsUser: 1000 - + service: type: ClusterIP port: 80 - + ingress: enabled: false className: "" @@ -57,7 +57,7 @@ nlk: # - secretName: chart-example-tls # hosts: # - chart-example.local - + resources: requests: cpu: 100m @@ -65,31 +65,31 @@ nlk: # limits: # cpu: 100m # memory: 128Mi - + autoscaling: enabled: false minReplicas: 1 maxReplicas: 3 targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 - + # Additional volumes on the output Deployment definition. volumes: [] # - name: foo # secret: # secretName: mysecret # optional: false - + # Additional volumeMounts on the output Deployment definition. volumeMounts: [] # - name: foo # mountPath: "/etc/foo" # readOnly: true - + nodeSelector: {} - + tolerations: [] - + affinity: {} config: diff --git a/version b/version index 6e8bf73a..d917d3e2 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.0 +0.1.2 From 04268e7b9d71f55a1313cbf9d20b73f3feb4057e Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 21:06:37 -0700 Subject: [PATCH 026/110] Update gitignore with new chart name Now that the chart has been renamed, gitignore needs to know about ignoring new charts. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 396ec9f7..f434c43c 100644 --- a/.gitignore +++ b/.gitignore @@ -88,4 +88,4 @@ results .go-build -nginx-loadbalancer-kubernetes-* +nginxaas-operator-* From 5d03f43c8112e843f06f855e8ab2c393e75dbbd7 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 25 Sep 2024 00:40:39 -0700 Subject: [PATCH 027/110] Add support for image pull secrets This lets us test the operator from the private registry. Also, in case a customer does not want to pull from docker hub and instead use their own registry, they can do so and specify a pull secret for the image. --- charts/nlk/templates/nlk-deployment.yaml | 4 ++++ version | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index fb55d77c..2653d0bc 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -15,6 +15,10 @@ spec: labels: app: nlk spec: + {{- with .Values.nlk.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} image: {{ include "nlk.image" .}} diff --git a/version b/version index d917d3e2..b1e80bb2 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.2 +0.1.3 From dbce0b185bdfb1f9ca28bca3443b0c61cafb7f44 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 26 Sep 2024 19:13:06 -0700 Subject: [PATCH 028/110] Mount configmap as a volume With the change to make the operator read a config file (to reduce code complexity), we need to make sure that the file exists for the service to start. --- charts/nlk/templates/nlk-configmap.yaml | 12 ++++++------ charts/nlk/templates/nlk-deployment.yaml | 7 +++++++ version | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 482b8cbf..ab8977f4 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -4,11 +4,11 @@ metadata: name: nlk-config namespace: nlk data: + config.yaml: | {{- if .Values.nlk.config.entries.hosts }} - nginx-hosts: "{{ .Values.nlk.config.entries.hosts }}" + nginx-hosts: "{{ .Values.nlk.config.entries.hosts }}" {{- end }} - tls-mode: "{{ index .Values.nlk.defaultTLS "tls-mode" }}" - ca-certificate: "{{ index .Values.nlk.defaultTLS "ca-certificate" }}" - client-certificate: "{{ index .Values.nlk.defaultTLS "client-certificate" }}" - log-level: "{{ .Values.nlk.logLevel }}" - + tls-mode: "{{ index .Values.nlk.defaultTLS "tls-mode" }}" + ca-certificate: "{{ index .Values.nlk.defaultTLS "ca-certificate" }}" + client-certificate: "{{ index .Values.nlk.defaultTLS "client-certificate" }}" + log-level: "{{ .Values.nlk.logLevel }}" diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 2653d0bc..3b5348ec 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -45,4 +45,11 @@ spec: initialDelaySeconds: {{ .Values.nlk.readyStatus.initialDelaySeconds }} periodSeconds: {{ .Values.nlk.readyStatus.periodSeconds }} {{- end }} + volumeMounts: + - name: config + mountPath: /etc/nginxaas-operator serviceAccountName: {{ include "nlk.fullname" . }} + volumes: + - name: config + configMap: + name: nlk-config diff --git a/version b/version index b1e80bb2..845639ee 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.3 +0.1.4 From 57f46b732f02f1cf9897225643a7ae4679a8db16 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 26 Sep 2024 19:24:21 -0700 Subject: [PATCH 029/110] Drop configmap access for operator Once the operator reads from the configmap mounted as a volume, it does not need access to the k8s API to read the config map itself. --- charts/nlk/templates/clusterrole.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index 4164475e..0ab90bb9 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -7,7 +7,6 @@ rules: - apiGroups: - "" resources: - - configmaps - nodes - secrets - services From 30524231d2cdc2a66754e0f3aa1f9d21d02d72db Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 27 Sep 2024 12:40:24 -0700 Subject: [PATCH 030/110] Grant permissions to read configmaps This is needed temporarily while nlk still needs to read the configmap and the code for swapping it out with a config file does not land. The order of merges that I am thinking: - Get this MR landed. - Land testenv changes to use helm. - Land code to read settings from configfile. - Remove configmap permissions. --- charts/nlk/templates/clusterrole.yaml | 1 + version | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index 0ab90bb9..4164475e 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -7,6 +7,7 @@ rules: - apiGroups: - "" resources: + - configmaps - nodes - secrets - services diff --git a/version b/version index 845639ee..9faa1b7a 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.4 +0.1.5 From 6b2578c5658ce9be189c6908717cc2468bc91c84 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 5 Sep 2024 08:54:06 -0600 Subject: [PATCH 031/110] NLB-5342 Configuration settings now read from config file This means that we do not have to grant the operator app any permissions to access kubernetes secrets. Reading from the config file changes the application's behavior in that settings are no longer changed whenever a config map is updated at run time, as they used to be. Settings are now concurrency-safe, because they pass new values instead of a shared pointer to their consumers in separate goroutines. Settings also no longer pass the application context to consumers as this is a well-document go anti-pattern. Context should always be passed as a function parameter. Any operator modules that need access to a kubernetes client are constructed with a reference to the client, instead of gaining access to the client through settings. --- cmd/configuration-test-harness/doc.go | 1 - cmd/configuration-test-harness/main.go | 81 ---- cmd/nginx-loadbalancer-kubernetes/main.go | 46 ++- cmd/tls-config-factory-test-harness/main.go | 2 +- go.mod | 40 +- go.sum | 377 ++++-------------- internal/authentication/factory.go | 2 +- internal/authentication/factory_test.go | 21 +- internal/certification/certificates_test.go | 5 +- internal/communication/factory.go | 4 +- internal/communication/factory_test.go | 17 +- internal/communication/roundtripper_test.go | 21 +- internal/configuration/configuration_test.go | 52 +++ internal/configuration/settings.go | 277 ++----------- internal/configuration/testdata/test.yaml | 11 + internal/observation/handler.go | 6 +- internal/observation/handler_test.go | 21 +- internal/observation/watcher.go | 53 +-- internal/observation/watcher_test.go | 5 +- internal/synchronization/synchronizer.go | 4 +- internal/synchronization/synchronizer_test.go | 74 ++-- 21 files changed, 355 insertions(+), 765 deletions(-) delete mode 100644 cmd/configuration-test-harness/doc.go delete mode 100644 cmd/configuration-test-harness/main.go create mode 100644 internal/configuration/configuration_test.go create mode 100644 internal/configuration/testdata/test.yaml diff --git a/cmd/configuration-test-harness/doc.go b/cmd/configuration-test-harness/doc.go deleted file mode 100644 index 06ab7d0f..00000000 --- a/cmd/configuration-test-harness/doc.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/cmd/configuration-test-harness/main.go b/cmd/configuration-test-harness/main.go deleted file mode 100644 index 5079a9d0..00000000 --- a/cmd/configuration-test-harness/main.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "path/filepath" - - configuration2 "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" -) - -func main() { - logrus.SetLevel(logrus.DebugLevel) - err := run() - if err != nil { - logrus.Fatal(err) - } -} - -func run() error { - logrus.Info("configuration-test-harness::run") - - ctx := context.Background() - var err error - - k8sClient, err := buildKubernetesClient() - if err != nil { - return fmt.Errorf(`error building a Kubernetes client: %w`, err) - } - - configuration, err := configuration2.NewSettings(ctx, k8sClient) - if err != nil { - return fmt.Errorf(`error occurred creating configuration: %w`, err) - } - - err = configuration.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing configuration: %w`, err) - } - - go configuration.Run() - - <-ctx.Done() - - return err -} - -func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") - - var kubeconfig *string - var k8sConfig *rest.Config - - k8sConfig, err := rest.InClusterConfig() - if errors.Is(err, rest.ErrNotInCluster) { - if home := homedir.HomeDir(); home != "" { - path := filepath.Join(home, ".kube", "config") - kubeconfig = &path - - k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - return nil, fmt.Errorf(`error occurred building the kubeconfig: %w`, err) - } - } else { - return nil, fmt.Errorf(`not running in a Cluster: %w`, err) - } - } else if err != nil { - return nil, fmt.Errorf(`error occurred getting the Cluster config: %w`, err) - } - - client, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - return nil, fmt.Errorf(`error occurred creating a client: %w`, err) - } - return client, nil -} diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 89f764f4..f8473c38 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -35,17 +35,12 @@ func run() error { return fmt.Errorf(`error building a Kubernetes client: %w`, err) } - settings, err := configuration.NewSettings(ctx, k8sClient) + settings, err := configuration.Read("config.yaml", "/etc/nginxaas-operator") if err != nil { - return fmt.Errorf(`error occurred creating settings: %w`, err) + return fmt.Errorf(`error occurred accessing configuration: %w`, err) } - err = settings.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing settings: %w`, err) - } - - go settings.Run() + setLogLevel(settings.LogLevel) synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) @@ -58,12 +53,12 @@ func run() error { handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue) - watcher, err := observation.NewWatcher(settings, handler) + watcher, err := observation.NewWatcher(settings, handler, k8sClient) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } - err = watcher.Initialize() + err = watcher.Initialize(ctx) if err != nil { return fmt.Errorf(`error occurred initializing the watcher: %w`, err) } @@ -74,7 +69,7 @@ func run() error { probeServer := probation.NewHealthServer() probeServer.Start() - err = watcher.Watch() + err = watcher.Watch(ctx) if err != nil { return fmt.Errorf(`error occurred watching for events: %w`, err) } @@ -83,6 +78,35 @@ func run() error { return nil } +func setLogLevel(logLevel string) { + logrus.Debugf("Settings::setLogLevel: %s", logLevel) + switch logLevel { + case "panic": + logrus.SetLevel(logrus.PanicLevel) + + case "fatal": + logrus.SetLevel(logrus.FatalLevel) + + case "error": + logrus.SetLevel(logrus.ErrorLevel) + + case "warn": + logrus.SetLevel(logrus.WarnLevel) + + case "info": + logrus.SetLevel(logrus.InfoLevel) + + case "debug": + logrus.SetLevel(logrus.DebugLevel) + + case "trace": + logrus.SetLevel(logrus.TraceLevel) + + default: + logrus.SetLevel(logrus.WarnLevel) + } +} + func buildKubernetesClient() (*kubernetes.Clientset, error) { logrus.Debug("Watcher::buildKubernetesClient") k8sConfig, err := rest.InClusterConfig() diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 51f1d1d9..7b853e07 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -31,7 +31,7 @@ func main() { logrus.Infof("\n\n\t*** Building TLS config for <<< %s >>>\n\n", name) - tlsConfig, err := authentication.NewTLSConfig(&settings.Settings) + tlsConfig, err := authentication.NewTLSConfig(settings.Settings) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 118de5f5..056c7e2f 100644 --- a/go.mod +++ b/go.mod @@ -11,42 +11,60 @@ toolchain go1.21.4 require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/sirupsen/logrus v1.9.0 + github.com/spf13/viper v1.19.0 + github.com/stretchr/testify v1.9.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.6 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.80.1 // indirect diff --git a/go.sum b/go.sum index 46a175a7..098adea1 100644 --- a/go.sum +++ b/go.sum @@ -1,63 +1,26 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -69,86 +32,59 @@ github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -164,296 +100,157 @@ github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -463,12 +260,7 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= @@ -481,9 +273,6 @@ k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+O k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index 32add620..a51d7ab4 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -18,7 +18,7 @@ import ( "github.com/sirupsen/logrus" ) -func NewTLSConfig(settings *configuration.Settings) (*tls.Config, error) { +func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { logrus.Debugf("authentication::NewTLSConfig Creating TLS config for mode: '%s'", settings.TLSMode) switch settings.TLSMode { diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go index e9015c07..6b5fcaf9 100644 --- a/internal/authentication/factory_test.go +++ b/internal/authentication/factory_test.go @@ -20,9 +20,8 @@ const ( func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { t.Parallel() - settings := configuration.Settings{} - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(configuration.Settings{}) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -50,7 +49,7 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { }, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -84,7 +83,7 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -108,7 +107,7 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -133,7 +132,7 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -168,7 +167,7 @@ func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -193,7 +192,7 @@ func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -209,7 +208,7 @@ func TestTlsFactory_CaTlsMode(t *testing.T) { TLSMode: configuration.CertificateAuthorityTLS, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -245,7 +244,7 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -280,7 +279,7 @@ func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index 89b21bf9..901964ac 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -94,9 +95,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { //nolint:govet,staticcheck go func() { err := certificates.Run() - if err != nil { - t.Fatalf("error running Certificates: %v", err) - } + assert.NoError(t, err, "expected no error running certificates") }() cache.WaitForCacheSync(ctx.Done(), certificates.informer.HasSynced) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 64ffe687..2a3c09a6 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -19,7 +19,7 @@ import ( // NewHTTPClient is a factory method to create a new Http Client with a default configuration. // RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, // the Headers are configured for JSON. -func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { +func NewHTTPClient(settings configuration.Settings) (*netHttp.Client, error) { headers := NewHeaders(settings.APIKey) tlsConfig := NewTLSConfig(settings) transport := NewTransport(tlsConfig) @@ -49,7 +49,7 @@ func NewHeaders(apiKey string) []string { // NewTLSConfig is a factory method to create a new basic Tls Config. // More attention should be given to the use of `InsecureSkipVerify: true`, as it is not recommended for production use. -func NewTLSConfig(settings *configuration.Settings) *tls.Config { +func NewTLSConfig(settings configuration.Settings) *tls.Config { tlsConfig, err := authentication.NewTLSConfig(settings) if err != nil { logrus.Warnf("Failed to create TLS config: %v", err) diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 398bb6ca..65f5e5bb 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -6,21 +6,13 @@ package communication import ( - "context" "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" ) func TestNewHTTPClient(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, err := configuration.NewSettings(context.Background(), k8sClient) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - client, err := NewHTTPClient(settings) + + client, err := NewHTTPClient(defaultSettings()) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -79,9 +71,8 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { func TestNewTransport(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, _ := configuration.NewSettings(context.Background(), k8sClient) - config := NewTLSConfig(settings) + + config := NewTLSConfig(defaultSettings()) transport := NewTransport(config) if transport == nil { diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index abee4555..9913600f 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -7,20 +7,17 @@ package communication import ( "bytes" - "context" netHttp "net/http" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" ) func TestNewRoundTripper(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, _ := configuration.NewSettings(context.Background(), k8sClient) + headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(settings)) + transport := NewTransport(NewTLSConfig(defaultSettings())) roundTripper := NewRoundTripper(headers, transport) if roundTripper == nil { @@ -54,13 +51,9 @@ func TestNewRoundTripper(t *testing.T) { func TestRoundTripperRoundTrip(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, err := configuration.NewSettings(context.Background(), k8sClient) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } + headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(settings)) + transport := NewTransport(NewTLSConfig(defaultSettings())) roundTripper := NewRoundTripper(headers, transport) request, err := NewRequest("GET", "http://example.com", nil) @@ -95,3 +88,9 @@ func NewRequest(method string, url string, body []byte) (*netHttp.Request, error return request, nil } + +func defaultSettings() configuration.Settings { + return configuration.Settings{ + TLSMode: configuration.NoTLS, + } +} diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go new file mode 100644 index 00000000..96949891 --- /dev/null +++ b/internal/configuration/configuration_test.go @@ -0,0 +1,52 @@ +package configuration_test + +import ( + "testing" + "time" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + + "github.com/stretchr/testify/require" +) + +func TestConfiguration(t *testing.T) { + t.Parallel() + expectedSettings := configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, + TLSMode: configuration.NoTLS, + Certificates: &certification.Certificates{ + CaCertificateSecretKey: "fakeCAKey", + ClientCertificateSecretKey: "fakeCertKey", + }, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, + } + + settings, err := configuration.Read("test", "./testdata") + require.NoError(t, err) + require.Equal(t, expectedSettings, settings) +} diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 400b3114..91ff27ce 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -6,21 +6,14 @@ package configuration import ( - "context" "encoding/base64" "fmt" - "os" - "strings" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" + + "github.com/spf13/viper" ) const ( @@ -111,8 +104,8 @@ type SynchronizerSettings struct { // Settings contains the configuration values needed by the application. type Settings struct { - // Context is the context used to control the application. - Context context.Context + // LogLevel is the user-specified log level. Defaults to warn. + LogLevel string // NginxPlusHosts is a list of Nginx Plus hosts that will be used to update the Border Servers. NginxPlusHosts []string @@ -127,15 +120,6 @@ type Settings struct { // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. Certificates *certification.Certificates - // K8sClient is the Kubernetes client used to communicate with the Kubernetes API. - K8sClient kubernetes.Interface - - // informer is the SharedInformer used to watch for changes to the ConfigMap . - informer cache.SharedInformer - - // eventHandlerRegistration is the object used to track the event handlers with the SharedInformer. - eventHandlerRegistration cache.ResourceEventHandlerRegistration - // Handler contains the configuration values needed by the Handler. Handler HandlerSettings @@ -146,17 +130,41 @@ type Settings struct { Watcher WatcherSettings } -// NewSettings creates a new Settings object with default values. -func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings, error) { - // get base64 encoded version of raw api key set by user - apiKey := base64.StdEncoding.EncodeToString([]byte(os.Getenv("NGINXAAS_DATAPLANE_API_KEY"))) - - settings := &Settings{ - Context: ctx, - K8sClient: k8sClient, - TLSMode: NoTLS, - APIKey: apiKey, - Certificates: nil, +// Read parses all the config and returns the values +func Read(configName, configPath string) (s Settings, err error) { + v := viper.New() + v.SetConfigName(configName) + v.SetConfigType("yaml") + v.AddConfigPath(configPath) + if err = v.ReadInConfig(); err != nil { + return s, err + } + + if err = v.BindEnv("NGINXAAS_DATAPLANE_API_KEY"); err != nil { + return s, err + } + + tlsMode := NoTLS + if t, err := validateTLSMode(v.GetString("tls-mode")); err != nil { + logrus.Errorf("could not validate tls mode: %v", err) + } else { + tlsMode = t + } + + serviceAnnotation := DefaultServiceAnnotation + if sa := v.GetString(ServiceAnnotationMatchKey); sa != "" { + serviceAnnotation = sa + } + + return Settings{ + LogLevel: v.GetString("log-level"), + NginxPlusHosts: v.GetStringSlice("nginx-hosts"), + TLSMode: tlsMode, + APIKey: base64.StdEncoding.EncodeToString([]byte(v.GetString("NGINXAAS_DATAPLANE_API_KEY"))), + Certificates: &certification.Certificates{ + CaCertificateSecretKey: v.GetString("ca-certificate"), + ClientCertificateSecretKey: v.GetString("client-certificate"), + }, Handler: HandlerSettings{ RetryCount: 5, Threads: 1, @@ -179,216 +187,15 @@ func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings }, Watcher: WatcherSettings{ ResyncPeriod: 0, - ServiceAnnotation: DefaultServiceAnnotation, + ServiceAnnotation: serviceAnnotation, }, - } - - return settings, nil -} - -// Initialize initializes the Settings object. Sets up a SharedInformer to watch for changes to the ConfigMap. -// This method must be called before the Run method. -func (s *Settings) Initialize() error { - logrus.Info("Settings::Initialize") - - var err error - - certificates := certification.NewCertificates(s.Context, s.K8sClient) - - err = certificates.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing certificates: %w`, err) - } - - s.Certificates = certificates - - go certificates.Run() //nolint:errcheck - - logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieving nlk-config ConfigMap") - configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get( - s.Context, "nlk-config", metav1.GetOptions{}, - ) - if err != nil { - return err - } - - s.handleUpdateEvent(nil, configMap) - logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieved nlk-config ConfigMap") - - informer := s.buildInformer() - - s.informer = informer - - err = s.initializeEventListeners() - if err != nil { - return fmt.Errorf(`error occurred initializing event listeners: %w`, err) - } - - return nil -} - -// Run starts the SharedInformer and waits for the Context to be canceled. -func (s *Settings) Run() { - logrus.Debug("Settings::Run") - - defer utilruntime.HandleCrash() - - go s.informer.Run(s.Context.Done()) - - <-s.Context.Done() -} - -func (s *Settings) buildInformer() cache.SharedInformer { - options := informers.WithNamespace(ConfigMapsNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(s.K8sClient, ResyncPeriod, options) - informer := factory.Core().V1().ConfigMaps().Informer() - - return informer -} - -func (s *Settings) initializeEventListeners() error { - logrus.Debug("Settings::initializeEventListeners") - - var err error - - handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: s.handleAddEvent, - UpdateFunc: s.handleUpdateEvent, - DeleteFunc: s.handleDeleteEvent, - } - - s.eventHandlerRegistration, err = s.informer.AddEventHandler(handlers) - if err != nil { - return fmt.Errorf(`error occurred registering event handlers: %w`, err) - } - - return nil + }, nil } -func (s *Settings) handleAddEvent(obj interface{}) { - logrus.Debug("Settings::handleAddEvent") - - if _, yes := isOurConfig(obj); yes { - s.handleUpdateEvent(nil, obj) - } -} - -func (s *Settings) handleDeleteEvent(obj interface{}) { - logrus.Debug("Settings::handleDeleteEvent") - - if _, yes := isOurConfig(obj); yes { - s.updateHosts([]string{}) - } -} - -func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { - logrus.Debug("Settings::handleUpdateEvent") - - configMap, yes := isOurConfig(newValue) - if !yes { - return - } - - hosts, found := configMap.Data["nginx-hosts"] - if found { - newHosts := s.parseHosts(hosts) - s.updateHosts(newHosts) - } else { - logrus.Warnf("Settings::handleUpdateEvent: nginx-hosts key not found in ConfigMap") - } - - tlsMode, err := validateTLSMode(configMap) - if err != nil { - // NOTE: the TLSMode defaults to NoTLS on startup, or the last known good value if previously set. - logrus.Errorf( - "Error with configured TLS Mode. TLS Mode has NOT been changed. The current mode is: '%v'. Error: %v. ", - s.TLSMode, err, - ) - } else { - s.TLSMode = tlsMode - } - - caCertificateSecretKey, found := configMap.Data["ca-certificate"] - if found { - s.Certificates.CaCertificateSecretKey = caCertificateSecretKey - logrus.Debugf("Settings::handleUpdateEvent: ca-certificate: %s", s.Certificates.CaCertificateSecretKey) - } else { - s.Certificates.CaCertificateSecretKey = "" - logrus.Warnf("Settings::handleUpdateEvent: ca-certificate key not found in ConfigMap") - } - - clientCertificateSecretKey, found := configMap.Data["client-certificate"] - if found { - s.Certificates.ClientCertificateSecretKey = clientCertificateSecretKey - logrus.Debugf("Settings::handleUpdateEvent: client-certificate: %s", s.Certificates.ClientCertificateSecretKey) - } else { - s.Certificates.ClientCertificateSecretKey = "" - logrus.Warnf("Settings::handleUpdateEvent: client-certificate key not found in ConfigMap") - } - - if serviceAnnotation, found := configMap.Data[ServiceAnnotationMatchKey]; found { - s.Watcher.ServiceAnnotation = serviceAnnotation - } else { - s.Watcher.ServiceAnnotation = DefaultServiceAnnotation - } - logrus.Debugf("Settings::handleUpdateEvent: %s: %s", ServiceAnnotationMatchKey, s.Watcher.ServiceAnnotation) - - setLogLevel(configMap.Data["log-level"]) - - logrus.Debugf("Settings::handleUpdateEvent: \n\tHosts: %v,\n\tSettings: %v ", s.NginxPlusHosts, configMap) -} - -func validateTLSMode(configMap *corev1.ConfigMap) (TLSMode, error) { - tlsConfigMode, tlsConfigModeFound := configMap.Data["tls-mode"] - if !tlsConfigModeFound { - return NoTLS, fmt.Errorf(`tls-mode key not found in ConfigMap`) - } - +func validateTLSMode(tlsConfigMode string) (TLSMode, error) { if tlsMode, tlsModeFound := TLSModeMap[tlsConfigMode]; tlsModeFound { return tlsMode, nil } return NoTLS, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) } - -func (s *Settings) parseHosts(hosts string) []string { - return strings.Split(hosts, ",") -} - -func (s *Settings) updateHosts(hosts []string) { - s.NginxPlusHosts = hosts -} - -func isOurConfig(obj interface{}) (*corev1.ConfigMap, bool) { - configMap, ok := obj.(*corev1.ConfigMap) - return configMap, ok && configMap.Name == ConfigMapName && configMap.Namespace == ConfigMapsNamespace -} - -func setLogLevel(logLevel string) { - logrus.Debugf("Settings::setLogLevel: %s", logLevel) - switch logLevel { - case "panic": - logrus.SetLevel(logrus.PanicLevel) - - case "fatal": - logrus.SetLevel(logrus.FatalLevel) - - case "error": - logrus.SetLevel(logrus.ErrorLevel) - - case "warn": - logrus.SetLevel(logrus.WarnLevel) - - case "info": - logrus.SetLevel(logrus.InfoLevel) - - case "debug": - logrus.SetLevel(logrus.DebugLevel) - - case "trace": - logrus.SetLevel(logrus.TraceLevel) - - default: - logrus.SetLevel(logrus.WarnLevel) - } -} diff --git a/internal/configuration/testdata/test.yaml b/internal/configuration/testdata/test.yaml new file mode 100644 index 00000000..717dcdbe --- /dev/null +++ b/internal/configuration/testdata/test.yaml @@ -0,0 +1,11 @@ +ca-certificate: fakeCAKey +client-certificate: fakeCertKey +log-level: warn +nginx-hosts: https://10.0.0.1:9000/api +tls-mode: no-tls +service-annotation-match: fakeServiceMatch +creationTimestamp: "2024-09-04T17:59:20Z" +name: nlk-config +namespace: nlk +resourceVersion: "5909" +uid: 66d49974-49d6-4ad8-8135-dcebda7b5c9e diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 55849391..bd823f61 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -19,7 +19,6 @@ import ( // HandlerInterface is the interface for the event handler type HandlerInterface interface { - // AddRateLimitedEvent defines the interface for adding an event to the event queue AddRateLimitedEvent(event *core.Event) @@ -35,12 +34,11 @@ type HandlerInterface interface { // The translation process may result in multiple events being generated. This fan-out mainly supports the differences // in NGINX Plus API calls for creating/updating Upstreams and deleting Upstreams. type Handler struct { - // eventQueue is the queue used to store events eventQueue workqueue.RateLimitingInterface // settings is the configuration settings - settings *configuration.Settings + settings configuration.Settings // synchronizer is the synchronizer used to synchronize the internal representation with a Border Server synchronizer synchronization.Interface @@ -48,7 +46,7 @@ type Handler struct { // NewHandler creates a new event handler func NewHandler( - settings *configuration.Settings, + settings configuration.Settings, synchronizer synchronization.Interface, eventQueue workqueue.RateLimitingInterface, ) *Handler { diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index 72b6d8f4..ba4add1f 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -6,23 +6,17 @@ package observation import ( - "context" - "fmt" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" v1 "k8s.io/api/core/v1" - "k8s.io/client-go/util/workqueue" ) func TestHandler_AddsEventToSynchronizer(t *testing.T) { t.Parallel() - _, _, synchronizer, handler, err := buildHandler() - if err != nil { - t.Errorf(`should have been no error, %v`, err) - } + synchronizer, handler := buildHandler() event := &core.Event{ Type: core.Created, @@ -47,19 +41,12 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { } func buildHandler() ( - *configuration.Settings, - workqueue.RateLimitingInterface, - *mocks.MockSynchronizer, *Handler, error, + *mocks.MockSynchronizer, *Handler, ) { - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - return nil, nil, nil, nil, fmt.Errorf(`should have been no error, %v`, err) - } - eventQueue := &mocks.MockRateLimiter{} synchronizer := &mocks.MockSynchronizer{} - handler := NewHandler(settings, synchronizer, eventQueue) + handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue) - return settings, eventQueue, synchronizer, handler, nil + return synchronizer, handler } diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index a07ef125..9cc9a165 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -6,6 +6,7 @@ package observation import ( + "context" "errors" "fmt" "time" @@ -17,6 +18,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" ) @@ -33,26 +35,31 @@ type Watcher struct { // informer is the informer used to watch for changes to Kubernetes resources informer cache.SharedIndexInformer + k8sClient kubernetes.Interface + // settings is the configuration settings - settings *configuration.Settings + settings configuration.Settings } // NewWatcher creates a new Watcher -func NewWatcher(settings *configuration.Settings, handler HandlerInterface) (*Watcher, error) { +func NewWatcher( + settings configuration.Settings, handler HandlerInterface, k8sClient kubernetes.Interface, +) (*Watcher, error) { return &Watcher{ - handler: handler, - settings: settings, + handler: handler, + settings: settings, + k8sClient: k8sClient, }, nil } // Initialize initializes the Watcher, must be called before Watch -func (w *Watcher) Initialize() error { +func (w *Watcher) Initialize(ctx context.Context) error { logrus.Debug("Watcher::Initialize") var err error w.informer = w.buildInformer() - err = w.initializeEventListeners() + err = w.initializeEventListeners(ctx) if err != nil { return fmt.Errorf(`initialization error: %w`, err) } @@ -62,7 +69,7 @@ func (w *Watcher) Initialize() error { // Watch starts the process of watching for changes to Kubernetes resources. // Initialize must be called before Watch. -func (w *Watcher) Watch() error { +func (w *Watcher) Watch(ctx context.Context) error { logrus.Debug("Watcher::Watch") if w.informer == nil { @@ -72,17 +79,17 @@ func (w *Watcher) Watch() error { defer utilruntime.HandleCrash() defer w.handler.ShutDown() - go w.informer.Run(w.settings.Context.Done()) + go w.informer.Run(ctx.Done()) if !cache.WaitForNamedCacheSync( w.settings.Handler.WorkQueueSettings.Name, - w.settings.Context.Done(), + ctx.Done(), w.informer.HasSynced, ) { return fmt.Errorf(`error occurred waiting for the cache to sync`) } - <-w.settings.Context.Done() + <-ctx.Done() return nil } @@ -98,7 +105,7 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. -func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { +func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) { logrus.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) @@ -106,7 +113,7 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { return } - nodeIps, err := w.retrieveNodeIps() + nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return @@ -120,7 +127,7 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { // buildEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. -func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { +func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface{}) { logrus.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) @@ -128,7 +135,7 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { return } - nodeIps, err := w.retrieveNodeIps() + nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return @@ -142,7 +149,7 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { // buildEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. -func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { +func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface{}, interface{}) { logrus.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API @@ -151,7 +158,7 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { return } - nodeIps, err := w.retrieveNodeIps() + nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return @@ -168,7 +175,7 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { logrus.Debug("Watcher::buildInformer") factory := informers.NewSharedInformerFactoryWithOptions( - w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, + w.k8sClient, w.settings.Watcher.ResyncPeriod, ) informer := factory.Core().V1().Services().Informer() @@ -176,14 +183,14 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { } // initializeEventListeners initializes the event listeners for the informer. -func (w *Watcher) initializeEventListeners() error { +func (w *Watcher) initializeEventListeners(ctx context.Context) error { logrus.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: w.buildEventHandlerForAdd(), - DeleteFunc: w.buildEventHandlerForDelete(), - UpdateFunc: w.buildEventHandlerForUpdate(), + AddFunc: w.buildEventHandlerForAdd(ctx), + DeleteFunc: w.buildEventHandlerForDelete(ctx), + UpdateFunc: w.buildEventHandlerForUpdate(ctx), } w.eventHandlerRegistration, err = w.informer.AddEventHandler(handlers) @@ -196,13 +203,13 @@ func (w *Watcher) initializeEventListeners() error { // notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is // because the master node may or may not be a worker node and thus may not be able to route traffic. -func (w *Watcher) retrieveNodeIps() ([]string, error) { +func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { started := time.Now() logrus.Debug("Watcher::retrieveNodeIps") var nodeIps []string - nodes, err := w.settings.K8sClient.CoreV1().Nodes().List(w.settings.Context, metav1.ListOptions{}) + nodes, err := w.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { logrus.Errorf(`error occurred retrieving the list of nodes: %v`, err) return nil, err diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 2a6d94b4..f8de8496 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -17,15 +17,14 @@ import ( func TestWatcher_MustInitialize(t *testing.T) { t.Parallel() watcher, _ := buildWatcher() - if err := watcher.Watch(); err == nil { + if err := watcher.Watch(context.Background()); err == nil { t.Errorf("Expected error, got %s", err) } } func buildWatcher() (*Watcher, error) { k8sClient := &kubernetes.Clientset{} - settings, _ := configuration.NewSettings(context.Background(), k8sClient) handler := &mocks.MockHandler{} - return NewWatcher(settings, handler) + return NewWatcher(configuration.Settings{}, handler, k8sClient) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 52807264..eadfbd8d 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -39,12 +39,12 @@ type Interface interface { // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { eventQueue workqueue.RateLimitingInterface - settings *configuration.Settings + settings configuration.Settings } // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( - settings *configuration.Settings, + settings configuration.Settings, eventQueue workqueue.RateLimitingInterface, ) (*Synchronizer, error) { synchronizer := Synchronizer{ diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index 36345139..d1710b22 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -6,9 +6,9 @@ package synchronization import ( - "context" "fmt" "testing" + "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" @@ -17,14 +17,10 @@ import ( func TestSynchronizer_NewSynchronizer(t *testing.T) { t.Parallel() - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(configuration.Settings{}, rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -44,13 +40,10 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { UpstreamName: "", UpstreamServers: nil, } - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } + rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -72,14 +65,10 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 1 events := buildEvents(1) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{"https://localhost:8080"} + rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -99,18 +88,15 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { t.Parallel() const expectedEventCount = 1 events := buildEvents(1) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{ + hosts := []string{ "https://localhost:8080", "https://localhost:8081", "https://localhost:8082", } + rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -130,13 +116,9 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 events := buildEvents(4) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -158,14 +140,9 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 4 events := buildEvents(4) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{"https://localhost:8080"} rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -186,18 +163,16 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { const eventCount = 4 events := buildEvents(eventCount) rateLimiter := &mocks.MockRateLimiter{} - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{ + + hosts := []string{ "https://localhost:8080", "https://localhost:8081", "https://localhost:8082", } - expectedEventCount := eventCount * len(settings.NginxPlusHosts) - synchronizer, err := NewSynchronizer(settings, rateLimiter) + expectedEventCount := eventCount * len(hosts) + + synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -226,3 +201,20 @@ func buildEvents(count int) core.ServerUpdateEvents { } return events } + +func defaultSettings(nginxHosts ...string) configuration.Settings { + return configuration.Settings{ + NginxPlusHosts: nginxHosts, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + } +} From c20ff243e363d60a9637572bda2c45faf41149c5 Mon Sep 17 00:00:00 2001 From: Nathan Bird Date: Mon, 30 Sep 2024 15:19:01 -0400 Subject: [PATCH 032/110] Get rid of helm chart cruft --- charts/nlk/Chart.yaml | 2 +- charts/nlk/values.yaml | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 1f986a3d..bea170ba 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -7,8 +7,8 @@ home: https://github.com/nginxinc/nginx-loadbalancer-kubernetes icon: https://raw.githubusercontent.com/nginxinc/nginx-loadbalancer-kubernetes/main/nlk-logo.svg keywords: - nginx +- nginxaas - loadbalancer -- ingress kubeVersion: '>= 1.22.0-0' maintainers: - name: "@ciroque" diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 11d082bd..b32fb4ca 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -42,22 +42,6 @@ nlk: type: ClusterIP port: 80 - ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - resources: requests: cpu: 100m @@ -66,13 +50,6 @@ nlk: # cpu: 100m # memory: 128Mi - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 3 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - # Additional volumes on the output Deployment definition. volumes: [] # - name: foo From 8bd4a354e6ddbf2da333d2977d9059ea13a218b9 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 30 Sep 2024 18:29:09 -0700 Subject: [PATCH 033/110] NLB-5678: Template the namespace We should respect the user supplied release namespace in the helm chart instead of hardcoding it to `nlk`. --- charts/nlk/templates/clusterrolebinding.yaml | 2 +- charts/nlk/templates/nlk-configmap.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- charts/nlk/templates/nlk-secret.yaml | 2 +- charts/nlk/templates/nlk-serviceaccount.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/nlk/templates/clusterrolebinding.yaml b/charts/nlk/templates/clusterrolebinding.yaml index 0ccd4551..8503a242 100644 --- a/charts/nlk/templates/clusterrolebinding.yaml +++ b/charts/nlk/templates/clusterrolebinding.yaml @@ -6,7 +6,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: {{ .Release.Namespace }}-{{ include "nlk.fullname" . }} diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index ab8977f4..8cd4d664 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: nlk-config - namespace: nlk + namespace: {{ .Release.Namespace }} data: config.yaml: | {{- if .Values.nlk.config.entries.hosts }} diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 3b5348ec..35de7f1e 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} labels: app: nlk spec: diff --git a/charts/nlk/templates/nlk-secret.yaml b/charts/nlk/templates/nlk-secret.yaml index ff7d7ff7..cb964866 100644 --- a/charts/nlk/templates/nlk-secret.yaml +++ b/charts/nlk/templates/nlk-secret.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} annotations: kubernetes.io/service-account.name: {{ include "nlk.fullname" . }} type: kubernetes.io/service-account-token diff --git a/charts/nlk/templates/nlk-serviceaccount.yaml b/charts/nlk/templates/nlk-serviceaccount.yaml index 5bdca4f7..d2cd8e42 100644 --- a/charts/nlk/templates/nlk-serviceaccount.yaml +++ b/charts/nlk/templates/nlk-serviceaccount.yaml @@ -3,5 +3,5 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} {{- end }} From 5c4d5272200338728e4814b3e8ce8967467cbd69 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 27 Sep 2024 16:25:57 -0700 Subject: [PATCH 034/110] NLB-5666: Inject dataplane API key via Helm THe operator needs the dataplane API Key to authenticate with the deployment dataplane endpoint. This commit adds the abailitiy for a user to supply a key via Helm, create a secret on their behalf, and then mounts it into the pod. --- charts/nlk/templates/_helpers.tpl | 4 ++++ charts/nlk/templates/dataplaneApiKey.yaml | 8 ++++++++ charts/nlk/templates/nlk-deployment.yaml | 6 ++++++ charts/nlk/values.yaml | 2 ++ version | 2 +- 5 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 charts/nlk/templates/dataplaneApiKey.yaml diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 17a64051..119c1640 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -48,6 +48,10 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} +{{- define "nlk.apikeyname" -}} +{{- printf "%s-nginxaas-api-key" (include "nlk.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + {{/* Common labels */}} diff --git a/charts/nlk/templates/dataplaneApiKey.yaml b/charts/nlk/templates/dataplaneApiKey.yaml new file mode 100644 index 00000000..11f76f8a --- /dev/null +++ b/charts/nlk/templates/dataplaneApiKey.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include nlk.apikeyname . }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + nginxaasApiKey: {{ .Values.nlk.dataplaneApiKey | toString | b64enc }} diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 35de7f1e..eb00b3a6 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -45,6 +45,12 @@ spec: initialDelaySeconds: {{ .Values.nlk.readyStatus.initialDelaySeconds }} periodSeconds: {{ .Values.nlk.readyStatus.periodSeconds }} {{- end }} + env: + - name: NGINXAAS_DATAPLANE_API_KEY + valueFrom: + secretKeyRef: + name: {{ include nlk.apikeyname . }} + key: nginxaasApiKey volumeMounts: - name: config mountPath: /etc/nginxaas-operator diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index b32fb4ca..33889c30 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -79,6 +79,8 @@ nlk: ca-certificate: "" client-certificate: "" + dataplaneApiKey: "" + logLevel: "warn" containerPort: diff --git a/version b/version index 9faa1b7a..0ea3a944 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.5 +0.2.0 From fa80b1568d44e6251f6b20cfb3bfe51d83c391d7 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 00:29:39 -0700 Subject: [PATCH 035/110] Cleanup helm chart values These are visible values that do not do anything on the chart itself for now. Cleaning these up and we can add them as we make progress on nlk. --- charts/nlk/Chart.yaml | 4 +--- charts/nlk/values.yaml | 47 +----------------------------------------- 2 files changed, 2 insertions(+), 49 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index bea170ba..64f735aa 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,10 +1,8 @@ --- apiVersion: v2 appVersion: 0.1.0 -description: NGINX LoadBalancer for Kubernetes +description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-operator -home: https://github.com/nginxinc/nginx-loadbalancer-kubernetes -icon: https://raw.githubusercontent.com/nginxinc/nginx-loadbalancer-kubernetes/main/nlk-logo.svg keywords: - nginx - nginxaas diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 33889c30..4e7e71d5 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -24,55 +24,10 @@ nlk: # Annotations to add to the service account annotations: {} - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - resources: - requests: - cpu: 100m - memory: 128Mi - # limits: - # cpu: 100m - # memory: 128Mi - - # Additional volumes on the output Deployment definition. - volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - nodeSelector: {} - - tolerations: [] - - affinity: {} - config: entries: hosts: - "http://10.1.1.4:9000/api,http://10.1.1.5:9000/api" + "" defaultTLS: tls-mode: "no-tls" From 5d5dc3e7fb8feac68aec00df6a72d3d2858bc28f Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 11:24:40 -0700 Subject: [PATCH 036/110] Quote the helm function --- charts/nlk/templates/dataplaneApiKey.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/templates/dataplaneApiKey.yaml b/charts/nlk/templates/dataplaneApiKey.yaml index 11f76f8a..20511385 100644 --- a/charts/nlk/templates/dataplaneApiKey.yaml +++ b/charts/nlk/templates/dataplaneApiKey.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ include nlk.apikeyname . }} + name: {{ include "nlk.apikeyname" . }} namespace: {{ .Release.Namespace }} type: Opaque data: diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index eb00b3a6..bc12f5cb 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: NGINXAAS_DATAPLANE_API_KEY valueFrom: secretKeyRef: - name: {{ include nlk.apikeyname . }} + name: {{ include "nlk.apikeyname" . }} key: nginxaasApiKey volumeMounts: - name: config From c0040fad9e0444da8d742a90e407679bf86af725 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 11:58:04 -0700 Subject: [PATCH 037/110] Default the API Key value in helm The default is set as not doing so causes helm to produce an empty secret cause the pod to fail the secret mount. This was missed on original testing as I did not realize that a user could skip specifying the secret as well. --- charts/nlk/values.yaml | 3 ++- version | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 4e7e71d5..19713f51 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -34,7 +34,8 @@ nlk: ca-certificate: "" client-certificate: "" - dataplaneApiKey: "" + # Override with your own NGINXaaS dataplane API Key. + dataplaneApiKey: "test" logLevel: "warn" diff --git a/version b/version index 0ea3a944..0c62199f 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.2.0 +0.2.1 From 940387db0fb6cd687bbd3c374931006edf766dd5 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 18:41:15 -0700 Subject: [PATCH 038/110] Rename nginxaas-operator to nginxaas-loadbalancer-kubernetes --- .gitignore | 2 +- Dockerfile | 6 +++--- charts/nlk/Chart.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- charts/nlk/values.yaml | 4 ++-- cmd/nginx-loadbalancer-kubernetes/main.go | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index f434c43c..e69f5389 100644 --- a/.gitignore +++ b/.gitignore @@ -88,4 +88,4 @@ results .go-build -nginxaas-operator-* +nginxaas-loadbalancer-kubernetes-* diff --git a/Dockerfile b/Dockerfile index d80281c1..e53aef48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,6 @@ COPY docker-user /etc/passwd USER 101 COPY --from=base-certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -FROM base as nginxaas-operator -ENTRYPOINT ["/nginxaas-operator"] -COPY build/nginxaas-operator / +FROM base as nginxaas-loadbalancer-kubernetes +ENTRYPOINT ["/nginxaas-loadbalancer-kubernetes"] +COPY build/nginxaas-loadbalancer-kubernetes / diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 64f735aa..5c8d97c8 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.1.0 description: NGINXaaS LoadBalancer for Kubernetes -name: nginxaas-operator +name: nginxaas-loadbalancer-kubernetes keywords: - nginx - nginxaas diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index bc12f5cb..826e251f 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -53,7 +53,7 @@ spec: key: nginxaasApiKey volumeMounts: - name: config - mountPath: /etc/nginxaas-operator + mountPath: /etc/nginxaas-loadbalancer-kubernetes serviceAccountName: {{ include "nlk.fullname" . }} volumes: - name: config diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 19713f51..3004231e 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -1,5 +1,5 @@ nlk: - name: nginxaas-operator + name: nginxaas-loadbalancer-kubernetes kind: deployment @@ -7,7 +7,7 @@ nlk: image: registry: registry-1.docker.io - repository: nginx/nginxaas-operator + repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: release-0.1.0 diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index f8473c38..a9f6cd93 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -35,7 +35,7 @@ func run() error { return fmt.Errorf(`error building a Kubernetes client: %w`, err) } - settings, err := configuration.Read("config.yaml", "/etc/nginxaas-operator") + settings, err := configuration.Read("config.yaml", "/etc/nginxaas-loadbalancer-kubernetes") if err != nil { return fmt.Errorf(`error occurred accessing configuration: %w`, err) } From f0ca869833b31a4d765b41a0bf8240cf425dd2f0 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 2 Oct 2024 13:32:30 -0700 Subject: [PATCH 039/110] Fix up helm and docker versions This just brings them up to the latest. --- charts/nlk/Chart.yaml | 4 ++-- charts/nlk/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 5c8d97c8..f7f4c9c0 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,6 +1,6 @@ --- apiVersion: v2 -appVersion: 0.1.0 +appVersion: 0.2.1 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: @@ -14,4 +14,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.1.0 +version: 0.2.1 diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 3004231e..14a2b4f0 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -10,7 +10,7 @@ nlk: repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: release-0.1.0 + tag: 0.2.1 imagePullSecrets: [] nameOverride: "" From d90c6a8fdd9089a39f02971bd8a378a94f00ac1a Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 18:08:22 -0700 Subject: [PATCH 040/110] Fixup nlk configmap name We should really be templating the configmap name rather than hardcoding it (similar to what we did for secrets). --- charts/nlk/templates/_helpers.tpl | 6 +----- charts/nlk/templates/nlk-configmap.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- version | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 119c1640..3f62fe94 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -80,11 +80,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} Expand the name of the configmap. */}} {{- define "nlk.configName" -}} -{{- if .Values.nlk.customConfigMap -}} -{{ .Values.nlk.customConfigMap }} -{{- else -}} -{{- default (include "nlk.fullname" .) .Values.nlk.config.name -}} -{{- end -}} +{{- printf "%s-nlk-config" (include "nlk.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} {{/* diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 8cd4d664..78d41404 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: nlk-config + name: {{ include "nlk.configName" . }} namespace: {{ .Release.Namespace }} data: config.yaml: | diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 826e251f..e49006fb 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -58,4 +58,4 @@ spec: volumes: - name: config configMap: - name: nlk-config + name: {{ include "nlk.configName" . }} diff --git a/version b/version index 0c62199f..ee1372d3 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.2.1 +0.2.2 From f4a9d1610f6ad7ad50e583d12de00c30cc22a7f1 Mon Sep 17 00:00:00 2001 From: Nathan Bird Date: Tue, 1 Oct 2024 17:15:23 -0400 Subject: [PATCH 041/110] Remove configmap and secret permissions This is not a permission we need right now. If we merge back to NLK upstream we might want to _optionally_ restore this if the user needs to read certificate information. --- charts/nlk/Chart.yaml | 4 ++-- charts/nlk/templates/clusterrole.yaml | 2 -- charts/nlk/values.yaml | 2 +- version | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index f7f4c9c0..a7e619ae 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,6 +1,6 @@ --- apiVersion: v2 -appVersion: 0.2.1 +appVersion: 0.3.0 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: @@ -14,4 +14,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.2.1 +version: 0.3.0 diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index 4164475e..c652c4c9 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -7,9 +7,7 @@ rules: - apiGroups: - "" resources: - - configmaps - nodes - - secrets - services verbs: - get diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 14a2b4f0..5fc1c0d5 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -10,7 +10,7 @@ nlk: repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: 0.2.1 + tag: 0.3.0 imagePullSecrets: [] nameOverride: "" diff --git a/version b/version index ee1372d3..0d91a54c 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.2.2 +0.3.0 From 721053b0157ad12756467983862114ae1e342636 Mon Sep 17 00:00:00 2001 From: Nathan Bird Date: Tue, 1 Oct 2024 17:18:34 -0400 Subject: [PATCH 042/110] Put config values in config key - group all the config vars together - make naming more consistent - update doc strings -- use ## to describe, # to show the key:value - remove references to non-supported ca certificate information - add nlk.config.serviceAnnotationMatch Documentation example ## trace,debug,info,warn,error,fatal,panic # logLevel: "warn" --- charts/nlk/Chart.yaml | 4 ++-- charts/nlk/templates/nlk-configmap.yaml | 15 +++++++----- charts/nlk/values.yaml | 32 +++++++++++++------------ version | 2 +- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index a7e619ae..4e1860f0 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,6 +1,6 @@ --- apiVersion: v2 -appVersion: 0.3.0 +appVersion: 0.4.0 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: @@ -14,4 +14,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.3.0 +version: 0.4.0 diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 78d41404..38475a9e 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -5,10 +5,13 @@ metadata: namespace: {{ .Release.Namespace }} data: config.yaml: | -{{- if .Values.nlk.config.entries.hosts }} - nginx-hosts: "{{ .Values.nlk.config.entries.hosts }}" +{{- with .Values.nlk.config.logLevel }} + log-level: "{{ . }}" +{{- end }} +{{- with .Values.nlk.config.nginxHosts }} + nginx-hosts: "{{ . }}" +{{- end }} + tls-mode: "{{ .Values.nlk.config.tls.mode }}" +{{- with .Values.nlk.config.serviceAnnotationMatch }} + service-annotation-match: "{{ . }}" {{- end }} - tls-mode: "{{ index .Values.nlk.defaultTLS "tls-mode" }}" - ca-certificate: "{{ index .Values.nlk.defaultTLS "ca-certificate" }}" - client-certificate: "{{ index .Values.nlk.defaultTLS "client-certificate" }}" - log-level: "{{ .Values.nlk.logLevel }}" diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 5fc1c0d5..75f6b541 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -9,35 +9,37 @@ nlk: registry: registry-1.docker.io repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: 0.3.0 + ## Overrides the image tag whose default is the chart appVersion. + tag: 0.4.0 imagePullSecrets: [] nameOverride: "" fullnameOverride: "" serviceAccount: - # Specifies whether a service account should be created + ## Specifies whether a service account should be created create: true - # Automatically mount a ServiceAccount's API credentials? + ## Automatically mount a ServiceAccount's API credentials? automount: true - # Annotations to add to the service account + ## Annotations to add to the service account annotations: {} config: - entries: - hosts: - "" + ## trace,debug,info,warn,error,fatal,panic + # logLevel: "warn" - defaultTLS: - tls-mode: "no-tls" - ca-certificate: "" - client-certificate: "" + ## the nginx hosts (comma-separated) to send upstream updates to + nginxHosts: "" - # Override with your own NGINXaaS dataplane API Key. - dataplaneApiKey: "test" + ## Sets the annotation value that NLK is looking for to watch a Service + # serviceAnnotationMatch: nginxaas + + tls: + ## can also be set to "no-tls" to disable server cert verification + mode: "ca-tls" - logLevel: "warn" + ## Override with your own NGINXaaS dataplane API Key. + dataplaneApiKey: "test" containerPort: http: 51031 diff --git a/version b/version index 0d91a54c..1d0ba9ea 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.3.0 +0.4.0 From d4a4eb0ffbedf1e9b329444473486369118fa3b8 Mon Sep 17 00:00:00 2001 From: sarna Date: Sun, 6 Oct 2024 22:57:51 -0700 Subject: [PATCH 043/110] Remove unused YAMLs We have officially transitions towards using Helm to install NLK. With that, we should also look at auto-generating these YAMLs via helm and not hardcode them into the repository. Removing them as they also add additional grep confusion while scanning through the repo. --- deployments/checks/sample-ingress-mod.yaml | 18 ---------- deployments/checks/sample-ingress.yaml | 18 ---------- deployments/deployment/configmap.yaml | 11 ------- deployments/deployment/deployment.yaml | 38 ---------------------- deployments/deployment/namespace.yaml | 6 ---- deployments/rbac/apply.sh | 12 ------- deployments/rbac/clusterrole.yaml | 10 ------ deployments/rbac/clusterrolebinding.yaml | 13 -------- deployments/rbac/secret.yaml | 8 ----- deployments/rbac/serviceaccount.yaml | 5 --- deployments/rbac/unapply.sh | 8 ----- 11 files changed, 147 deletions(-) delete mode 100644 deployments/checks/sample-ingress-mod.yaml delete mode 100644 deployments/checks/sample-ingress.yaml delete mode 100644 deployments/deployment/configmap.yaml delete mode 100644 deployments/deployment/deployment.yaml delete mode 100644 deployments/deployment/namespace.yaml delete mode 100755 deployments/rbac/apply.sh delete mode 100644 deployments/rbac/clusterrole.yaml delete mode 100644 deployments/rbac/clusterrolebinding.yaml delete mode 100644 deployments/rbac/secret.yaml delete mode 100644 deployments/rbac/serviceaccount.yaml delete mode 100755 deployments/rbac/unapply.sh diff --git a/deployments/checks/sample-ingress-mod.yaml b/deployments/checks/sample-ingress-mod.yaml deleted file mode 100644 index cd793055..00000000 --- a/deployments/checks/sample-ingress-mod.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: example-ingress - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - rules: - - host: hello-world.net - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: web - port: - number: 8080 \ No newline at end of file diff --git a/deployments/checks/sample-ingress.yaml b/deployments/checks/sample-ingress.yaml deleted file mode 100644 index 7ff7fc5f..00000000 --- a/deployments/checks/sample-ingress.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: example-ingress - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - rules: - - host: hello-world.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: web - port: - number: 8080 \ No newline at end of file diff --git a/deployments/deployment/configmap.yaml b/deployments/deployment/configmap.yaml deleted file mode 100644 index fd30dbe4..00000000 --- a/deployments/deployment/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - nginx-hosts: "https://10.0.0.1:9000/api" - tls-mode: "no-tls" - ca-certificate: "" - client-certificate: "" - log-level: "warn" -metadata: - name: nlk-config - namespace: nlk diff --git a/deployments/deployment/deployment.yaml b/deployments/deployment/deployment.yaml deleted file mode 100644 index 4c871c21..00000000 --- a/deployments/deployment/deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nlk-deployment - namespace: nlk - labels: - app: nlk -spec: - replicas: 1 - selector: - matchLabels: - app: nlk - template: - metadata: - labels: - app: nlk - spec: - containers: - - name: nginx-loadbalancer-kubernetes - image: ghcr.io/nginxinc/nginx-loadbalancer-kubernetes:latest - imagePullPolicy: Always - ports: - - name: http - containerPort: 51031 - protocol: TCP - livenessProbe: - httpGet: - path: /livez - port: 51031 - initialDelaySeconds: 5 - periodSeconds: 2 - readinessProbe: - httpGet: - path: /readyz - port: 51031 - initialDelaySeconds: 5 - periodSeconds: 2 - serviceAccountName: nginx-loadbalancer-kubernetes diff --git a/deployments/deployment/namespace.yaml b/deployments/deployment/namespace.yaml deleted file mode 100644 index 8f9e3822..00000000 --- a/deployments/deployment/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: nlk - labels: - name: nlk diff --git a/deployments/rbac/apply.sh b/deployments/rbac/apply.sh deleted file mode 100755 index 58248da1..00000000 --- a/deployments/rbac/apply.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -pushd "$(dirname "$0")" - -echo "Applying all RBAC resources..." - -kubectl apply -f serviceaccount.yaml -kubectl apply -f clusterrole.yaml -kubectl apply -f clusterrolebinding.yaml -kubectl apply -f secret.yaml - -popd diff --git a/deployments/rbac/clusterrole.yaml b/deployments/rbac/clusterrole.yaml deleted file mode 100644 index c50bed85..00000000 --- a/deployments/rbac/clusterrole.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: resource-get-watch-list - namespace: nlk -rules: - - apiGroups: - - "" - resources: ["services", "nodes", "configmaps", "secrets"] - verbs: ["get", "watch", "list"] diff --git a/deployments/rbac/clusterrolebinding.yaml b/deployments/rbac/clusterrolebinding.yaml deleted file mode 100644 index d48ffb84..00000000 --- a/deployments/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: "nginx-loadbalancer-kubernetes:resource-get-watch-list" - namespace: nlk -subjects: - - kind: ServiceAccount - name: nginx-loadbalancer-kubernetes - namespace: nlk -roleRef: - kind: ClusterRole - name: resource-get-watch-list - apiGroup: rbac.authorization.k8s.io diff --git a/deployments/rbac/secret.yaml b/deployments/rbac/secret.yaml deleted file mode 100644 index 71576bfb..00000000 --- a/deployments/rbac/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: nginx-loadbalancer-kubernetes-secret - namespace: nlk - annotations: - kubernetes.io/service-account.name: nginx-loadbalancer-kubernetes -type: kubernetes.io/service-account-token diff --git a/deployments/rbac/serviceaccount.yaml b/deployments/rbac/serviceaccount.yaml deleted file mode 100644 index 76f238c0..00000000 --- a/deployments/rbac/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-loadbalancer-kubernetes - namespace: nlk diff --git a/deployments/rbac/unapply.sh b/deployments/rbac/unapply.sh deleted file mode 100755 index f29f90df..00000000 --- a/deployments/rbac/unapply.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -echo "Unapplying all RBAC resources..." - -kubectl delete -f serviceaccount.yaml -kubectl delete -f clusterrole.yaml -kubectl delete -f clusterrolebinding.yaml -kubectl delete -f secret.yaml From ce7c643bf14d2dd51e99697630b435621eae88d1 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 8 Oct 2024 15:08:01 -0600 Subject: [PATCH 044/110] NLB-5679 Added checksum field to helm nlk-deployment to force restart on configmap change --- charts/nlk/templates/nlk-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index e49006fb..62708221 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -14,6 +14,8 @@ spec: metadata: labels: app: nlk + annotations: + checksum: {{ tpl (toYaml .Values.nlk) . | sha256sum }} spec: {{- with .Values.nlk.imagePullSecrets }} imagePullSecrets: From d9979b98eef8a480893b784b5ddfacae5e10d353 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 9 Oct 2024 10:34:39 -0600 Subject: [PATCH 045/110] NLB-5679 bumped to v0.4.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 1d0ba9ea..267577d4 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.4.0 +0.4.1 From 6b3a39a0dd2763aa97284e09a1aff5f2fcd577fc Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 7 Oct 2024 17:43:56 -0700 Subject: [PATCH 046/110] NLB-5717: Add manifest for marketplace app This is the manifest describing the app on Azure marketplace. --- charts/manifest.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 charts/manifest.yaml diff --git a/charts/manifest.yaml b/charts/manifest.yaml new file mode 100644 index 00000000..9138b819 --- /dev/null +++ b/charts/manifest.yaml @@ -0,0 +1,10 @@ +applicationName: "marketplace/nginxaas-loadbalancer-kubernetes" +publisher: "F5, Inc." +description: "A component that manages NGINXaaS for Azure deployment and makes it act as Load Balancer for kubernetes workloads." +version: 0.4.0 +helmChart: "./nlk" +clusterArmTemplate: "./armTemplate.json" +uiDefinition: "./createUIDefinition.json" +registryServer: "nlbmarketplaceacrprod.azurecr.io" +extensionRegistrationParameters: + defaultScope: "namespace" From e346a5be2d5fe60b2d3c651846bb28b4af53a5e6 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 17:46:43 -0700 Subject: [PATCH 047/110] NLB-5707: Create UI definition for AKS marketplace This file describes what the UX will look like for the NLK extension on the AKS marketplace. --- charts/createUIDefinition.json | 254 +++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 charts/createUIDefinition.json diff --git a/charts/createUIDefinition.json b/charts/createUIDefinition.json new file mode 100644 index 00000000..906e08e9 --- /dev/null +++ b/charts/createUIDefinition.json @@ -0,0 +1,254 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#", + "handler": "Microsoft.Azure.CreateUIDef", + "version": "0.1.2-preview", + "parameters": { + "config": { + "isWizard": false, + "basics": { + "location": { + "visible": "[basics('createNewCluster')]", + "resourceTypes": ["Nginx.NginxPlus/nginxDeployments"] + }, + "resourceGroup": { + "allowExisting": true + } + } + }, + "basics": [ + { + "name": "createNewCluster", + "type": "Microsoft.Common.OptionsGroup", + "label": "Create new AKS cluster", + "defaultValue": "No", + "toolTip": "Create a new AKS cluster to install the extension.", + "constraints": { + "allowedValues": [ + { + "label": "Yes", + "value": true + }, + { + "label": "No", + "value": false + } + ], + "required": true + }, + "visible": true + } + ], + "steps": [ + { + "name": "clusterDetails", + "label": "Cluster Details", + "elements": [ + { + "name": "existingClusterSection", + "type": "Microsoft.Common.Section", + "elements": [ + { + "name": "clusterLookupControl", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id, '/resourcegroups/', resourceGroup().name, '/providers/Microsoft.ContainerService/managedClusters?api-version=2022-03-01')]" + } + }, + { + "name": "existingClusterResourceName", + "type": "Microsoft.Common.DropDown", + "label": "AKS Cluster Name", + "toolTip": "The resource name of the existing AKS cluster.", + "constraints": { + "allowedValues": "[map(steps('clusterDetails').existingClusterSection.clusterLookupControl.value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.name, '\"}')))]", + "required": true + } + } + ], + "visible": "[equals(basics('createNewCluster'), false)]" + }, + { + "name": "newClusterSection", + "type": "Microsoft.Common.Section", + "elements": [ + { + "name": "aksVersionLookupControl", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id, '/providers/Microsoft.ContainerService/locations/', location(), '/orchestrators?api-version=2019-04-01&resource-type=managedClusters')]" + } + }, + { + "name": "newClusterResourceName", + "type": "Microsoft.Common.TextBox", + "label": "AKS cluster name", + "defaultValue": "", + "toolTip": "The resource name of the new AKS cluster. Use only allowed characters", + "constraints": { + "required": true, + "regex": "^[a-z0-9A-Z]{6,30}$", + "validationMessage": "Only alphanumeric characters are allowed, and the value must be 6-30 characters long." + } + }, + { + "name": "kubernetesVersion", + "type": "Microsoft.Common.DropDown", + "label": "Kubernetes version", + "toolTip": "The version of Kubernetes that should be used for this cluster. You will be able to upgrade this version after creating the cluster.", + "constraints": { + "allowedValues": "[map(steps('clusterDetails').newClusterSection.aksVersionLookupControl.properties.orchestrators, (item) => parse(concat('{\"label\":\"', item.orchestratorVersion, '\",\"value\":\"', item.orchestratorVersion, '\"}')))]", + "required": true + } + }, + { + "name": "vmSize", + "type": "Microsoft.Compute.SizeSelector", + "label": "VM size", + "toolTip": "The size of virtual machine of AKS worker nodes.", + "recommendedSizes": [ + "Standard_B4ms", + "Standard_DS2_v2", + "Standard_D4s_v3" + ], + "constraints": { + "allowedSizes": [ + "Standard_B4ms", + "Standard_DS2_v2", + "Standard_D4s_v3" + ], + "excludedSizes": [] + }, + "osPlatform": "Linux" + }, + { + "name": "osSKU", + "type": "Microsoft.Common.DropDown", + "label": "OS SKU", + "toolTip": "The SKU of Linux OS for VM.", + "defaultValue": "Ubuntu", + "constraints": { + "allowedValues": [ + { + "label": "Ubuntu", + "value": "Ubuntu" + }, + { + "label": "AzureLinux", + "value": "AzureLinux" + } + ], + "required": true + } + }, + { + "name": "enableAutoScaling", + "type": "Microsoft.Common.CheckBox", + "label": "Enable auto scaling", + "toolTip": "Enable auto scaling", + "defaultValue": true + }, + { + "name": "vmCount", + "type": "Microsoft.Common.Slider", + "min": 1, + "max": 10, + "label": "Number of AKS worker nodes", + "subLabel": "", + "defaultValue": 1, + "showStepMarkers": false, + "toolTip": "Specify the number of AKS worker nodes.", + "constraints": { + "required": false + }, + "visible": true + } + ], + "visible": "[basics('createNewCluster')]" + } + ] + }, + { + "name": "applicationDetails", + "label": "Application Details", + "elements": [ + { + "name": "extensionResourceName", + "type": "Microsoft.Common.TextBox", + "label": "Cluster extension resource name", + "defaultValue": "", + "toolTip": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long.", + "constraints": { + "required": true, + "regex": "^[a-z0-9]{6,30}$", + "validationMessage": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long." + }, + "visible": true + }, + { + "name": "extensionNamespace", + "type": "Microsoft.Common.TextBox", + "label": "Installation namespace", + "defaultValue": "nlk", + "toolTip": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long.", + "constraints": { + "required": true, + "regex": "^[a-z0-9]{3,30}$", + "validationMessage": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long." + }, + "visible": true + }, + { + "name": "extensionAutoUpgrade", + "type": "Microsoft.Common.CheckBox", + "label": "Allow minor version upgrades of extension", + "toolTip": "Allow exntension to be upgraded automatically to latest minor version.", + "visible": true + }, + { + "name": "nginxaasDataplaneApiKey", + "type": "Microsoft.Common.TextBox", + "label": "NGINXaaS Dataplane API Key", + "defaultValue": "", + "toolTip": "The Dataplane API Key for your NGINXaaS for Azure deployment.", + "visible": true + }, + { + "name": "nginxaasDataplaneApiEndpoint", + "type": "Microsoft.Common.TextBox", + "label": "NGINXaaS Dataplane API Endpoint", + "defaultValue": "", + "toolTip": "The Dataplane API Endpoint for your NGINXaaS for Azure deployment.", + "visible": true + }, + { + "name": "additionalProductInfo", + "type": "Microsoft.Common.InfoBox", + "visible": true, + "options": { + "icon": "Info", + "text": "Learn more about NGINXaaS for Azure.", + "uri": "https://docs.nginx.com/nginxaas/azure/" + } + } + ] + } + ], + "outputs": { + "location": "[location()]", + "createNewCluster": "[basics('createNewCluster')]", + "clusterResourceName": "[if(basics('createNewCluster'), steps('clusterDetails').newClusterSection.newClusterResourceName, steps('clusterDetails').existingClusterSection.existingClusterResourceName)]", + "kubernetesVersion": "[steps('clusterDetails').newClusterSection.kubernetesVersion]", + "vmSize": "[steps('clusterDetails').newClusterSection.vmSize]", + "osSKU": "[steps('clusterDetails').newClusterSection.osSKU]", + "vmEnableAutoScale": "[steps('clusterDetails').newClusterSection.enableAutoScaling]", + "vmCount": "[steps('clusterDetails').newClusterSection.vmCount]", + "extensionResourceName": "[steps('applicationDetails').extensionResourceName]", + "extensionAutoUpgrade": "[steps('applicationDetails').extensionAutoUpgrade]", + "extensionNamespace": "[steps('applicationDetails').extensionNamespace]", + "nginxaasDataplaneApiKey": "[steps('applicationDetails').nginxaasDataplaneApiKey]", + "nginxaasDataplaneApiEndpoint": "[steps('applicationDetails').nginxaasDataplaneApiEndpoint]" + } + } +} From 8808ef893d2f23d05663e9b3ad0fddca94cb5c52 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 14:52:16 -0700 Subject: [PATCH 048/110] NLB-5733: Update Helm chart for Azure Marketplace Azure requires `global.azure.images` field in the Chart values so it can parse the Helm chart and copy them into an Azure-managed/owned ACR. This commit allows for non-Azure Marketplace users to continue to use the helm chart regularly and we will update the Chart values dynamically when we publish to markerplace to allow for newer workflows. --- charts/nlk/templates/_helpers.tpl | 8 +++++++- charts/nlk/templates/nlk-deployment.yaml | 3 +++ charts/nlk/values.yaml | 24 +++++++++++------------- version | 2 +- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 3f62fe94..6f5677dd 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -91,14 +91,20 @@ Expand service account name. {{- end -}} {{- define "nlk.tag" -}} +{{- if .Values.global.azure -}} +{{- printf "%s" .Values.global.azure.images.nlk.tag -}} +{{- else -}} {{- default .Chart.AppVersion .Values.nlk.image.tag -}} {{- end -}} +{{- end -}} {{/* Expand image name. */}} {{- define "nlk.image" -}} -{{- if .Values.nlk.image.digest -}} +{{- if .Values.global.azure -}} +{{- printf "%s/%s:%s" .Values.global.azure.images.nlk.registry .Values.global.azure.images.nlk.repository (include "nlk.tag" .) -}} +{{- else if .Values.nlk.image.digest -}} {{- printf "%s/%s@%s" .Values.nlk.image.registry .Values.nlk.image.repository .Values.nlk.image.digest -}} {{- else -}} {{- printf "%s/%s:%s" .Values.nlk.image.registry .Values.nlk.image.repository (include "nlk.tag" .) -}} diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 62708221..93fcd383 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -14,6 +14,9 @@ spec: metadata: labels: app: nlk +{{- if .Values.global.azure }} + azure-extensions-usage-release-identifier: {{ .Release.Name }} +{{- end }} annotations: checksum: {{ tpl (toYaml .Values.nlk) . | sha256sum }} spec: diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 75f6b541..4ce75ae6 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -1,21 +1,27 @@ +##################################### +# Global Azure Marketplace configuration for AKS integration. +# DO NOT REMOVE +global: + azure: + # images: + # nlk: + # registry: registry-1.docker.io + # repository: nginx/nginxaas-loadbalancer-kubernetes + # tag: 0.4.0 +##################################### nlk: name: nginxaas-loadbalancer-kubernetes - kind: deployment - replicaCount: 1 - image: registry: registry-1.docker.io repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always ## Overrides the image tag whose default is the chart appVersion. tag: 0.4.0 - imagePullSecrets: [] nameOverride: "" fullnameOverride: "" - serviceAccount: ## Specifies whether a service account should be created create: true @@ -23,39 +29,31 @@ nlk: automount: true ## Annotations to add to the service account annotations: {} - config: ## trace,debug,info,warn,error,fatal,panic # logLevel: "warn" ## the nginx hosts (comma-separated) to send upstream updates to nginxHosts: "" - ## Sets the annotation value that NLK is looking for to watch a Service # serviceAnnotationMatch: nginxaas - tls: ## can also be set to "no-tls" to disable server cert verification mode: "ca-tls" - ## Override with your own NGINXaaS dataplane API Key. dataplaneApiKey: "test" - containerPort: http: 51031 - liveStatus: enable: true port: 51031 initialDelaySeconds: 5 periodSeconds: 2 - readyStatus: enable: true port: 51031 initialDelaySeconds: 5 periodSeconds: 2 - rbac: ## Configures RBAC. create: true diff --git a/version b/version index 267577d4..8f0916f7 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.4.1 +0.5.0 From 78011522f2af622bcda2a433a55f1e47ec2a7649 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 14:52:16 -0700 Subject: [PATCH 049/110] NLB-5733: Update Helm chart for Azure Marketplace Azure requires `global.azure.images` field in the Chart values so it can parse the Helm chart and copy them into an Azure-managed/owned ACR. This commit allows for non-Azure Marketplace users to continue to use the helm chart regularly and we will update the Chart values dynamically when we publish to markerplace to allow for newer workflows. --- charts/nlk/templates/_helpers.tpl | 2 +- charts/nlk/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 6f5677dd..27dbe94a 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -103,7 +103,7 @@ Expand image name. */}} {{- define "nlk.image" -}} {{- if .Values.global.azure -}} -{{- printf "%s/%s:%s" .Values.global.azure.images.nlk.registry .Values.global.azure.images.nlk.repository (include "nlk.tag" .) -}} +{{- printf "%s/%s:%s" .Values.global.azure.images.nlk.registry .Values.global.azure.images.nlk.image (include "nlk.tag" .) -}} {{- else if .Values.nlk.image.digest -}} {{- printf "%s/%s@%s" .Values.nlk.image.registry .Values.nlk.image.repository .Values.nlk.image.digest -}} {{- else -}} diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 4ce75ae6..f5be2444 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -6,7 +6,7 @@ global: # images: # nlk: # registry: registry-1.docker.io - # repository: nginx/nginxaas-loadbalancer-kubernetes + # image: nginx/nginxaas-loadbalancer-kubernetes # tag: 0.4.0 ##################################### nlk: From ab44d3db3d8645fc9b97bafc148e565c7f006be5 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 17:47:50 -0700 Subject: [PATCH 050/110] NLB-5708: Create ARM template for AKS extension This defines the resources that will get created as part of the extension deployment. For the time being, Azure only allows creating AKS and extensions. --- charts/armTemplate.json | 256 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 charts/armTemplate.json diff --git a/charts/armTemplate.json b/charts/armTemplate.json new file mode 100644 index 00000000..1353fdb2 --- /dev/null +++ b/charts/armTemplate.json @@ -0,0 +1,256 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "extensionResourceName": { + "type": "string", + "metadata": { + "description": "The name of the extension." + } + }, + "extensionNamespace": { + "type": "string", + "defaultValue": "nlk" + }, + "clusterResourceName": { + "type": "String", + "metadata": { + "description": "The name of the Managed Cluster resource." + } + }, + "createNewCluster": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "When set to 'true', creates new AKS cluster. Otherwise, an existing cluster is used." + } + }, + "location": { + "type": "String", + "metadata": { + "description": "The location of AKS resource." + } + }, + "extensionAutoUpgrade": { + "defaultValue": false, + "metadata": { + "description": "Allow auto upgrade of minor version for the extension." + }, + "type": "bool" + }, + "nginxaasDataplaneApiKey": { + "type": "String" + }, + "nginxaasDataplaneApiEndpoint": { + "type": "String" + }, + "vmSize": { + "type": "String", + "defaultValue": "Standard_DS2_v2", + "metadata": { + "description": "VM size" + } + }, + "vmEnableAutoScale": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "enable auto scaling" + } + }, + "vmCount": { + "type": "Int", + "defaultValue": 3, + "metadata": { + "description": "VM count" + } + }, + "dnsPrefix": { + "defaultValue": "[concat(parameters('clusterResourceName'),'-dns')]", + "type": "String", + "metadata": { + "description": "Optional DNS prefix to use with hosted Kubernetes API server FQDN." + } + }, + "osDiskSizeGB": { + "defaultValue": 0, + "minValue": 0, + "maxValue": 1023, + "type": "Int", + "metadata": { + "description": "Disk size (in GiB) to provision for each of the agent pool nodes. This value ranges from 0 to 1023. Specifying 0 will apply the default disk size for that agentVMSize." + } + }, + "kubernetesVersion": { + "type": "String", + "defaultValue": "1.26.3", + "metadata": { + "description": "The version of Kubernetes." + } + }, + "networkPlugin": { + "defaultValue": "kubenet", + "allowedValues": [ + "azure", + "kubenet" + ], + "type": "String", + "metadata": { + "description": "Network plugin used for building Kubernetes network." + } + }, + "enableRBAC": { + "defaultValue": true, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off of RBAC." + } + }, + "enablePrivateCluster": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Enable private network access to the Kubernetes cluster." + } + }, + "enableHttpApplicationRouting": { + "defaultValue": true, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off http application routing." + } + }, + "enableAzurePolicy": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off Azure Policy addon." + } + }, + "enableSecretStoreCSIDriver": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off secret store CSI driver." + } + }, + "osSKU": { + "type": "string", + "defaultValue": "AzureLinux", + "allowedValues": [ + "AzureLinux", + "Ubuntu" + ], + "metadata": { + "description": "The Linux SKU to use." + } + }, + "enableFIPS": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "Enable FIPS. https://learn.microsoft.com/en-us/azure/aks/create-node-pools#fips-enabled-node-pools" + } + } + }, + "variables": { + "plan-name": "DONOTMODIFY", + "plan-publisher": "DONOTMODIFY", + "plan-offerID": "DONOTMODIFY", + "releaseTrain": "DONOTMODIFY", + "clusterExtensionTypeName": "DONOTMODIFY" + }, + "resources": [ + { + "type": "Microsoft.ContainerService/managedClusters", + "condition": "[parameters('createNewCluster')]", + "apiVersion": "2022-11-01", + "name": "[parameters('clusterResourceName')]", + "location": "[parameters('location')]", + "dependsOn": [], + "tags": {}, + "sku": { + "name": "Basic", + "tier": "Free" + }, + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "kubernetesVersion": "[parameters('kubernetesVersion')]", + "enableRBAC": "[parameters('enableRBAC')]", + "dnsPrefix": "[parameters('dnsPrefix')]", + "agentPoolProfiles": [ + { + "name": "agentpool", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "count": "[parameters('vmCount')]", + "enableAutoScaling": "[parameters('vmEnableAutoScale')]", + "enableFIPS": "[parameters('enableFIPS')]", + "minCount": "[if(parameters('vmEnableAutoScale'), 1, json('null'))]", + "maxCount": "[if(parameters('vmEnableAutoScale'), 10, json('null'))]", + "vmSize": "[parameters('vmSize')]", + "osType": "Linux", + "osSKU": "[parameters('osSKU')]", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "mode": "System", + "maxPods": 110, + "availabilityZones": [], + "enableNodePublicIP": false, + "tags": {} + } + ], + "networkProfile": { + "loadBalancerSku": "standard", + "networkPlugin": "[parameters('networkPlugin')]" + }, + "apiServerAccessProfile": { + "enablePrivateCluster": "[parameters('enablePrivateCluster')]" + }, + "addonProfiles": { + "httpApplicationRouting": { + "enabled": "[parameters('enableHttpApplicationRouting')]" + }, + "azurepolicy": { + "enabled": "[parameters('enableAzurePolicy')]" + }, + "azureKeyvaultSecretsProvider": { + "enabled": "[parameters('enableSecretStoreCSIDriver')]" + } + } + } + }, + { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2023-05-01", + "name": "[parameters('extensionResourceName')]", + "properties": { + "extensionType": "[variables('clusterExtensionTypeName')]", + "autoUpgradeMinorVersion": "[parameters('extensionAutoUpgrade')]", + "releaseTrain": "[variables('releaseTrain')]", + "configurationSettings": { + "nlk.config.dataplaneApiKey": "[parameters('nginxaasDataplaneApiKey')]", + "nlk.config.nginxHosts": "[parameters('nginxaasDataplaneApiEndpoint')]" + }, + "configurationProtectedSettings": {}, + "scope": { + "namespace": { + "targetNamespace": "[parameters('extensionNamespace')]" + } + } + }, + "plan": { + "name": "[variables('plan-name')]", + "publisher": "[variables('plan-publisher')]", + "product": "[variables('plan-offerID')]" + }, + "scope": "[concat('Microsoft.ContainerService/managedClusters/', parameters('clusterResourceName'))]", + "dependsOn": [ + "[resourceId('Microsoft.ContainerService/managedClusters/', parameters('clusterResourceName'))]" + ] + } + ], + "outputs": { + } +} From 54228b0ff0aa656ce62b3b969c516a1fa1144a54 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 15 Oct 2024 12:59:55 -0700 Subject: [PATCH 051/110] Add constraints to dataplane-related user inputs contraints are required on textboxes. --- charts/createUIDefinition.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/charts/createUIDefinition.json b/charts/createUIDefinition.json index 906e08e9..c7566d24 100644 --- a/charts/createUIDefinition.json +++ b/charts/createUIDefinition.json @@ -212,6 +212,11 @@ "label": "NGINXaaS Dataplane API Key", "defaultValue": "", "toolTip": "The Dataplane API Key for your NGINXaaS for Azure deployment.", + "constraints": { + "required": false, + "regex": ".*", + "validationMessage": "Use the dataplane API key for your deployment." + }, "visible": true }, { @@ -220,6 +225,11 @@ "label": "NGINXaaS Dataplane API Endpoint", "defaultValue": "", "toolTip": "The Dataplane API Endpoint for your NGINXaaS for Azure deployment.", + "constraints": { + "required": false, + "regex": ".*", + "validationMessage": "Retreive the dataplane API endpoint from your deployment." + }, "visible": true }, { From ba514362a87f0d722c7da38f4b5eb430889eee9f Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 17 Oct 2024 12:13:49 -0700 Subject: [PATCH 052/110] Change extension to ClusterScope When trying to install the extension in a namespaced mode (which is the default on the current offer/plan), the AKS extension machinery creates a k8s service account (different from the NLK service account) which is restricted to the extension namespace. Due to the aforementioned, the service account cannot read Cluster Roles that we create for NLK causing the extension to fail. Switching over to the cluster level extension will help here as when the AKS extension gets installed cluster-wide, the undelying service account has all the necessary permissions for the extension to be installed. --- charts/manifest.yaml | 3 ++- version | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/manifest.yaml b/charts/manifest.yaml index 9138b819..6883ea96 100644 --- a/charts/manifest.yaml +++ b/charts/manifest.yaml @@ -7,4 +7,5 @@ clusterArmTemplate: "./armTemplate.json" uiDefinition: "./createUIDefinition.json" registryServer: "nlbmarketplaceacrprod.azurecr.io" extensionRegistrationParameters: - defaultScope: "namespace" + defaultScope: "cluster" + namespace: "nlk" diff --git a/version b/version index 8f0916f7..4b9fcbec 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.0 +0.5.1 From 4ab060fe30f96c7293969a763087844f4fc5197d Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 21 Oct 2024 10:14:45 -0700 Subject: [PATCH 053/110] Use release namespace everywhere This was missed as part of the manifest rollout. --- charts/armTemplate.json | 4 ++-- version | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/armTemplate.json b/charts/armTemplate.json index 1353fdb2..ae3b699f 100644 --- a/charts/armTemplate.json +++ b/charts/armTemplate.json @@ -235,8 +235,8 @@ }, "configurationProtectedSettings": {}, "scope": { - "namespace": { - "targetNamespace": "[parameters('extensionNamespace')]" + "cluster": { + "releaseNamespace": "[parameters('extensionNamespace')]" } } }, diff --git a/version b/version index 4b9fcbec..cb0c939a 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.1 +0.5.2 From 5b16c4cda66235e123634e098b4552f25f078b94 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 21 Oct 2024 16:31:39 -0700 Subject: [PATCH 054/110] Fixup helm value supplied in extension It is `nlk.dataplaneApiKey` and not `nlk.config.dataplaneApiKey`. --- charts/armTemplate.json | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/armTemplate.json b/charts/armTemplate.json index ae3b699f..510c7842 100644 --- a/charts/armTemplate.json +++ b/charts/armTemplate.json @@ -230,7 +230,7 @@ "autoUpgradeMinorVersion": "[parameters('extensionAutoUpgrade')]", "releaseTrain": "[variables('releaseTrain')]", "configurationSettings": { - "nlk.config.dataplaneApiKey": "[parameters('nginxaasDataplaneApiKey')]", + "nlk.dataplaneApiKey": "[parameters('nginxaasDataplaneApiKey')]", "nlk.config.nginxHosts": "[parameters('nginxaasDataplaneApiEndpoint')]" }, "configurationProtectedSettings": {}, diff --git a/version b/version index cb0c939a..be14282b 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.2 +0.5.3 From 0e351605019590c1127ad858a4613b518de8dc13 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 18 Oct 2024 15:52:36 -0700 Subject: [PATCH 055/110] Allow NLK to READ endpoint slices We will need to read endpoint slices in AKS to support the use-case for Cluster IP using Azure CNI. --- charts/nlk/templates/clusterrole.yaml | 8 ++++++++ version | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index c652c4c9..47f95dd4 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -13,4 +13,12 @@ rules: - get - list - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch {{- end }} diff --git a/version b/version index be14282b..a918a2aa 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.3 +0.6.0 From 45e5328d5df867b200cc2eeee59c7722b55817eb Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:15:03 -0600 Subject: [PATCH 056/110] NLB-5753 Added buildinfo package and injected vars at build time The code will then be able to reference build information, such as semver, easily at run time. --- pkg/buildinfo/buildinfo.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 pkg/buildinfo/buildinfo.go diff --git a/pkg/buildinfo/buildinfo.go b/pkg/buildinfo/buildinfo.go new file mode 100644 index 00000000..5d8839d1 --- /dev/null +++ b/pkg/buildinfo/buildinfo.go @@ -0,0 +1,15 @@ +package buildinfo + +var semVer string + +// SemVer is the version number of this build as provided by build pipeline +func SemVer() string { + return semVer +} + +var shortHash string + +// ShortHash is the 8 char git shorthash +func ShortHash() string { + return shortHash +} From 6111e7af1ee74cec790c1a46f937ecf2cc168801 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:16:14 -0600 Subject: [PATCH 057/110] NLB-5753 Replaced logrus with slog as the nlk logger; added version info to all log lines Go's slog now provides comparable functionality to logrus while being more up-to-date. Added version to all log lines. --- cmd/certificates-test-harness/main.go | 14 ++++--- cmd/nginx-loadbalancer-kubernetes/main.go | 46 ++++++++++----------- cmd/tls-config-factory-test-harness/main.go | 20 +++++---- go.mod | 1 - go.sum | 4 -- internal/application/border_client.go | 7 ++-- internal/application/null_border_client.go | 10 ++--- internal/authentication/factory.go | 18 ++++---- internal/certification/certificates.go | 28 ++++++------- internal/communication/factory.go | 4 +- internal/configuration/settings.go | 4 +- internal/observation/handler.go | 19 ++++----- internal/observation/watcher.go | 30 +++++++------- internal/probation/server.go | 16 ++++--- internal/probation/server_test.go | 4 +- internal/synchronization/synchronizer.go | 40 +++++++++--------- internal/translation/translator.go | 10 ++--- 17 files changed, 135 insertions(+), 140 deletions(-) diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index 056d3033..f3468a9d 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -4,10 +4,11 @@ import ( "context" "errors" "fmt" + "log/slog" + "os" "path/filepath" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -15,15 +16,18 @@ import ( ) func main() { - logrus.SetLevel(logrus.DebugLevel) + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + logger := slog.New(handler) + slog.SetDefault(logger) err := run() if err != nil { - logrus.Fatal(err) + slog.Error(err.Error()) + os.Exit(1) } } func run() error { - logrus.Info("certificates-test-harness::run") + slog.Info("certificates-test-harness::run") ctx := context.Background() var err error @@ -47,7 +51,7 @@ func run() error { } func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") + slog.Debug("Watcher::buildKubernetesClient") var kubeconfig *string var k8sConfig *rest.Config diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index a9f6cd93..80643482 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -8,12 +8,14 @@ package main import ( "context" "fmt" + "log/slog" + "os" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/observation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/probation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "github.com/sirupsen/logrus" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" @@ -22,7 +24,8 @@ import ( func main() { err := run() if err != nil { - logrus.Fatal(err) + slog.Error(err.Error()) + os.Exit(1) } } @@ -40,7 +43,7 @@ func run() error { return fmt.Errorf(`error occurred accessing configuration: %w`, err) } - setLogLevel(settings.LogLevel) + initializeLogger(settings.LogLevel) synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) @@ -78,37 +81,30 @@ func run() error { return nil } -func setLogLevel(logLevel string) { - logrus.Debugf("Settings::setLogLevel: %s", logLevel) - switch logLevel { - case "panic": - logrus.SetLevel(logrus.PanicLevel) - - case "fatal": - logrus.SetLevel(logrus.FatalLevel) +func initializeLogger(logLevel string) { + programLevel := new(slog.LevelVar) + switch logLevel { case "error": - logrus.SetLevel(logrus.ErrorLevel) - + programLevel.Set(slog.LevelError) case "warn": - logrus.SetLevel(logrus.WarnLevel) - + programLevel.Set(slog.LevelWarn) case "info": - logrus.SetLevel(logrus.InfoLevel) - + programLevel.Set(slog.LevelInfo) case "debug": - logrus.SetLevel(logrus.DebugLevel) - - case "trace": - logrus.SetLevel(logrus.TraceLevel) - + programLevel.Set(slog.LevelDebug) default: - logrus.SetLevel(logrus.WarnLevel) + programLevel.Set(slog.LevelWarn) } + + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: programLevel}) + logger := slog.New(handler).With("version", buildinfo.SemVer()) + slog.SetDefault(logger) + slog.Debug("Settings::setLogLevel", slog.String("level", logLevel)) } func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") + slog.Debug("Watcher::buildKubernetesClient") k8sConfig, err := rest.InClusterConfig() if err == rest.ErrNotInCluster { return nil, fmt.Errorf(`not running in a Cluster: %w`, err) @@ -125,7 +121,7 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { } func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { - logrus.Debug("Watcher::buildSynchronizerWorkQueue") + slog.Debug("Watcher::buildSynchronizerWorkQueue") rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 7b853e07..7883d65a 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -2,13 +2,13 @@ package main import ( "bufio" + "log/slog" "os" "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) const ( @@ -22,14 +22,16 @@ type TLSConfiguration struct { } func main() { - logrus.SetLevel(logrus.DebugLevel) + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + logger := slog.New(handler) + slog.SetDefault(logger) configurations := buildConfigMap() for name, settings := range configurations { - logrus.Infof("\033[H\033[2J") + slog.Info("\033[H\033[2J") - logrus.Infof("\n\n\t*** Building TLS config for <<< %s >>>\n\n", name) + slog.Info("\n\n\t*** Building TLS config\n\n", "name", name) tlsConfig, err := authentication.NewTLSConfig(settings.Settings) if err != nil { @@ -47,15 +49,17 @@ func main() { certificateCount = len(tlsConfig.Certificates) } - logrus.Infof("Successfully built TLS config: \n\tDescription: %s \n\tRootCA count: %v\n\tCertificate count: %v", - settings.Description, rootCaCount, certificateCount, + slog.Info("Successfully built TLS config", + "description", settings.Description, + "rootCaCount", rootCaCount, + "certificateCount", certificateCount, ) _, _ = bufio.NewReader(os.Stdin).ReadBytes('\n') } - logrus.Infof("\033[H\033[2J") - logrus.Infof("\n\n\t*** All done! ***\n\n") + slog.Info("\033[H\033[2J") + slog.Info("\n\n\t*** All done! ***\n\n") } func buildConfigMap() map[string]TLSConfiguration { diff --git a/go.mod b/go.mod index 056c7e2f..94a13195 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ toolchain go1.21.4 require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 - github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 k8s.io/api v0.26.0 diff --git a/go.sum b/go.sum index 098adea1..1433805d 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,6 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -134,7 +132,6 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -190,7 +187,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/internal/application/border_client.go b/internal/application/border_client.go index a5109532..e4eded8b 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -7,9 +7,9 @@ package application import ( "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) // Interface defines the functions required to implement a Border Client. @@ -19,8 +19,7 @@ type Interface interface { } // BorderClient defines any state need by the Border Client. -type BorderClient struct { -} +type BorderClient struct{} // NewBorderClient is the Factory function for creating a Border Client. // @@ -29,7 +28,7 @@ type BorderClient struct { // 2. Add a new constant in application_constants.go that acts as a key for selecting the client; // 3. Update the NewBorderClient factory method in border_client.go that returns the client; func NewBorderClient(clientType string, borderClient interface{}) (Interface, error) { - logrus.Debugf(`NewBorderClient for type: %s`, clientType) + slog.Debug("NewBorderClient", slog.String("client", clientType)) switch clientType { case ClientTypeNginxStream: diff --git a/internal/application/null_border_client.go b/internal/application/null_border_client.go index b59ca229..295ca62f 100644 --- a/internal/application/null_border_client.go +++ b/internal/application/null_border_client.go @@ -6,15 +6,15 @@ package application import ( + "log/slog" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) // NullBorderClient is a BorderClient that does nothing. // It serves only to prevent a panic if the BorderClient // is not set correctly and errors from the factory methods are ignored. -type NullBorderClient struct { -} +type NullBorderClient struct{} // NewNullBorderClient is the Factory function for creating a NullBorderClient func NewNullBorderClient() (Interface, error) { @@ -23,12 +23,12 @@ func NewNullBorderClient() (Interface, error) { // Update logs a Warning. It is, after all, a NullObject Pattern implementation. func (nbc *NullBorderClient) Update(_ *core.ServerUpdateEvent) error { - logrus.Warn("NullBorderClient.Update called") + slog.Warn("NullBorderClient.Update called") return nil } // Delete logs a Warning. It is, after all, a NullObject Pattern implementation. func (nbc *NullBorderClient) Delete(_ *core.ServerUpdateEvent) error { - logrus.Warn("NullBorderClient.Delete called") + slog.Warn("NullBorderClient.Delete called") return nil } diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index a51d7ab4..c0664f65 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -12,14 +12,14 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" ) func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { - logrus.Debugf("authentication::NewTLSConfig Creating TLS config for mode: '%s'", settings.TLSMode) + slog.Debug("authentication::NewTLSConfig Creating TLS config", "mode", settings.TLSMode) switch settings.TLSMode { case configuration.NoTLS: @@ -43,7 +43,7 @@ func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { } func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") + slog.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) if err != nil { return nil, err @@ -57,7 +57,7 @@ func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Co } func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildSelfSignedMtlsConfig Building self-signed mTLS config") + slog.Debug("authentication::buildSelfSignedMtlsConfig Building self-signed mTLS config") certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) if err != nil { return nil, err @@ -67,7 +67,7 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C if err != nil { return nil, err } - logrus.Debugf("buildSelfSignedMtlsConfig Certificate: %v", certificate) + slog.Debug("buildSelfSignedMtlsConfig Certificate", "certificate", certificate) //nolint:gosec return &tls.Config{ @@ -79,14 +79,14 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C } func buildBasicTLSConfig(skipVerify bool) *tls.Config { - logrus.Debugf("authentication::buildBasicTLSConfig skipVerify(%v)", skipVerify) + slog.Debug("authentication::buildBasicTLSConfig", slog.Bool("skipVerify", skipVerify)) return &tls.Config{ InsecureSkipVerify: skipVerify, //nolint:gosec } } func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildCATLSConfig") + slog.Debug("authentication::buildCATLSConfig") certificate, err := buildCertificates(certificates.GetClientCertificate()) if err != nil { return nil, err @@ -100,12 +100,12 @@ func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, er } func buildCertificates(privateKeyPEM []byte, certificatePEM []byte) (tls.Certificate, error) { - logrus.Debug("authentication::buildCertificates") + slog.Debug("authentication::buildCertificates") return tls.X509KeyPair(certificatePEM, privateKeyPEM) } func buildCaCertificatePool(caCert []byte) (*x509.CertPool, error) { - logrus.Debug("authentication::buildCaCertificatePool") + slog.Debug("authentication::buildCaCertificatePool") block, _ := pem.Decode(caCert) if block == nil { return nil, fmt.Errorf("failed to decode PEM block containing CA certificate") diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go index 3ecbc46c..c59eb0ea 100644 --- a/internal/certification/certificates.go +++ b/internal/certification/certificates.go @@ -12,8 +12,8 @@ package certification import ( "context" "fmt" + "log/slog" - "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -81,7 +81,7 @@ func (c *Certificates) GetClientCertificate() (core.SecretBytes, core.SecretByte // Initialize initializes the Certificates object. Sets up a SharedInformer for the Secrets Resource. func (c *Certificates) Initialize() error { - logrus.Info("Certificates::Initialize") + slog.Info("Certificates::Initialize") var err error @@ -101,7 +101,7 @@ func (c *Certificates) Initialize() error { // Run starts the SharedInformer. func (c *Certificates) Run() error { - logrus.Info("Certificates::Run") + slog.Info("Certificates::Run") if c.informer == nil { return fmt.Errorf(`initialize must be called before Run`) @@ -115,7 +115,7 @@ func (c *Certificates) Run() error { } func (c *Certificates) buildInformer() cache.SharedInformer { - logrus.Debug("Certificates::buildInformer") + slog.Debug("Certificates::buildInformer") options := informers.WithNamespace(SecretsNamespace) factory := informers.NewSharedInformerFactoryWithOptions(c.k8sClient, 0, options) @@ -125,7 +125,7 @@ func (c *Certificates) buildInformer() cache.SharedInformer { } func (c *Certificates) initializeEventHandlers() error { - logrus.Debug("Certificates::initializeEventHandlers") + slog.Debug("Certificates::initializeEventHandlers") var err error @@ -144,11 +144,11 @@ func (c *Certificates) initializeEventHandlers() error { } func (c *Certificates) handleAddEvent(obj interface{}) { - logrus.Debug("Certificates::handleAddEvent") + slog.Debug("Certificates::handleAddEvent") secret, ok := obj.(*corev1.Secret) if !ok { - logrus.Errorf("Certificates::handleAddEvent: unable to cast object to Secret") + slog.Error("Certificates::handleAddEvent: unable to cast object to Secret") return } @@ -162,15 +162,15 @@ func (c *Certificates) handleAddEvent(obj interface{}) { c.Certificates[secret.Name][k] = core.SecretBytes(v) } - logrus.Debugf("Certificates::handleAddEvent: certificates (%d)", len(c.Certificates)) + slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.Certificates))) } func (c *Certificates) handleDeleteEvent(obj interface{}) { - logrus.Debug("Certificates::handleDeleteEvent") + slog.Debug("Certificates::handleDeleteEvent") secret, ok := obj.(*corev1.Secret) if !ok { - logrus.Errorf("Certificates::handleDeleteEvent: unable to cast object to Secret") + slog.Error("Certificates::handleDeleteEvent: unable to cast object to Secret") return } @@ -178,15 +178,15 @@ func (c *Certificates) handleDeleteEvent(obj interface{}) { delete(c.Certificates, secret.Name) } - logrus.Debugf("Certificates::handleDeleteEvent: certificates (%d)", len(c.Certificates)) + slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.Certificates))) } func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { - logrus.Debug("Certificates::handleUpdateEvent") + slog.Debug("Certificates::handleUpdateEvent") secret, ok := newValue.(*corev1.Secret) if !ok { - logrus.Errorf("Certificates::handleUpdateEvent: unable to cast object to Secret") + slog.Error("Certificates::handleUpdateEvent: unable to cast object to Secret") return } @@ -194,5 +194,5 @@ func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { c.Certificates[secret.Name][k] = v } - logrus.Debugf("Certificates::handleUpdateEvent: certificates (%d)", len(c.Certificates)) + slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.Certificates))) } diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 2a3c09a6..8664e9d6 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -8,12 +8,12 @@ package communication import ( "crypto/tls" "fmt" + "log/slog" netHttp "net/http" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" ) // NewHTTPClient is a factory method to create a new Http Client with a default configuration. @@ -52,7 +52,7 @@ func NewHeaders(apiKey string) []string { func NewTLSConfig(settings configuration.Settings) *tls.Config { tlsConfig, err := authentication.NewTLSConfig(settings) if err != nil { - logrus.Warnf("Failed to create TLS config: %v", err) + slog.Warn("Failed to create TLS config", "error", err) return &tls.Config{InsecureSkipVerify: true} //nolint:gosec } diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 91ff27ce..ebeacc46 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -8,10 +8,10 @@ package configuration import ( "encoding/base64" "fmt" + "log/slog" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/sirupsen/logrus" "github.com/spf13/viper" ) @@ -146,7 +146,7 @@ func Read(configName, configPath string) (s Settings, err error) { tlsMode := NoTLS if t, err := validateTLSMode(v.GetString("tls-mode")); err != nil { - logrus.Errorf("could not validate tls mode: %v", err) + slog.Error("could not validate tls mode", "error", err) } else { tlsMode = t } diff --git a/internal/observation/handler.go b/internal/observation/handler.go index bd823f61..9f443597 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -7,12 +7,12 @@ package observation import ( "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" - "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ) @@ -59,13 +59,13 @@ func NewHandler( // AddRateLimitedEvent adds an event to the event queue func (h *Handler) AddRateLimitedEvent(event *core.Event) { - logrus.Debugf(`Handler::AddRateLimitedEvent: %#v`, event) + slog.Debug(`Handler::AddRateLimitedEvent`, "event", event) h.eventQueue.AddRateLimited(event) } // Run starts the event handler, spins up Goroutines to process events, and waits for a stop signal func (h *Handler) Run(stopCh <-chan struct{}) { - logrus.Debug("Handler::Run") + slog.Debug("Handler::Run") for i := 0; i < h.settings.Handler.Threads; i++ { go wait.Until(h.worker, 0, stopCh) @@ -76,13 +76,13 @@ func (h *Handler) Run(stopCh <-chan struct{}) { // ShutDown stops the event handler and shuts down the event queue func (h *Handler) ShutDown() { - logrus.Debug("Handler::ShutDown") + slog.Debug("Handler::ShutDown") h.eventQueue.ShutDown() } // handleEvent feeds translated events to the synchronizer func (h *Handler) handleEvent(e *core.Event) error { - logrus.Debugf(`Handler::handleEvent: %#v`, e) + slog.Debug("Handler::handleEvent", "event", e) // TODO: Add Telemetry events, err := translation.Translate(e) @@ -97,9 +97,8 @@ func (h *Handler) handleEvent(e *core.Event) error { // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic func (h *Handler) handleNextEvent() bool { - logrus.Debug("Handler::handleNextEvent") evt, quit := h.eventQueue.Get() - logrus.Debugf(`Handler::handleNextEvent: %#v, quit: %v`, evt, quit) + slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) if quit { return false } @@ -121,15 +120,15 @@ func (h *Handler) worker() { // withRetry handles errors from the event handler and requeues events that fail func (h *Handler) withRetry(err error, event *core.Event) { - logrus.Debug("Handler::withRetry") + slog.Debug("Handler::withRetry") if err != nil { // TODO: Add Telemetry if h.eventQueue.NumRequeues(event) < h.settings.Handler.RetryCount { h.eventQueue.AddRateLimited(event) - logrus.Infof(`Handler::withRetry: requeued event: %#v; error: %v`, event, err) + slog.Info("Handler::withRetry: requeued event", "event", event, "error", err) } else { h.eventQueue.Forget(event) - logrus.Warnf(`Handler::withRetry: event %#v has been dropped due to too many retries`, event) + slog.Warn(`Handler::withRetry: event has been dropped due to too many retries`, "event", event) } } // TODO: Add error logging } diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 9cc9a165..c4ea32bc 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -9,11 +9,11 @@ import ( "context" "errors" "fmt" + "log/slog" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -54,7 +54,7 @@ func NewWatcher( // Initialize initializes the Watcher, must be called before Watch func (w *Watcher) Initialize(ctx context.Context) error { - logrus.Debug("Watcher::Initialize") + slog.Debug("Watcher::Initialize") var err error w.informer = w.buildInformer() @@ -70,7 +70,7 @@ func (w *Watcher) Initialize(ctx context.Context) error { // Watch starts the process of watching for changes to Kubernetes resources. // Initialize must be called before Watch. func (w *Watcher) Watch(ctx context.Context) error { - logrus.Debug("Watcher::Watch") + slog.Debug("Watcher::Watch") if w.informer == nil { return errors.New("error: Initialize must be called before Watch") @@ -106,7 +106,7 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) { - logrus.Info("Watcher::buildEventHandlerForAdd") + slog.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -115,7 +115,7 @@ func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Error("error occurred retrieving node ips", "error", err) return } @@ -128,7 +128,7 @@ func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) // buildEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface{}) { - logrus.Info("Watcher::buildEventHandlerForDelete") + slog.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -137,7 +137,7 @@ func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Error("error occurred retrieving node ips", "error", err) return } @@ -150,7 +150,7 @@ func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface // buildEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface{}, interface{}) { - logrus.Info("Watcher::buildEventHandlerForUpdate") + slog.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API service := updated.(*v1.Service) @@ -160,7 +160,7 @@ func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Error("error occurred retrieving node ips", "error", err) return } @@ -172,7 +172,7 @@ func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface // buildInformer creates the informer used to watch for changes to Kubernetes resources. func (w *Watcher) buildInformer() cache.SharedIndexInformer { - logrus.Debug("Watcher::buildInformer") + slog.Debug("Watcher::buildInformer") factory := informers.NewSharedInformerFactoryWithOptions( w.k8sClient, w.settings.Watcher.ResyncPeriod, @@ -184,7 +184,7 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { // initializeEventListeners initializes the event listeners for the informer. func (w *Watcher) initializeEventListeners(ctx context.Context) error { - logrus.Debug("Watcher::initializeEventListeners") + slog.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ @@ -205,13 +205,13 @@ func (w *Watcher) initializeEventListeners(ctx context.Context) error { // because the master node may or may not be a worker node and thus may not be able to route traffic. func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { started := time.Now() - logrus.Debug("Watcher::retrieveNodeIps") + slog.Debug("Watcher::retrieveNodeIps") var nodeIps []string nodes, err := w.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - logrus.Errorf(`error occurred retrieving the list of nodes: %v`, err) + slog.Error("error occurred retrieving the list of nodes", "error", err) return nil, err } @@ -226,14 +226,14 @@ func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { } } - logrus.Debugf("Watcher::retrieveNodeIps duration: %d", time.Since(started).Nanoseconds()) + slog.Debug("Watcher::retrieveNodeIps duration", "duration", time.Since(started).Nanoseconds()) return nodeIps, nil } // notMasterNode determines if the node is a master node. func (w *Watcher) notMasterNode(node v1.Node) bool { - logrus.Debug("Watcher::notMasterNode") + slog.Debug("Watcher::notMasterNode") _, found := node.Labels["node-role.kubernetes.io/master"] diff --git a/internal/probation/server.go b/internal/probation/server.go index 84b5b674..9520fc76 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -7,10 +7,9 @@ package probation import ( "fmt" + "log/slog" "net/http" "time" - - "github.com/sirupsen/logrus" ) const ( @@ -27,7 +26,6 @@ const ( // HealthServer is a server that spins up endpoints for the various k8s health checks. type HealthServer struct { - // The underlying HTTP server. httpServer *http.Server @@ -52,7 +50,7 @@ func NewHealthServer() *HealthServer { // Start spins up the health server. func (hs *HealthServer) Start() { - logrus.Debugf("Starting probe listener on port %d", ListenPort) + slog.Debug("Starting probe listener", "port", ListenPort) address := fmt.Sprintf(":%d", ListenPort) @@ -64,17 +62,17 @@ func (hs *HealthServer) Start() { go func() { if err := hs.httpServer.ListenAndServe(); err != nil { - logrus.Errorf("unable to start probe listener on %s: %v", hs.httpServer.Addr, err) + slog.Error("unable to start probe listener", "address", hs.httpServer.Addr, "error", err) } }() - logrus.Info("Started probe listener on", hs.httpServer.Addr) + slog.Info("Started probe listener", "address", hs.httpServer.Addr) } // Stop shuts down the health server. func (hs *HealthServer) Stop() { if err := hs.httpServer.Close(); err != nil { - logrus.Errorf("unable to stop probe listener on %s: %v", hs.httpServer.Addr, err) + slog.Error("unable to stop probe listener", "address", hs.httpServer.Addr, "error", err) } } @@ -99,14 +97,14 @@ func (hs *HealthServer) handleProbe(writer http.ResponseWriter, _ *http.Request, writer.WriteHeader(http.StatusOK) if _, err := fmt.Fprint(writer, Ok); err != nil { - logrus.Error(err) + slog.Error(err.Error()) } } else { writer.WriteHeader(http.StatusServiceUnavailable) if _, err := fmt.Fprint(writer, ServiceNotAvailable); err != nil { - logrus.Error(err) + slog.Error(err.Error()) } } } diff --git a/internal/probation/server_test.go b/internal/probation/server_test.go index 9c7d37db..981aa3b7 100644 --- a/internal/probation/server_test.go +++ b/internal/probation/server_test.go @@ -6,11 +6,11 @@ package probation import ( + "log/slog" "net/http" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "github.com/sirupsen/logrus" ) func TestHealthServer_HandleLive(t *testing.T) { @@ -76,5 +76,5 @@ func TestHealthServer_Start(t *testing.T) { t.Errorf("Expected status code %v, got %v", http.StatusAccepted, response.StatusCode) } - logrus.Infof("received a response from the probe server: %v", response) + slog.Info("received a response from the probe server", "response", response) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index eadfbd8d..20bc99a7 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -7,13 +7,13 @@ package synchronization import ( "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" - "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ) @@ -58,10 +58,10 @@ func NewSynchronizer( // AddEvents adds a list of events to the queue. If no hosts are specified this is a null operation. // Events will fan out to the number of hosts specified before being added to the queue. func (s *Synchronizer) AddEvents(events core.ServerUpdateEvents) { - logrus.Debugf(`Synchronizer::AddEvents adding %d events`, len(events)) + slog.Debug(`Synchronizer::AddEvents adding events`, slog.Int("eventCount", len(events))) if len(s.settings.NginxPlusHosts) == 0 { - logrus.Warnf(`No Nginx Plus hosts were specified. Skipping synchronization.`) + slog.Warn(`No Nginx Plus hosts were specified. Skipping synchronization.`) return } @@ -75,10 +75,10 @@ func (s *Synchronizer) AddEvents(events core.ServerUpdateEvents) { // AddEvent adds an event to the queue. If no hosts are specified this is a null operation. // Events will be added to the queue after a random delay between MinMillisecondsJitter and MaxMillisecondsJitter. func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { - logrus.Debugf(`Synchronizer::AddEvent: %#v`, event) + slog.Debug(`Synchronizer::AddEvent`, "event", event) if event.NginxHost == `` { - logrus.Warnf(`Nginx host was not specified. Skipping synchronization.`) + slog.Warn(`Nginx host was not specified. Skipping synchronization.`) return } @@ -91,7 +91,7 @@ func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { // Run starts the Synchronizer, spins up Goroutines to process events, and waits for a stop signal. func (s *Synchronizer) Run(stopCh <-chan struct{}) { - logrus.Debug(`Synchronizer::Run`) + slog.Debug(`Synchronizer::Run`) for i := 0; i < s.settings.Synchronizer.Threads; i++ { go wait.Until(s.worker, 0, stopCh) @@ -102,7 +102,7 @@ func (s *Synchronizer) Run(stopCh <-chan struct{}) { // ShutDown stops the Synchronizer and shuts down the event queue func (s *Synchronizer) ShutDown() { - logrus.Debugf(`Synchronizer::ShutDown`) + slog.Debug(`Synchronizer::ShutDown`) s.eventQueue.ShutDownWithDrain() } @@ -110,7 +110,7 @@ func (s *Synchronizer) ShutDown() { // NOTE: There is an open issue (https://github.com/nginxinc/nginx-loadbalancer-kubernetes/issues/36) to move creation // of the underlying Border Server client to the NewBorderClient function. func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (application.Interface, error) { - logrus.Debugf(`Synchronizer::buildBorderClient`) + slog.Debug(`Synchronizer::buildBorderClient`) var err error @@ -129,7 +129,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica // fanOutEventToHosts takes a list of events and returns a list of events, one for each Border Server. func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.ServerUpdateEvents { - logrus.Debugf(`Synchronizer::fanOutEventToHosts: %#v`, event) + slog.Debug(`Synchronizer::fanOutEventToHosts`, "event", event) var events core.ServerUpdateEvents @@ -147,7 +147,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se // handleEvent dispatches an event to the proper handler function. func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleEvent: Id: %s`, event.ID) + slog.Debug(`Synchronizer::handleEvent`, slog.String("eventID", event.ID)) var err error @@ -162,13 +162,13 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { err = s.handleDeletedEvent(event) default: - logrus.Warnf(`Synchronizer::handleEvent: unknown event type: %d`, event.Type) + slog.Warn(`Synchronizer::handleEvent: unknown event type`, "type", event.Type) } if err == nil { - logrus.Infof( - `Synchronizer::handleEvent: successfully %s the nginx+ host(s) for Upstream: %s: Id(%s)`, - event.TypeName(), event.UpstreamName, event.ID) + slog.Info( + "Synchronizer::handleEvent: successfully handled the event", + "type", event.TypeName(), "upstreamName", event.UpstreamName, "eventID", event.ID) } return err @@ -176,7 +176,7 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { // handleCreatedUpdatedEvent handles events of type Created or Updated. func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleCreatedUpdatedEvent: Id: %s`, serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -194,7 +194,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU // handleDeletedEvent handles events of type Deleted. func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleDeletedEvent: Id: %s`, serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleDeletedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -212,7 +212,7 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic func (s *Synchronizer) handleNextEvent() bool { - logrus.Debug(`Synchronizer::handleNextEvent`) + slog.Debug(`Synchronizer::handleNextEvent`) evt, quit := s.eventQueue.Get() if quit { @@ -229,18 +229,18 @@ func (s *Synchronizer) handleNextEvent() bool { // worker is the main message loop func (s *Synchronizer) worker() { - logrus.Debug(`Synchronizer::worker`) + slog.Debug(`Synchronizer::worker`) for s.handleNextEvent() { } } // withRetry handles errors from the event handler and requeues events that fail func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { - logrus.Debug("Synchronizer::withRetry") + slog.Debug("Synchronizer::withRetry") if err != nil { // TODO: Add Telemetry s.eventQueue.AddRateLimited(event) - logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) + slog.Info(`Synchronizer::withRetry: requeued event`, "eventID", event.ID, "error", err) } else { s.eventQueue.Forget(event) } // TODO: Add error logging diff --git a/internal/translation/translator.go b/internal/translation/translator.go index fe8532c8..dceecf58 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -7,17 +7,17 @@ package translation import ( "fmt" + "log/slog" "strings" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" ) // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations // and used to update the Border Servers. func Translate(event *core.Event) (core.ServerUpdateEvents, error) { - logrus.Debug("Translate::Translate") + slog.Debug("Translate::Translate") return buildServerUpdateEvents(event.Service.Spec.Ports, event) } @@ -29,13 +29,13 @@ func Translate(event *core.Event) (core.ServerUpdateEvents, error) { // The NGINX+ Client uses a single server for Deleted events; // so the list of servers is broken up into individual events. func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.ServerUpdateEvents, error) { - logrus.Debugf("Translate::buildServerUpdateEvents(ports=%#v)", ports) + slog.Debug("Translate::buildServerUpdateEvents", "ports", ports) events := core.ServerUpdateEvents{} for _, port := range ports { context, upstreamName, err := getContextAndUpstreamName(port) if err != nil { - logrus.Info(err) + slog.Info(err.Error()) continue } @@ -56,7 +56,7 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se } default: - logrus.Warnf(`Translator::buildServerUpdateEvents: unknown event type: %d`, event.Type) + slog.Warn(`Translator::buildServerUpdateEvents: unknown event type`, "type", event.Type) } } From 7689cd6dbcd5ef381d86ea1df4e3729ddc214565 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:31:59 -0600 Subject: [PATCH 058/110] NLB-6753 Added nlk version to outgoing request headers --- internal/communication/factory.go | 2 ++ internal/communication/factory_test.go | 16 ++++++++++++---- internal/communication/roundtripper_test.go | 10 +++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 8664e9d6..eec593b3 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -14,6 +14,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" ) // NewHTTPClient is a factory method to create a new Http Client with a default configuration. @@ -38,6 +39,7 @@ func NewHeaders(apiKey string) []string { headers := []string{ "Content-Type: application/json", "Accept: application/json", + fmt.Sprintf("X-NLK-Version: %s", buildinfo.SemVer()), } if apiKey != "" { diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 65f5e5bb..7562484f 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -31,7 +31,7 @@ func TestNewHeaders(t *testing.T) { t.Fatalf(`headers should not be nil`) } - if len(headers) != 3 { + if len(headers) != 4 { t.Fatalf(`headers should have 3 elements`) } @@ -43,8 +43,12 @@ func TestNewHeaders(t *testing.T) { t.Fatalf(`headers[1] should be "Accept: application/json"`) } - if headers[2] != "Authorization: ApiKey fakeKey" { - t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + if headers[2] != "X-NLK-Version: " { + t.Fatalf(`headers[2] should be "X-NLK-Version: "`) + } + + if headers[3] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`headers[3] should be "Accept: Authorization: ApiKey fakeKey"`) } } @@ -56,7 +60,7 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { t.Fatalf(`headers should not be nil`) } - if len(headers) != 2 { + if len(headers) != 3 { t.Fatalf(`headers should have 2 elements`) } @@ -67,6 +71,10 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { if headers[1] != "Accept: application/json" { t.Fatalf(`headers[1] should be "Accept: application/json"`) } + + if headers[2] != "X-NLK-Version: " { + t.Fatalf(`headers[2] should be "X-NLK-Version: "`) + } } func TestNewTransport(t *testing.T) { diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index 9913600f..55dea883 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -28,7 +28,7 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers should not be nil`) } - if len(roundTripper.Headers) != 3 { + if len(roundTripper.Headers) != 4 { t.Fatalf(`roundTripper.Headers should have 3 elements`) } @@ -40,8 +40,12 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers[1] should be "Accept: application/json"`) } - if roundTripper.Headers[2] != "Authorization: ApiKey fakeKey" { - t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + if roundTripper.Headers[2] != "X-NLK-Version: " { + t.Fatalf(`roundTripper.Headers[2] should be "X-NLK-Version: "`) + } + + if roundTripper.Headers[3] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`roundTripper.Headers[3] should be "Accept: Authorization: ApiKey fakeKey"`) } if roundTripper.RoundTripper == nil { From db869bab84a507e5966d7f51b291279b6241ed50 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:18:04 -0600 Subject: [PATCH 059/110] NLB-5753 Bumped to version 0.6.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index a918a2aa..ee6cdce3 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.6.0 +0.6.1 From e2b363a666ff8deb5249a630b339f2e93df6af8a Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 7 Nov 2024 02:10:12 +0530 Subject: [PATCH 060/110] Update AKS API version in marketplace bundle The marketplace package needs to have the AKS API as the latest or anything that is under 2 years. Very conservatively choosing the latest version from last year. --- charts/armTemplate.json | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/armTemplate.json b/charts/armTemplate.json index 510c7842..06c37031 100644 --- a/charts/armTemplate.json +++ b/charts/armTemplate.json @@ -164,7 +164,7 @@ { "type": "Microsoft.ContainerService/managedClusters", "condition": "[parameters('createNewCluster')]", - "apiVersion": "2022-11-01", + "apiVersion": "2023-11-01", "name": "[parameters('clusterResourceName')]", "location": "[parameters('location')]", "dependsOn": [], diff --git a/version b/version index ee6cdce3..b6160487 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.6.1 +0.6.2 From 5875547ab0a2031f9ac019537f9b9abbeb2ce886 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 7 Nov 2024 12:40:33 -0700 Subject: [PATCH 061/110] NLB-4468 Added package pointer from ARP --- pkg/pointer/pointer.go | 56 +++++++++++++++++++++++++++++++++ pkg/pointer/pointer_test.go | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 pkg/pointer/pointer.go create mode 100644 pkg/pointer/pointer_test.go diff --git a/pkg/pointer/pointer.go b/pkg/pointer/pointer.go new file mode 100644 index 00000000..08ff6672 --- /dev/null +++ b/pkg/pointer/pointer.go @@ -0,0 +1,56 @@ +// Package pointer provides utilities that assist in working with pointers. +package pointer + +// To returns a pointer to the given value +func To[T any](v T) *T { return &v } + +// From dereferences the pointer if it is not nil or returns d +func From[T any](p *T, d T) T { + if p != nil { + return *p + } + return d +} + +// ToSlice returns a slice of pointers to the given values. +func ToSlice[T any](values []T) []*T { + if len(values) == 0 { + return nil + } + ret := make([]*T, 0, len(values)) + for _, v := range values { + v := v + ret = append(ret, &v) + } + return ret +} + +// FromSlice returns a slice of values to the given pointers, dropping any nils. +func FromSlice[T any](values []*T) []T { + if len(values) == 0 { + return nil + } + ret := make([]T, 0, len(values)) + for _, v := range values { + if v != nil { + ret = append(ret, *v) + } + } + return ret +} + +// Equal reports if p is a pointer to a value equal to v +func Equal[T comparable](p *T, v T) bool { + if p == nil { + return false + } + return *p == v +} + +// ValueEqual reports if value of pointer referenced by p is equal to value of pointer referenced by q +func ValueEqual[T comparable](p *T, q *T) bool { + if p == nil || q == nil { + return p == q + } + return *p == *q +} diff --git a/pkg/pointer/pointer_test.go b/pkg/pointer/pointer_test.go new file mode 100644 index 00000000..e929e58a --- /dev/null +++ b/pkg/pointer/pointer_test.go @@ -0,0 +1,62 @@ +package pointer_test + +import ( + "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" + "github.com/stretchr/testify/require" +) + +func TestTo(t *testing.T) { + t.Parallel() + + for _, v := range []string{"", "hello"} { + require.Equal(t, v, *pointer.To(v)) + } + for _, v := range []int{0, 123456, -123456} { + require.Equal(t, v, *pointer.To(v)) + } + for _, v := range []int64{0, 123456, -123456} { + require.Equal(t, v, *pointer.To(v)) + } +} + +func TestFrom(t *testing.T) { + t.Parallel() + + sv := "s" + sd := "default" + require.Equal(t, sd, pointer.From(nil, sd)) + require.Equal(t, sv, pointer.From(&sv, sd)) + + iv := 1 + id := 2 + require.Equal(t, id, pointer.From(nil, id)) + require.Equal(t, iv, pointer.From(&iv, id)) + + i64v := int64(1) + i64d := int64(2) + require.Equal(t, i64d, pointer.From(nil, i64d)) + require.Equal(t, i64v, pointer.From(&i64v, i64d)) +} + +func TestToSlice_FromSlice(t *testing.T) { + t.Parallel() + + v := []int{1, 2, 3} + require.Equal(t, v, pointer.FromSlice(pointer.ToSlice(v))) + require.Nil(t, pointer.ToSlice([]string{})) + require.Nil(t, pointer.FromSlice([]*string{})) + require.Equal(t, []string{"A", "B"}, pointer.FromSlice([]*string{pointer.To("A"), nil, pointer.To("B")})) +} + +func TestEqual(t *testing.T) { + t.Parallel() + + require.True(t, pointer.Equal(pointer.To(1), 1)) + require.False(t, pointer.Equal(nil, 1)) + require.False(t, pointer.Equal(pointer.To(1), 2)) + + s := new(struct{}) + require.False(t, pointer.Equal(&s, nil)) +} From c8ad54ed64d318e6ff7207d70c20ec27598305a4 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 31 Oct 2024 07:13:17 -0600 Subject: [PATCH 062/110] NLB-4468 Added support for cluster IP services, service IP addresses fetched by translator The watcher's role no longer includes sending node IPs to the translator. The watcher simply alerts the translator of a change event with respect to a specific service. It is now the translator's role to determine the upstream server addresses from the nodeport IPs or the cluster IPs, depending on the service type. There were a number of reasons for this change. The main one is that any transitory failure to fetch IP addresses using the kubernetes client will cause the update event to be readded to the workqueue, instead of abandoned. Another benefit is that we can now leverage the existing unit tests for the translator to test the cluster IP functionality here. This commit adds support for cluster IP services. The watcher fetches the service IP addresses and ports from the endpoint slices relevant to the service. The other main change here is to the way that events are handled when a service is deleted. Instead of issuing multiple requests to the nginx hosts to delete specific upstream servers, we now send a single update servers request with an empty list of servers. The nginx plus client extrapolates from the empty list and deletes all existing servers for the upstream. --- cmd/nginx-loadbalancer-kubernetes/main.go | 7 +- go.mod | 2 +- internal/core/event.go | 8 +- internal/core/event_test.go | 7 +- internal/observation/handler.go | 41 +- internal/observation/handler_test.go | 11 +- internal/observation/watcher.go | 83 +- internal/translation/translator.go | 170 ++- internal/translation/translator_test.go | 1489 +++++++++++++++------ test/mocks/mock_handler.go | 15 +- 10 files changed, 1316 insertions(+), 517 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 80643482..00be5d42 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -15,6 +15,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/observation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/probation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -54,19 +55,19 @@ func run() error { handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue) + handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) watcher, err := observation.NewWatcher(settings, handler, k8sClient) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } - err = watcher.Initialize(ctx) + err = watcher.Initialize() if err != nil { return fmt.Errorf(`error occurred initializing the watcher: %w`, err) } - go handler.Run(ctx.Done()) + go handler.Run(ctx) go synchronizer.Run(ctx.Done()) probeServer := probation.NewHealthServer() diff --git a/go.mod b/go.mod index 94a13195..6b261830 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + golang.org/x/net v0.23.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 @@ -54,7 +55,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect diff --git a/internal/core/event.go b/internal/core/event.go index 09776c94..16d5d946 100644 --- a/internal/core/event.go +++ b/internal/core/event.go @@ -24,7 +24,6 @@ const ( // Event represents a service event type Event struct { - // Type represents the event type, one of the constant values defined above. Type EventType @@ -33,18 +32,13 @@ type Event struct { // PreviousService represents the service object in its previous state PreviousService *v1.Service - - // NodeIps represents the list of node IPs in the Cluster. This is populated by the Watcher when an event is created. - // The Node IPs are needed by the BorderClient. - NodeIps []string } // NewEvent factory method to create a new Event -func NewEvent(eventType EventType, service *v1.Service, previousService *v1.Service, nodeIps []string) Event { +func NewEvent(eventType EventType, service *v1.Service, previousService *v1.Service) Event { return Event{ Type: eventType, Service: service, PreviousService: previousService, - NodeIps: nodeIps, } } diff --git a/internal/core/event_test.go b/internal/core/event_test.go index 662eb8f1..f0184fbe 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -11,9 +11,8 @@ func TestNewEvent(t *testing.T) { expectedType := Created expectedService := &v1.Service{} expectedPreviousService := &v1.Service{} - expectedNodeIps := []string{"127.0.0.1"} - event := NewEvent(expectedType, expectedService, expectedPreviousService, expectedNodeIps) + event := NewEvent(expectedType, expectedService, expectedPreviousService) if event.Type != expectedType { t.Errorf("expected Created, got %v", event.Type) @@ -26,8 +25,4 @@ func TestNewEvent(t *testing.T) { if event.PreviousService != expectedPreviousService { t.Errorf("expected previous service, got %#v", event.PreviousService) } - - if event.NodeIps[0] != expectedNodeIps[0] { - t.Errorf("expected node ips, got %#v", event.NodeIps) - } } diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 9f443597..2b5bcfbe 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -6,13 +6,13 @@ package observation import ( + "context" "fmt" "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ) @@ -23,7 +23,7 @@ type HandlerInterface interface { AddRateLimitedEvent(event *core.Event) // Run defines the interface used to start the event handler - Run(stopCh <-chan struct{}) + Run(ctx context.Context) // ShutDown defines the interface used to stop the event handler ShutDown() @@ -42,6 +42,12 @@ type Handler struct { // synchronizer is the synchronizer used to synchronize the internal representation with a Border Server synchronizer synchronization.Interface + + translator Translator +} + +type Translator interface { + Translate(context.Context, *core.Event) (core.ServerUpdateEvents, error) } // NewHandler creates a new event handler @@ -49,11 +55,13 @@ func NewHandler( settings configuration.Settings, synchronizer synchronization.Interface, eventQueue workqueue.RateLimitingInterface, + translator Translator, ) *Handler { return &Handler{ eventQueue: eventQueue, settings: settings, synchronizer: synchronizer, + translator: translator, } } @@ -63,15 +71,21 @@ func (h *Handler) AddRateLimitedEvent(event *core.Event) { h.eventQueue.AddRateLimited(event) } -// Run starts the event handler, spins up Goroutines to process events, and waits for a stop signal -func (h *Handler) Run(stopCh <-chan struct{}) { +// Run starts the event handler, spins up Goroutines to process events, and waits for context to be done +func (h *Handler) Run(ctx context.Context) { slog.Debug("Handler::Run") + worker := func() { + for h.handleNextEvent(ctx) { + // TODO: Add Telemetry + } + } + for i := 0; i < h.settings.Handler.Threads; i++ { - go wait.Until(h.worker, 0, stopCh) + go wait.Until(worker, 0, ctx.Done()) } - <-stopCh + <-ctx.Done() } // ShutDown stops the event handler and shuts down the event queue @@ -81,11 +95,11 @@ func (h *Handler) ShutDown() { } // handleEvent feeds translated events to the synchronizer -func (h *Handler) handleEvent(e *core.Event) error { +func (h *Handler) handleEvent(ctx context.Context, e *core.Event) error { slog.Debug("Handler::handleEvent", "event", e) // TODO: Add Telemetry - events, err := translation.Translate(e) + events, err := h.translator.Translate(ctx, e) if err != nil { return fmt.Errorf(`Handler::handleEvent error translating: %v`, err) } @@ -96,7 +110,7 @@ func (h *Handler) handleEvent(e *core.Event) error { } // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (h *Handler) handleNextEvent() bool { +func (h *Handler) handleNextEvent(ctx context.Context) bool { evt, quit := h.eventQueue.Get() slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) if quit { @@ -106,18 +120,11 @@ func (h *Handler) handleNextEvent() bool { defer h.eventQueue.Done(evt) event := evt.(*core.Event) - h.withRetry(h.handleEvent(event), event) + h.withRetry(h.handleEvent(ctx, event), event) return true } -// worker is the main message loop -func (h *Handler) worker() { - for h.handleNextEvent() { - // TODO: Add Telemetry - } -} - // withRetry handles errors from the event handler and requeues events that fail func (h *Handler) withRetry(err error, event *core.Event) { slog.Debug("Handler::withRetry") diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index ba4add1f..9dd736c0 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -6,6 +6,7 @@ package observation import ( + "context" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" @@ -33,7 +34,7 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { handler.AddRateLimitedEvent(event) - handler.handleNextEvent() + handler.handleNextEvent(context.Background()) if len(synchronizer.Events) != 1 { t.Errorf(`handler.AddRateLimitedEvent did not add the event to the queue`) @@ -46,7 +47,13 @@ func buildHandler() ( eventQueue := &mocks.MockRateLimiter{} synchronizer := &mocks.MockSynchronizer{} - handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue) + handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue, &fakeTranslator{}) return synchronizer, handler } + +type fakeTranslator struct{} + +func (t *fakeTranslator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { + return core.ServerUpdateEvents{{}}, nil +} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index c4ea32bc..21ab0687 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -10,12 +10,10 @@ import ( "errors" "fmt" "log/slog" - "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -53,13 +51,13 @@ func NewWatcher( } // Initialize initializes the Watcher, must be called before Watch -func (w *Watcher) Initialize(ctx context.Context) error { +func (w *Watcher) Initialize() error { slog.Debug("Watcher::Initialize") var err error w.informer = w.buildInformer() - err = w.initializeEventListeners(ctx) + err = w.initializeEventListeners() if err != nil { return fmt.Errorf(`initialization error: %w`, err) } @@ -105,7 +103,7 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. -func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) { +func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { slog.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) @@ -113,21 +111,15 @@ func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) return } - nodeIps, err := w.retrieveNodeIps(ctx) - if err != nil { - slog.Error("error occurred retrieving node ips", "error", err) - return - } - var previousService *v1.Service - e := core.NewEvent(core.Created, service, previousService, nodeIps) + e := core.NewEvent(core.Created, service, previousService) w.handler.AddRateLimitedEvent(&e) } } // buildEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. -func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface{}) { +func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { slog.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) @@ -135,21 +127,15 @@ func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface return } - nodeIps, err := w.retrieveNodeIps(ctx) - if err != nil { - slog.Error("error occurred retrieving node ips", "error", err) - return - } - var previousService *v1.Service - e := core.NewEvent(core.Deleted, service, previousService, nodeIps) + e := core.NewEvent(core.Deleted, service, previousService) w.handler.AddRateLimitedEvent(&e) } } // buildEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. -func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface{}, interface{}) { +func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { slog.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API @@ -158,14 +144,8 @@ func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface return } - nodeIps, err := w.retrieveNodeIps(ctx) - if err != nil { - slog.Error("error occurred retrieving node ips", "error", err) - return - } - previousService := previous.(*v1.Service) - e := core.NewEvent(core.Updated, service, previousService, nodeIps) + e := core.NewEvent(core.Updated, service, previousService) w.handler.AddRateLimitedEvent(&e) } } @@ -183,14 +163,14 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { } // initializeEventListeners initializes the event listeners for the informer. -func (w *Watcher) initializeEventListeners(ctx context.Context) error { +func (w *Watcher) initializeEventListeners() error { slog.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: w.buildEventHandlerForAdd(ctx), - DeleteFunc: w.buildEventHandlerForDelete(ctx), - UpdateFunc: w.buildEventHandlerForUpdate(ctx), + AddFunc: w.buildEventHandlerForAdd(), + DeleteFunc: w.buildEventHandlerForDelete(), + UpdateFunc: w.buildEventHandlerForUpdate(), } w.eventHandlerRegistration, err = w.informer.AddEventHandler(handlers) @@ -200,42 +180,3 @@ func (w *Watcher) initializeEventListeners(ctx context.Context) error { return nil } - -// notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is -// because the master node may or may not be a worker node and thus may not be able to route traffic. -func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { - started := time.Now() - slog.Debug("Watcher::retrieveNodeIps") - - var nodeIps []string - - nodes, err := w.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - slog.Error("error occurred retrieving the list of nodes", "error", err) - return nil, err - } - - for _, node := range nodes.Items { - // this is kind of a broad assumption, should probably make this a configurable option - if w.notMasterNode(node) { - for _, address := range node.Status.Addresses { - if address.Type == v1.NodeInternalIP { - nodeIps = append(nodeIps, address.Address) - } - } - } - } - - slog.Debug("Watcher::retrieveNodeIps duration", "duration", time.Since(started).Nanoseconds()) - - return nodeIps, nil -} - -// notMasterNode determines if the node is a master node. -func (w *Watcher) notMasterNode(node v1.Node) bool { - slog.Debug("Watcher::notMasterNode") - - _, found := node.Labels["node-role.kubernetes.io/master"] - - return !found -} diff --git a/internal/translation/translator.go b/internal/translation/translator.go index dceecf58..6d7928d9 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -6,20 +6,32 @@ package translation import ( + "context" "fmt" "log/slog" "strings" + "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) +type Translator struct { + k8sClient kubernetes.Interface +} + +func NewTranslator(k8sClient kubernetes.Interface) *Translator { + return &Translator{k8sClient} +} + // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations // and used to update the Border Servers. -func Translate(event *core.Event) (core.ServerUpdateEvents, error) { +func (t *Translator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { slog.Debug("Translate::Translate") - return buildServerUpdateEvents(event.Service.Spec.Ports, event) + return t.buildServerUpdateEvents(ctx, event.Service.Spec.Ports, event) } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type @@ -28,18 +40,108 @@ func Translate(event *core.Event) (core.ServerUpdateEvents, error) { // and the list of servers in NGINX+. // The NGINX+ Client uses a single server for Deleted events; // so the list of servers is broken up into individual events. -func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.ServerUpdateEvents, error) { +func (t *Translator) buildServerUpdateEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +) (events core.ServerUpdateEvents, err error) { slog.Debug("Translate::buildServerUpdateEvents", "ports", ports) + switch event.Service.Spec.Type { + case v1.ServiceTypeNodePort: + return t.buildNodeIPEvents(ctx, ports, event) + case v1.ServiceTypeClusterIP: + return t.buildClusterIPEvents(ctx, event) + default: + return events, fmt.Errorf("unsupported service type: %s", event.Service.Spec.Type) + } +} + +type upstream struct { + context string + name string +} + +func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event, +) (events core.ServerUpdateEvents, err error) { + namespace := event.Service.GetObjectMeta().GetNamespace() + serviceName := event.Service.Name + + logger := slog.With("namespace", namespace, "serviceName", serviceName) + logger.Debug("Translate::buildClusterIPEvents") + + if event.Type == core.Deleted { + for _, port := range event.Service.Spec.Ports { + context, upstreamName, pErr := getContextAndUpstreamName(port.Name) + if pErr != nil { + logger.Info(pErr.Error()) + continue + } + events = append(events, core.NewServerUpdateEvent(core.Updated, upstreamName, context, nil)) + } + return events, nil + } + + s := t.k8sClient.DiscoveryV1().EndpointSlices(namespace) + list, err := s.List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("kubernetes.io/service-name=%s", serviceName)}) + if err != nil { + logger.Error(`error occurred retrieving the list of endpoint slices`, "error", err) + return events, err + } + + upstreams := make(map[upstream][]*core.UpstreamServer) + + for _, endpointSlice := range list.Items { + for _, port := range endpointSlice.Ports { + if port.Name == nil || port.Port == nil { + continue + } + + context, upstreamName, err := getContextAndUpstreamName(*port.Name) + if err != nil { + logger.Info(err.Error()) + continue + } + + u := upstream{ + context: context, + name: upstreamName, + } + servers := upstreams[u] + + for _, endpoint := range endpointSlice.Endpoints { + for _, address := range endpoint.Addresses { + host := fmt.Sprintf("%s:%d", address, *port.Port) + servers = append(servers, core.NewUpstreamServer(host)) + } + } + + upstreams[u] = servers + } + } + + for u, servers := range upstreams { + events = append(events, core.NewServerUpdateEvent(core.Updated, u.name, u.context, servers)) + } + + return events, nil +} + +func (t *Translator) buildNodeIPEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +) (core.ServerUpdateEvents, error) { + slog.Debug("Translate::buildNodeIPEvents", "ports", ports) + events := core.ServerUpdateEvents{} for _, port := range ports { - context, upstreamName, err := getContextAndUpstreamName(port) + context, upstreamName, err := getContextAndUpstreamName(port.Name) if err != nil { slog.Info(err.Error()) continue } - upstreamServers := buildUpstreamServers(event.NodeIps, port) + addresses, err := t.retrieveNodeIps(ctx) + if err != nil { + return nil, err + } + + upstreamServers := buildUpstreamServers(addresses, port) switch event.Type { case core.Created: @@ -49,14 +151,11 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) case core.Deleted: - for _, server := range upstreamServers { - events = append(events, core.NewServerUpdateEvent( - event.Type, upstreamName, context, core.UpstreamServers{server}, - )) - } - + events = append(events, core.NewServerUpdateEvent( + core.Updated, upstreamName, context, nil, + )) default: - slog.Warn(`Translator::buildServerUpdateEvents: unknown event type`, "type", event.Type) + slog.Warn(`Translator::buildNodeIPEvents: unknown event type`, "type", event.Type) } } @@ -78,15 +177,54 @@ func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamSe // getContextAndUpstreamName returns the nginx context being supplied by the port (either "http" or "stream") // and the upstream name. -func getContextAndUpstreamName(port v1.ServicePort) (clientType string, appName string, err error) { - context, upstreamName, found := strings.Cut(port.Name, "-") +func getContextAndUpstreamName(portName string) (clientType string, appName string, err error) { + context, upstreamName, found := strings.Cut(portName, "-") switch { case !found: return clientType, appName, - fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", port.Name) + fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", portName) case context != "http" && context != "stream": - return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", port.Name) + return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", portName) default: return context, upstreamName, nil } } + +// notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is +// because the master node may or may not be a worker node and thus may not be able to route traffic. +func (t *Translator) retrieveNodeIps(ctx context.Context) ([]string, error) { + started := time.Now() + slog.Debug("Translator::retrieveNodeIps") + + var nodeIps []string + + nodes, err := t.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Error("error occurred retrieving the list of nodes", "error", err) + return nil, err + } + + for _, node := range nodes.Items { + // this is kind of a broad assumption, should probably make this a configurable option + if notMasterNode(node) { + for _, address := range node.Status.Addresses { + if address.Type == v1.NodeInternalIP { + nodeIps = append(nodeIps, address.Address) + } + } + } + } + + slog.Debug("Translator::retrieveNodeIps duration", "duration", time.Since(started).Nanoseconds()) + + return nodeIps, nil +} + +// notMasterNode determines if the node is a master node. +func notMasterNode(node v1.Node) bool { + slog.Debug("Translator::notMasterNode") + + _, found := node.Labels["node-role.kubernetes.io/master"] + + return !found +} diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 5b508c3b..73c99a45 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -12,7 +12,12 @@ import ( "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" + "golang.org/x/net/context" v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" ) const ( @@ -20,6 +25,9 @@ const ( ManyNodes = 7 NoNodes = 0 OneNode = 1 + ManyEndpointSlices = 7 + NoEndpointSlices = 0 + OneEndpointSlice = 1 TranslateErrorFormat = "Translate() error = %v" ) @@ -29,130 +37,266 @@ const ( func TestCreatedTranslateNoPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 + testcases := map[string]struct{ serviceType v1.ServiceType }{ + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + } - service := defaultService() - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildCreatedEvent(service) + + translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestCreatedTranslateNoInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 + testcases := map[string]struct{ serviceType v1.ServiceType }{ + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + } - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) + + translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } +//nolint:dupl func TestCreatedTranslateOneInterestingPort(t *testing.T) { t.Parallel() - const expectedEventCount = 1 - const portCount = 1 + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + expectedServerCount: OneEndpointSlice, + }, + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 1 + const portCount = 1 - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) + } } +//nolint:dupl func TestCreatedTranslateManyInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 4 - const portCount = 4 + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + expectedServerCount: OneEndpointSlice, + }, + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 4 + const portCount = 4 - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) + } } +//nolint:dupl func TestCreatedTranslateManyMixedPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) + } } func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + expectedServerCount: ManyNodes, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + expectedServerCount: ManyEndpointSlices, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, ManyNodes, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } /* @@ -161,130 +305,289 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { func TestUpdatedTranslateNoPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - service := defaultService() - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 - - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestUpdatedTranslateOneInterestingPort(t *testing.T) { t.Parallel() - const expectedEventCount = 1 - const portCount = 1 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 1 + const portCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, OneNode, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 4 - const portCount = 4 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 4 + const portCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyMixedPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + expectedServerCount: ManyNodes, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + expectedServerCount: ManyEndpointSlices, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } /* @@ -293,331 +596,682 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - service := defaultService() - event := buildDeletedEvent(service, NoNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 1, 1), + }, + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 1 + const portCount = 1 - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 4 - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 4, 4), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 6, 2), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) - assertExpectedServerCount(t, ManyNodes, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - - service := defaultService() - event := buildDeletedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service) - assertExpectedServerCount(t, ManyNodes, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 - - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const expectedEventCount = 0 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 1 - const portCount = 1 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const portCount = 1 + const expectedEventCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - assertExpectedServerCount(t, OneNode, translatedEvents) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 4 - const portCount = 4 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - service := defaultService() - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 0, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 1 - const updatablePortCount = 0 - const expectedEventCount = updatablePortCount * ManyNodes - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const updatablePortCount = 0 + const expectedEventCount = updatablePortCount * ManyNodes + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 1 - const expectedEventCount = portCount * ManyNodes - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 1), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const expectedEventCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 4 - const expectedEventCount = portCount * ManyNodes - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 4, 4), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 6 - const updatablePortCount = 2 - const expectedEventCount = updatablePortCount * ManyNodes - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } func assertExpectedServerCount(t *testing.T, expectedCount int, events core.ServerUpdateEvents) { @@ -629,46 +1283,98 @@ func assertExpectedServerCount(t *testing.T, expectedCount int, events core.Serv } } -func defaultService() *v1.Service { - return &v1.Service{} +func defaultService(serviceType v1.ServiceType) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-service", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Spec: v1.ServiceSpec{ + Type: serviceType, + }, + } } -func serviceWithPorts(ports []v1.ServicePort) *v1.Service { +func serviceWithPorts(serviceType v1.ServiceType, ports []v1.ServicePort) *v1.Service { return &v1.Service{ Spec: v1.ServiceSpec{ + Type: serviceType, Ports: ports, }, } } -func buildCreatedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Created, service, nodeCount) +func buildCreatedEvent(service *v1.Service) core.Event { + return buildEvent(core.Created, service) } -func buildDeletedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Deleted, service, nodeCount) +func buildDeletedEvent(service *v1.Service) core.Event { + return buildEvent(core.Deleted, service) } -func buildUpdatedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Updated, service, nodeCount) +func buildUpdatedEvent(service *v1.Service) core.Event { + return buildEvent(core.Updated, service) } -func buildEvent(eventType core.EventType, service *v1.Service, nodeCount int) core.Event { - previousService := defaultService() +func buildEvent(eventType core.EventType, service *v1.Service) core.Event { + previousService := defaultService(service.Spec.Type) - nodeIps := generateNodeIps(nodeCount) + event := core.NewEvent(eventType, service, previousService) + event.Service.Name = "default-service" + return event +} - return core.NewEvent(eventType, service, previousService, nodeIps) +func generateNodes(count int) (nodes []v1.Node) { + for i := 0; i < count; i++ { + nodes = append(nodes, v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("node%d", i), + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: fmt.Sprintf("10.0.0.%v", i), + }, + }, + }, + }) + } + + return nodes } -func generateNodeIps(count int) []string { - var nodeIps []string +func generateEndpointSlices(endpointCount, portCount, updatablePortCount int, +) (endpointSlices []discovery.EndpointSlice) { + servicePorts := generateUpdatablePorts(portCount, updatablePortCount) - for i := 0; i < count; i++ { - nodeIps = append(nodeIps, fmt.Sprintf("10.0.0.%v", i)) + ports := make([]discovery.EndpointPort, 0, len(servicePorts)) + for _, servicePort := range servicePorts { + ports = append(ports, discovery.EndpointPort{ + Name: pointer.To(servicePort.Name), + Port: pointer.To(int32(8080)), + }) + } + + var endpoints []discovery.Endpoint + for i := 0; i < endpointCount; i++ { + endpoints = append(endpoints, discovery.Endpoint{ + Addresses: []string{ + fmt.Sprintf("10.0.0.%v", i), + }, + }) } - return nodeIps + endpointSlices = append(endpointSlices, discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "endpointSlice", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Endpoints: endpoints, + Ports: ports, + }) + + return endpointSlices } func generatePorts(portCount int) []v1.ServicePort { @@ -708,3 +1414,14 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort return ports } + +func NewFakeClient(endpointSlices []discovery.EndpointSlice, nodes []v1.Node) *fake.Clientset { + return fake.NewSimpleClientset( + &discovery.EndpointSliceList{ + Items: endpointSlices, + }, + &v1.NodeList{ + Items: nodes, + }, + ) +} diff --git a/test/mocks/mock_handler.go b/test/mocks/mock_handler.go index b854db9e..f144cea0 100644 --- a/test/mocks/mock_handler.go +++ b/test/mocks/mock_handler.go @@ -5,23 +5,22 @@ package mocks -import "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" +import ( + "context" -type MockHandler struct { -} + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" +) -func (h *MockHandler) AddRateLimitedEvent(_ *core.Event) { +type MockHandler struct{} +func (h *MockHandler) AddRateLimitedEvent(_ *core.Event) { } func (h *MockHandler) Initialize() { - } -func (h *MockHandler) Run(_ <-chan struct{}) { - +func (h *MockHandler) Run(ctx context.Context) { } func (h *MockHandler) ShutDown() { - } From 64bcad4c95870ee196afac91deba0fad6ec1239a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 7 Nov 2024 14:58:10 -0700 Subject: [PATCH 063/110] NLB-4468 Bumped version to 0.7.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index b6160487..faef31a4 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.6.2 +0.7.0 From 8f6f59f3bf552ab1e919b4335922c66ed3769753 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 8 Nov 2024 02:02:38 +0530 Subject: [PATCH 064/110] NLB-5868: Use same image tag as appVersion We have always aimed at keeping the chart and app versions the same, but we also set the image tag in the values file. Doing so forces the image tag to be set to what's in the values files instead of using the appVersion of the Chart. This leads to issues where a newer version of the chart has an older image tag. What we want instead is to use the appVersion of the Helm chart as the image tag and only use the image tag in the values file when it is set. --- charts/nlk/values.yaml | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index f5be2444..851f67db 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -18,7 +18,7 @@ nlk: repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always ## Overrides the image tag whose default is the chart appVersion. - tag: 0.4.0 + # tag: 0.4.0 imagePullSecrets: [] nameOverride: "" fullnameOverride: "" diff --git a/version b/version index faef31a4..39e898a4 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.7.0 +0.7.1 From 8352d578f5fa0634dee3a3ab12630a0e249473ee Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 11 Nov 2024 09:13:04 -0700 Subject: [PATCH 065/110] NLB-5872 NLK main routine uses a shared informer factory All NLK modules can now consume shared informer resources that are created from a single factory. In this commit the only consumer of this shared factory is the watcher, but in future the translator will make use of it as well. The pattern makes initializing shared resources easier (e.g. a single call to wait for sync, a single call to start the factory). It will also help to avoid circular dependencies as modules will not be dependent on each other to expose shared informer cache resources. I've followed patterns laid down by the kubernetes project's sample controller with respect to the watcher. The watcher constructor now adds event handler routines to its shared informer before the informers are started. This eliminates the need for a separate Initialize() routine. --- cmd/nginx-loadbalancer-kubernetes/main.go | 20 ++-- internal/observation/watcher.go | 110 ++++++++-------------- internal/observation/watcher_test.go | 17 ++-- 3 files changed, 61 insertions(+), 86 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 00be5d42..99240d3a 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -17,6 +17,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" @@ -53,18 +54,25 @@ func run() error { return fmt.Errorf(`error initializing synchronizer: %w`, err) } + factory := informers.NewSharedInformerFactoryWithOptions( + k8sClient, settings.Watcher.ResyncPeriod, + ) + handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) - watcher, err := observation.NewWatcher(settings, handler, k8sClient) + watcher, err := observation.NewWatcher(settings, handler, factory.Core().V1().Services()) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } - err = watcher.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing the watcher: %w`, err) + factory.Start(ctx.Done()) + results := factory.WaitForCacheSync(ctx.Done()) + for name, success := range results { + if !success { + return fmt.Errorf(`error occurred waiting for cache sync for %s`, name) + } } go handler.Run(ctx) @@ -73,9 +81,9 @@ func run() error { probeServer := probation.NewHealthServer() probeServer.Start() - err = watcher.Watch(ctx) + err = watcher.Run(ctx) if err != nil { - return fmt.Errorf(`error occurred watching for events: %w`, err) + return fmt.Errorf(`error occurred running watcher: %w`, err) } <-ctx.Done() diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 21ab0687..5a2905e9 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -7,7 +7,6 @@ package observation import ( "context" - "errors" "fmt" "log/slog" @@ -15,8 +14,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" + coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/cache" ) @@ -24,69 +22,53 @@ import ( // Particularly, Services in the namespace defined in the WatcherSettings::NginxIngressNamespace setting. // When a change is detected, an Event is generated and added to the Handler's queue. type Watcher struct { - // eventHandlerRegistration is used to track the event handlers - eventHandlerRegistration interface{} - // handler is the event handler handler HandlerInterface - // informer is the informer used to watch for changes to Kubernetes resources - informer cache.SharedIndexInformer - - k8sClient kubernetes.Interface - // settings is the configuration settings settings configuration.Settings + + // servicesInformer is the informer used to watch for changes to services + servicesInformer cache.SharedIndexInformer } // NewWatcher creates a new Watcher func NewWatcher( - settings configuration.Settings, handler HandlerInterface, k8sClient kubernetes.Interface, + settings configuration.Settings, + handler HandlerInterface, + serviceInformer coreinformers.ServiceInformer, ) (*Watcher, error) { - return &Watcher{ - handler: handler, - settings: settings, - k8sClient: k8sClient, - }, nil -} + if serviceInformer == nil { + return nil, fmt.Errorf("service informer cannot be nil") + } -// Initialize initializes the Watcher, must be called before Watch -func (w *Watcher) Initialize() error { - slog.Debug("Watcher::Initialize") - var err error + servicesInformer := serviceInformer.Informer() - w.informer = w.buildInformer() + w := &Watcher{ + handler: handler, + settings: settings, + servicesInformer: servicesInformer, + } - err = w.initializeEventListeners() - if err != nil { - return fmt.Errorf(`initialization error: %w`, err) + if err := w.initializeEventListeners(servicesInformer); err != nil { + return nil, err } - return nil + return w, nil } -// Watch starts the process of watching for changes to Kubernetes resources. +// Run starts the process of watching for changes to Kubernetes resources. // Initialize must be called before Watch. -func (w *Watcher) Watch(ctx context.Context) error { - slog.Debug("Watcher::Watch") - - if w.informer == nil { - return errors.New("error: Initialize must be called before Watch") +func (w *Watcher) Run(ctx context.Context) error { + if w.servicesInformer == nil { + return fmt.Errorf(`servicesInformer is nil`) } + slog.Debug("Watcher::Watch") + defer utilruntime.HandleCrash() defer w.handler.ShutDown() - go w.informer.Run(ctx.Done()) - - if !cache.WaitForNamedCacheSync( - w.settings.Handler.WorkQueueSettings.Name, - ctx.Done(), - w.informer.HasSynced, - ) { - return fmt.Errorf(`error occurred waiting for the cache to sync`) - } - <-ctx.Done() return nil } @@ -101,10 +83,10 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { return annotation == w.settings.Watcher.ServiceAnnotation } -// buildEventHandlerForAdd creates a function that is used as an event handler +// buildServiceEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. -func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { - slog.Info("Watcher::buildEventHandlerForAdd") +func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -117,10 +99,10 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { } } -// buildEventHandlerForDelete creates a function that is used as an event handler +// buildServiceEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. -func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { - slog.Info("Watcher::buildEventHandlerForDelete") +func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -133,10 +115,10 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { } } -// buildEventHandlerForUpdate creates a function that is used as an event handler +// buildServiceEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. -func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { - slog.Info("Watcher::buildEventHandlerForUpdate") +func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API service := updated.(*v1.Service) @@ -150,30 +132,20 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { } } -// buildInformer creates the informer used to watch for changes to Kubernetes resources. -func (w *Watcher) buildInformer() cache.SharedIndexInformer { - slog.Debug("Watcher::buildInformer") - - factory := informers.NewSharedInformerFactoryWithOptions( - w.k8sClient, w.settings.Watcher.ResyncPeriod, - ) - informer := factory.Core().V1().Services().Informer() - - return informer -} - // initializeEventListeners initializes the event listeners for the informer. -func (w *Watcher) initializeEventListeners() error { +func (w *Watcher) initializeEventListeners( + servicesInformer cache.SharedIndexInformer, +) error { slog.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: w.buildEventHandlerForAdd(), - DeleteFunc: w.buildEventHandlerForDelete(), - UpdateFunc: w.buildEventHandlerForUpdate(), + AddFunc: w.buildServiceEventHandlerForAdd(), + DeleteFunc: w.buildServiceEventHandlerForDelete(), + UpdateFunc: w.buildServiceEventHandlerForUpdate(), } - w.eventHandlerRegistration, err = w.informer.AddEventHandler(handlers) + _, err = servicesInformer.AddEventHandler(handlers) if err != nil { return fmt.Errorf(`error occurred adding event handlers: %w`, err) } diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index f8de8496..6c782977 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -6,25 +6,20 @@ package observation import ( - "context" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "k8s.io/client-go/kubernetes" + "github.com/stretchr/testify/require" ) -func TestWatcher_MustInitialize(t *testing.T) { +func TestWatcher_ErrWithNilInformer(t *testing.T) { t.Parallel() - watcher, _ := buildWatcher() - if err := watcher.Watch(context.Background()); err == nil { - t.Errorf("Expected error, got %s", err) - } + _, err := buildWatcherWithNilInformer() + require.Error(t, err, "expected construction of watcher with nil informer to fail") } -func buildWatcher() (*Watcher, error) { - k8sClient := &kubernetes.Clientset{} +func buildWatcherWithNilInformer() (*Watcher, error) { handler := &mocks.MockHandler{} - - return NewWatcher(configuration.Settings{}, handler, k8sClient) + return NewWatcher(configuration.Settings{}, handler, nil) } From d5a7946e1468a312d9d6606713493847d0ae5ea6 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 13 Nov 2024 15:07:27 -0700 Subject: [PATCH 066/110] NLB-5872 Added event handlers for endpoint slice events Also added the register of services we care about --- cmd/nginx-loadbalancer-kubernetes/main.go | 5 +- internal/observation/register.go | 49 ++++++++++ internal/observation/watcher.go | 107 ++++++++++++++++++++-- internal/observation/watcher_test.go | 2 +- 4 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 internal/observation/register.go diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 99240d3a..21c609c5 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -58,11 +58,14 @@ func run() error { k8sClient, settings.Watcher.ResyncPeriod, ) + serviceInformer := factory.Core().V1().Services() + endpointSliceInformer := factory.Discovery().V1().EndpointSlices() + handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) - watcher, err := observation.NewWatcher(settings, handler, factory.Core().V1().Services()) + watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } diff --git a/internal/observation/register.go b/internal/observation/register.go new file mode 100644 index 00000000..26ab86d6 --- /dev/null +++ b/internal/observation/register.go @@ -0,0 +1,49 @@ +package observation + +import ( + "sync" + + v1 "k8s.io/api/core/v1" +) + +// register holds references to the services that the user has configured for use with NLK +type register struct { + mu sync.RWMutex // protects register + services map[registerKey]*v1.Service +} + +type registerKey struct { + serviceName string + namespace string +} + +func newRegister() *register { + return ®ister{ + services: make(map[registerKey]*v1.Service), + } +} + +// addOrUpdateService adds the service to the register if not found, else updates the existing service +func (r *register) addOrUpdateService(service *v1.Service) { + r.mu.Lock() + defer r.mu.Unlock() + + r.services[registerKey{namespace: service.Namespace, serviceName: service.Name}] = service +} + +// removeService removes the service from the register +func (r *register) removeService(service *v1.Service) { + r.mu.Lock() + defer r.mu.Unlock() + + delete(r.services, registerKey{namespace: service.Namespace, serviceName: service.Name}) +} + +// getService returns the service from the register if found +func (r *register) getService(namespace string, serviceName string) (*v1.Service, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + s, ok := r.services[registerKey{namespace: namespace, serviceName: serviceName}] + return s, ok +} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 5a2905e9..9752b38a 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -13,8 +13,10 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" coreinformers "k8s.io/client-go/informers/core/v1" + discoveryinformers "k8s.io/client-go/informers/discovery/v1" "k8s.io/client-go/tools/cache" ) @@ -30,6 +32,11 @@ type Watcher struct { // servicesInformer is the informer used to watch for changes to services servicesInformer cache.SharedIndexInformer + + // endpointSliceInformer is the informer used to watch for changes to endpoint slices + endpointSliceInformer cache.SharedIndexInformer + + register *register } // NewWatcher creates a new Watcher @@ -37,17 +44,25 @@ func NewWatcher( settings configuration.Settings, handler HandlerInterface, serviceInformer coreinformers.ServiceInformer, + endpointSliceInformer discoveryinformers.EndpointSliceInformer, ) (*Watcher, error) { if serviceInformer == nil { return nil, fmt.Errorf("service informer cannot be nil") } + if endpointSliceInformer == nil { + return nil, fmt.Errorf("endpoint slice informer cannot be nil") + } + servicesInformer := serviceInformer.Informer() + endpointSlicesInformer := endpointSliceInformer.Informer() w := &Watcher{ - handler: handler, - settings: settings, - servicesInformer: servicesInformer, + handler: handler, + settings: settings, + servicesInformer: servicesInformer, + endpointSliceInformer: endpointSlicesInformer, + register: newRegister(), } if err := w.initializeEventListeners(servicesInformer); err != nil { @@ -83,6 +98,69 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { return annotation == w.settings.Watcher.ServiceAnnotation } +func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForAdd") + return func(obj interface{}) { + endpointSlice, ok := obj.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", obj) + return + } + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } +} + +func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForUpdate") + return func(previous, updated interface{}) { + endpointSlice, ok := updated.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", updated) + return + } + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } +} + +func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForDelete") + return func(obj interface{}) { + endpointSlice, ok := obj.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", obj) + return + } + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + var previousService *v1.Service + e := core.NewEvent(core.Deleted, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } +} + // buildServiceEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { @@ -93,6 +171,8 @@ func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { return } + w.register.addOrUpdateService(service) + var previousService *v1.Service e := core.NewEvent(core.Created, service, previousService) w.handler.AddRateLimitedEvent(&e) @@ -109,6 +189,8 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { return } + w.register.removeService(service) + var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService) w.handler.AddRateLimitedEvent(&e) @@ -126,6 +208,8 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac return } + w.register.addOrUpdateService(service) + previousService := previous.(*v1.Service) e := core.NewEvent(core.Updated, service, previousService) w.handler.AddRateLimitedEvent(&e) @@ -139,15 +223,26 @@ func (w *Watcher) initializeEventListeners( slog.Debug("Watcher::initializeEventListeners") var err error - handlers := cache.ResourceEventHandlerFuncs{ + serviceHandlers := cache.ResourceEventHandlerFuncs{ AddFunc: w.buildServiceEventHandlerForAdd(), DeleteFunc: w.buildServiceEventHandlerForDelete(), UpdateFunc: w.buildServiceEventHandlerForUpdate(), } - _, err = servicesInformer.AddEventHandler(handlers) + endpointSliceHandlers := cache.ResourceEventHandlerFuncs{ + AddFunc: w.buildEndpointSlicesEventHandlerForAdd(), + DeleteFunc: w.buildEndpointSlicesEventHandlerForDelete(), + UpdateFunc: w.buildEndpointSlicesEventHandlerForUpdate(), + } + + _, err = servicesInformer.AddEventHandler(serviceHandlers) + if err != nil { + return fmt.Errorf(`error occurred adding service event handlers: %w`, err) + } + + _, err = w.endpointSliceInformer.AddEventHandler(endpointSliceHandlers) if err != nil { - return fmt.Errorf(`error occurred adding event handlers: %w`, err) + return fmt.Errorf(`error occurred adding endpoint slice event handlers: %w`, err) } return nil diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 6c782977..b39a9890 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -21,5 +21,5 @@ func TestWatcher_ErrWithNilInformer(t *testing.T) { func buildWatcherWithNilInformer() (*Watcher, error) { handler := &mocks.MockHandler{} - return NewWatcher(configuration.Settings{}, handler, nil) + return NewWatcher(configuration.Settings{}, handler, nil, nil) } From 0aba7eea26f6e2b813bede7d1958c45ca48ffe9f Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 14 Nov 2024 09:44:58 -0700 Subject: [PATCH 067/110] NLB-5872 Added node informer to watcher Any node event from the k8s cluster will cause a service event for every registered service to be added to the handler's rate-limited work queue. --- cmd/nginx-loadbalancer-kubernetes/main.go | 3 +- internal/observation/register.go | 14 ++++++ internal/observation/watcher.go | 60 +++++++++++++++++++++++ internal/observation/watcher_test.go | 4 +- 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 21c609c5..21f3e098 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -60,12 +60,13 @@ func run() error { serviceInformer := factory.Core().V1().Services() endpointSliceInformer := factory.Discovery().V1().EndpointSlices() + nodesInformer := factory.Core().V1().Nodes() handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) - watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer) + watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer, nodesInformer) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } diff --git a/internal/observation/register.go b/internal/observation/register.go index 26ab86d6..bfe61f80 100644 --- a/internal/observation/register.go +++ b/internal/observation/register.go @@ -47,3 +47,17 @@ func (r *register) getService(namespace string, serviceName string) (*v1.Service s, ok := r.services[registerKey{namespace: namespace, serviceName: serviceName}] return s, ok } + +// listServices returns all the services in the register +func (r *register) listServices() []*v1.Service { + r.mu.RLock() + defer r.mu.RUnlock() + + services := make([]*v1.Service, 0, len(r.services)) + + for _, service := range r.services { + services = append(services, service) + } + + return services +} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 9752b38a..21d6e028 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -36,6 +36,9 @@ type Watcher struct { // endpointSliceInformer is the informer used to watch for changes to endpoint slices endpointSliceInformer cache.SharedIndexInformer + // nodesInformer is the informer used to watch for changes to nodes + nodesInformer cache.SharedIndexInformer + register *register } @@ -45,6 +48,7 @@ func NewWatcher( handler HandlerInterface, serviceInformer coreinformers.ServiceInformer, endpointSliceInformer discoveryinformers.EndpointSliceInformer, + nodeInformer coreinformers.NodeInformer, ) (*Watcher, error) { if serviceInformer == nil { return nil, fmt.Errorf("service informer cannot be nil") @@ -54,14 +58,20 @@ func NewWatcher( return nil, fmt.Errorf("endpoint slice informer cannot be nil") } + if nodeInformer == nil { + return nil, fmt.Errorf("node informer cannot be nil") + } + servicesInformer := serviceInformer.Informer() endpointSlicesInformer := endpointSliceInformer.Informer() + nodesInformer := nodeInformer.Informer() w := &Watcher{ handler: handler, settings: settings, servicesInformer: servicesInformer, endpointSliceInformer: endpointSlicesInformer, + nodesInformer: nodesInformer, register: newRegister(), } @@ -98,9 +108,46 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { return annotation == w.settings.Watcher.ServiceAnnotation } +func (w *Watcher) buildNodesEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForAdd") + return func(obj interface{}) { + slog.Debug("received node add event") + for _, service := range w.register.listServices() { + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } + } +} + +func (w *Watcher) buildNodesEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForUpdate") + return func(previous, updated interface{}) { + slog.Debug("received node update event") + for _, service := range w.register.listServices() { + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } + } +} + +func (w *Watcher) buildNodesEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForDelete") + return func(obj interface{}) { + slog.Debug("received node delete event") + for _, service := range w.register.listServices() { + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } + } +} + func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { slog.Info("Watcher::buildEndpointSlicesEventHandlerForAdd") return func(obj interface{}) { + slog.Debug("received endpoint slice add event") endpointSlice, ok := obj.(*discovery.EndpointSlice) if !ok { slog.Error("could not convert event object to EndpointSlice", "obj", obj) @@ -122,6 +169,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, interface{}) { slog.Info("Watcher::buildEndpointSlicesEventHandlerForUpdate") return func(previous, updated interface{}) { + slog.Debug("received endpoint slice update event") endpointSlice, ok := updated.(*discovery.EndpointSlice) if !ok { slog.Error("could not convert event object to EndpointSlice", "obj", updated) @@ -143,6 +191,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, i func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { slog.Info("Watcher::buildEndpointSlicesEventHandlerForDelete") return func(obj interface{}) { + slog.Debug("received endpoint slice delete event") endpointSlice, ok := obj.(*discovery.EndpointSlice) if !ok { slog.Error("could not convert event object to EndpointSlice", "obj", obj) @@ -235,6 +284,12 @@ func (w *Watcher) initializeEventListeners( UpdateFunc: w.buildEndpointSlicesEventHandlerForUpdate(), } + nodeHandlers := cache.ResourceEventHandlerFuncs{ + AddFunc: w.buildNodesEventHandlerForAdd(), + DeleteFunc: w.buildNodesEventHandlerForDelete(), + UpdateFunc: w.buildNodesEventHandlerForUpdate(), + } + _, err = servicesInformer.AddEventHandler(serviceHandlers) if err != nil { return fmt.Errorf(`error occurred adding service event handlers: %w`, err) @@ -245,5 +300,10 @@ func (w *Watcher) initializeEventListeners( return fmt.Errorf(`error occurred adding endpoint slice event handlers: %w`, err) } + _, err = w.nodesInformer.AddEventHandler(nodeHandlers) + if err != nil { + return fmt.Errorf(`error occurred adding node event handlers: %w`, err) + } + return nil } diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index b39a9890..46184089 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestWatcher_ErrWithNilInformer(t *testing.T) { +func TestWatcher_ErrWithNilInformers(t *testing.T) { t.Parallel() _, err := buildWatcherWithNilInformer() require.Error(t, err, "expected construction of watcher with nil informer to fail") @@ -21,5 +21,5 @@ func TestWatcher_ErrWithNilInformer(t *testing.T) { func buildWatcherWithNilInformer() (*Watcher, error) { handler := &mocks.MockHandler{} - return NewWatcher(configuration.Settings{}, handler, nil, nil) + return NewWatcher(configuration.Settings{}, handler, nil, nil, nil) } From 80210caca76f10a8e049856b6416cb8efca54ec8 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 15 Nov 2024 10:03:12 -0700 Subject: [PATCH 068/110] NLB-5872 Translator now uses shared listers to access endpoint slices and node ports This will save us from redundant kubernetes API calls. --- cmd/nginx-loadbalancer-kubernetes/main.go | 5 +- go.mod | 2 +- internal/observation/handler.go | 12 +- internal/observation/handler_test.go | 5 +- internal/translation/translator.go | 61 +++-- internal/translation/translator_test.go | 279 +++++++++++++--------- 6 files changed, 212 insertions(+), 152 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 21f3e098..b7381f06 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -60,11 +60,14 @@ func run() error { serviceInformer := factory.Core().V1().Services() endpointSliceInformer := factory.Discovery().V1().EndpointSlices() + endpointSliceLister := endpointSliceInformer.Lister() nodesInformer := factory.Core().V1().Nodes() + nodesLister := nodesInformer.Lister() handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) + translator := translation.NewTranslator(endpointSliceLister, nodesLister) + handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translator) watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer, nodesInformer) if err != nil { diff --git a/go.mod b/go.mod index 6b261830..94a13195 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/net v0.23.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 @@ -55,6 +54,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 2b5bcfbe..45ae6c4e 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -47,7 +47,7 @@ type Handler struct { } type Translator interface { - Translate(context.Context, *core.Event) (core.ServerUpdateEvents, error) + Translate(*core.Event) (core.ServerUpdateEvents, error) } // NewHandler creates a new event handler @@ -76,7 +76,7 @@ func (h *Handler) Run(ctx context.Context) { slog.Debug("Handler::Run") worker := func() { - for h.handleNextEvent(ctx) { + for h.handleNextEvent() { // TODO: Add Telemetry } } @@ -95,11 +95,11 @@ func (h *Handler) ShutDown() { } // handleEvent feeds translated events to the synchronizer -func (h *Handler) handleEvent(ctx context.Context, e *core.Event) error { +func (h *Handler) handleEvent(e *core.Event) error { slog.Debug("Handler::handleEvent", "event", e) // TODO: Add Telemetry - events, err := h.translator.Translate(ctx, e) + events, err := h.translator.Translate(e) if err != nil { return fmt.Errorf(`Handler::handleEvent error translating: %v`, err) } @@ -110,7 +110,7 @@ func (h *Handler) handleEvent(ctx context.Context, e *core.Event) error { } // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (h *Handler) handleNextEvent(ctx context.Context) bool { +func (h *Handler) handleNextEvent() bool { evt, quit := h.eventQueue.Get() slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) if quit { @@ -120,7 +120,7 @@ func (h *Handler) handleNextEvent(ctx context.Context) bool { defer h.eventQueue.Done(evt) event := evt.(*core.Event) - h.withRetry(h.handleEvent(ctx, event), event) + h.withRetry(h.handleEvent(event), event) return true } diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index 9dd736c0..550b318c 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -6,7 +6,6 @@ package observation import ( - "context" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" @@ -34,7 +33,7 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { handler.AddRateLimitedEvent(event) - handler.handleNextEvent(context.Background()) + handler.handleNextEvent() if len(synchronizer.Events) != 1 { t.Errorf(`handler.AddRateLimitedEvent did not add the event to the queue`) @@ -54,6 +53,6 @@ func buildHandler() ( type fakeTranslator struct{} -func (t *fakeTranslator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { +func (t *fakeTranslator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { return core.ServerUpdateEvents{{}}, nil } diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 6d7928d9..117c0b91 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -6,7 +6,6 @@ package translation import ( - "context" "fmt" "log/slog" "strings" @@ -14,24 +13,32 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" ) type Translator struct { - k8sClient kubernetes.Interface + endpointSliceLister discoverylisters.EndpointSliceLister + nodeLister corelisters.NodeLister } -func NewTranslator(k8sClient kubernetes.Interface) *Translator { - return &Translator{k8sClient} +func NewTranslator( + endpointSliceLister discoverylisters.EndpointSliceLister, + nodeLister corelisters.NodeLister, +) *Translator { + return &Translator{ + endpointSliceLister: endpointSliceLister, + nodeLister: nodeLister, + } } // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations // and used to update the Border Servers. -func (t *Translator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { +func (t *Translator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { slog.Debug("Translate::Translate") - return t.buildServerUpdateEvents(ctx, event.Service.Spec.Ports, event) + return t.buildServerUpdateEvents(event.Service.Spec.Ports, event) } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type @@ -40,15 +47,15 @@ func (t *Translator) Translate(ctx context.Context, event *core.Event) (core.Ser // and the list of servers in NGINX+. // The NGINX+ Client uses a single server for Deleted events; // so the list of servers is broken up into individual events. -func (t *Translator) buildServerUpdateEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +func (t *Translator) buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event, ) (events core.ServerUpdateEvents, err error) { slog.Debug("Translate::buildServerUpdateEvents", "ports", ports) switch event.Service.Spec.Type { case v1.ServiceTypeNodePort: - return t.buildNodeIPEvents(ctx, ports, event) + return t.buildNodeIPEvents(ports, event) case v1.ServiceTypeClusterIP: - return t.buildClusterIPEvents(ctx, event) + return t.buildClusterIPEvents(event) default: return events, fmt.Errorf("unsupported service type: %s", event.Service.Spec.Type) } @@ -59,8 +66,7 @@ type upstream struct { name string } -func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event, -) (events core.ServerUpdateEvents, err error) { +func (t *Translator) buildClusterIPEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { namespace := event.Service.GetObjectMeta().GetNamespace() serviceName := event.Service.Name @@ -79,8 +85,14 @@ func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event return events, nil } - s := t.k8sClient.DiscoveryV1().EndpointSlices(namespace) - list, err := s.List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("kubernetes.io/service-name=%s", serviceName)}) + lister := t.endpointSliceLister.EndpointSlices(namespace) + selector, err := labels.Parse(fmt.Sprintf("kubernetes.io/service-name=%s", serviceName)) + if err != nil { + logger.Error(`error occurred parsing the selector`, "error", err) + return events, err + } + + list, err := lister.List(selector) if err != nil { logger.Error(`error occurred retrieving the list of endpoint slices`, "error", err) return events, err @@ -88,7 +100,7 @@ func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event upstreams := make(map[upstream][]*core.UpstreamServer) - for _, endpointSlice := range list.Items { + for _, endpointSlice := range list { for _, port := range endpointSlice.Ports { if port.Name == nil || port.Port == nil { continue @@ -124,7 +136,7 @@ func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event return events, nil } -func (t *Translator) buildNodeIPEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +func (t *Translator) buildNodeIPEvents(ports []v1.ServicePort, event *core.Event, ) (core.ServerUpdateEvents, error) { slog.Debug("Translate::buildNodeIPEvents", "ports", ports) @@ -136,7 +148,7 @@ func (t *Translator) buildNodeIPEvents(ctx context.Context, ports []v1.ServicePo continue } - addresses, err := t.retrieveNodeIps(ctx) + addresses, err := t.retrieveNodeIps() if err != nil { return nil, err } @@ -192,21 +204,26 @@ func getContextAndUpstreamName(portName string) (clientType string, appName stri // notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is // because the master node may or may not be a worker node and thus may not be able to route traffic. -func (t *Translator) retrieveNodeIps(ctx context.Context) ([]string, error) { +func (t *Translator) retrieveNodeIps() ([]string, error) { started := time.Now() slog.Debug("Translator::retrieveNodeIps") var nodeIps []string - nodes, err := t.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + nodes, err := t.nodeLister.List(labels.Everything()) if err != nil { slog.Error("error occurred retrieving the list of nodes", "error", err) return nil, err } - for _, node := range nodes.Items { + for _, node := range nodes { + if node == nil { + slog.Error("list contains nil node") + continue + } + // this is kind of a broad assumption, should probably make this a configurable option - if notMasterNode(node) { + if notMasterNode(*node) { for _, address := range node.Status.Addresses { if address.Type == v1.NodeInternalIP { nodeIps = append(nodeIps, address.Address) diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 73c99a45..c6d42f5f 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -13,11 +13,12 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" - "golang.org/x/net/context" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" ) const ( @@ -52,9 +53,12 @@ func TestCreatedTranslateNoPorts(t *testing.T) { service := defaultService(tc.serviceType) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) + translator := NewTranslator( + NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), + NewFakeNodeLister([]*v1.Node{}, nil), + ) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -86,9 +90,12 @@ func TestCreatedTranslateNoInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) + translator := NewTranslator( + NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), + NewFakeNodeLister([]*v1.Node{}, nil), + ) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -106,8 +113,8 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { t.Parallel() testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -134,8 +141,8 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -155,8 +162,8 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { t.Parallel() testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -183,8 +190,8 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -205,8 +212,8 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -234,8 +241,8 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -255,8 +262,8 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -283,8 +290,8 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -308,8 +315,8 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -333,8 +340,8 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { service := defaultService(tc.serviceType) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -352,8 +359,8 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -379,8 +386,8 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -398,8 +405,8 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -425,8 +432,8 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -447,8 +454,8 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -474,8 +481,8 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -496,8 +503,8 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -524,8 +531,8 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -546,8 +553,8 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -574,8 +581,8 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -599,8 +606,8 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -620,8 +627,8 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { service := defaultService(tc.serviceType) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -641,8 +648,8 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -663,8 +670,8 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -685,8 +692,8 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -709,8 +716,8 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -731,8 +738,8 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -754,8 +761,8 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -775,8 +782,8 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -800,8 +807,8 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -822,8 +829,8 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -845,8 +852,8 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { service := defaultService(tc.serviceType) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -866,8 +873,8 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -890,8 +897,8 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -912,8 +919,8 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -937,8 +944,8 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -959,8 +966,8 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -984,8 +991,8 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1005,8 +1012,8 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1030,8 +1037,8 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1052,8 +1059,8 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1074,8 +1081,8 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { service := defaultService(tc.serviceType) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1095,8 +1102,8 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1120,8 +1127,8 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1142,8 +1149,8 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1166,8 +1173,8 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1188,8 +1195,8 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1212,8 +1219,8 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1233,8 +1240,8 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1258,8 +1265,8 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1324,9 +1331,9 @@ func buildEvent(eventType core.EventType, service *v1.Service) core.Event { return event } -func generateNodes(count int) (nodes []v1.Node) { +func generateNodes(count int) (nodes []*v1.Node) { for i := 0; i < count; i++ { - nodes = append(nodes, v1.Node{ + nodes = append(nodes, &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("node%d", i), }, @@ -1345,7 +1352,7 @@ func generateNodes(count int) (nodes []v1.Node) { } func generateEndpointSlices(endpointCount, portCount, updatablePortCount int, -) (endpointSlices []discovery.EndpointSlice) { +) (endpointSlices []*discovery.EndpointSlice) { servicePorts := generateUpdatablePorts(portCount, updatablePortCount) ports := make([]discovery.EndpointPort, 0, len(servicePorts)) @@ -1365,7 +1372,7 @@ func generateEndpointSlices(endpointCount, portCount, updatablePortCount int, }) } - endpointSlices = append(endpointSlices, discovery.EndpointSlice{ + endpointSlices = append(endpointSlices, &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "endpointSlice", Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, @@ -1415,13 +1422,47 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort return ports } -func NewFakeClient(endpointSlices []discovery.EndpointSlice, nodes []v1.Node) *fake.Clientset { - return fake.NewSimpleClientset( - &discovery.EndpointSliceList{ - Items: endpointSlices, - }, - &v1.NodeList{ - Items: nodes, - }, - ) +func NewFakeEndpointSliceLister(list []*discovery.EndpointSlice, err error) discoverylisters.EndpointSliceLister { + return &endpointSliceLister{ + list: list, + err: err, + } +} + +func NewFakeNodeLister(list []*v1.Node, err error) corelisters.NodeLister { + return &nodeLister{ + list: list, + err: err, + } +} + +type nodeLister struct { + list []*v1.Node + err error +} + +func (l *nodeLister) List(selector labels.Selector) (ret []*v1.Node, err error) { + return l.list, l.err +} + +// currently unused +func (l *nodeLister) Get(name string) (*v1.Node, error) { + return nil, nil +} + +type endpointSliceLister struct { + list []*discovery.EndpointSlice + err error +} + +func (l *endpointSliceLister) List(selector labels.Selector) (ret []*discovery.EndpointSlice, err error) { + return l.list, l.err +} + +func (l *endpointSliceLister) Get(name string) (*discovery.EndpointSlice, error) { + return nil, nil +} + +func (l *endpointSliceLister) EndpointSlices(name string) discoverylisters.EndpointSliceNamespaceLister { + return l } From 65693ac59c76cc1eecf18b3af04142e4f6ffa848 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 19 Nov 2024 15:04:45 -0700 Subject: [PATCH 069/110] NLB-5872 If user removes nginxaas service annotation remove the service from the watcher's registe nodePort: 31575r --- internal/observation/watcher.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 21d6e028..83fc37aa 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -252,14 +252,20 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac slog.Info("Watcher::buildServiceEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API + previousService := previous.(*v1.Service) service := updated.(*v1.Service) + + if w.isDesiredService(previousService) && !w.isDesiredService(service) { + w.register.removeService(previousService) + return + } + if !w.isDesiredService(service) { return } w.register.addOrUpdateService(service) - previousService := previous.(*v1.Service) e := core.NewEvent(core.Updated, service, previousService) w.handler.AddRateLimitedEvent(&e) } From df4f3b7902079be2278d439b91c0e3cff17eb650 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 15 Nov 2024 10:04:26 -0700 Subject: [PATCH 070/110] NLB-5872 Bumped version to 0.8.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 39e898a4..a3df0a69 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.7.1 +0.8.0 From 44d8fccc12710850b6bddabb2a30e42f11873fb2 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 6 Dec 2024 16:01:27 -0800 Subject: [PATCH 071/110] Update chart,app version to 0.8.0 --- charts/nlk/Chart.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 4e1860f0..b11ab862 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,17 +1,16 @@ --- apiVersion: v2 -appVersion: 0.4.0 +appVersion: 0.8.0 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: -- nginx -- nginxaas -- loadbalancer + - nginx + - nginxaas + - loadbalancer kubeVersion: '>= 1.22.0-0' maintainers: -- name: "@ciroque" -- name: "@chrisakker" -- name: "@abdennour" - + - name: "@ciroque" + - name: "@chrisakker" + - name: "@abdennour" type: application -version: 0.4.0 +version: 0.8.0 From 70ff596f0e3d575449e9f5b5b0b821c6c2947e5c Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 9 Dec 2024 14:47:51 -0800 Subject: [PATCH 072/110] Update default logger to be info The default log-level of "warn" does not help much even when there are issues with nlk: - While talking to the API. - updating upstreams. This commit updates the default loglevel to be info so that the customer sees useful logs while debugging the component. The user can choose to turn it down if they want to but having a slightly louder log level helps with the initial experience especially when the product is new (ish). --- charts/nlk/values.yaml | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 851f67db..1f2f1f2f 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -31,7 +31,7 @@ nlk: annotations: {} config: ## trace,debug,info,warn,error,fatal,panic - # logLevel: "warn" + logLevel: "info" ## the nginx hosts (comma-separated) to send upstream updates to nginxHosts: "" diff --git a/version b/version index a3df0a69..6f4eebdf 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.0 +0.8.1 From de50615d296fd456919d8cde43301571ef647d7f Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 11 Dec 2024 08:48:04 -0700 Subject: [PATCH 073/110] NLB-5933 Remove handler; responsibilities for looking up upstream servers moved to synchronizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synchronizer now assumes the responsibility of calling the translator to find the server addresses that need to be updated and handling errors by re-adding the work item to the queue. It continues to handle errors from the border client by requeuing workitems. The synchronizer needs to use a service identifier as the key in the workqueue (service ID=service name+namespace). If a workitem is dequeued, the synchronizer looks up the service in the shared informer cache to gain the most up-to-date state. The only time we use our local cache to discover the last known state of the service is in the case of a service deletion. The shared informer cache will no longer contain data on the deleted service, so we need to rely on our own local event cache to find details of the service's ports to determine the upstream and context, etc. Any incoming events overwrite stale events in the local event cache. Because the synchronizer always refers to the shared informer’s cache to find the current service addresses when the work item is dequeued we reduce the risk of applying stale state. This change simplifies event handling: there is only one rate-limited workqueue to be configured and managed, and we don't have to worry about potentially stale state being dequeued from two separate queues. --- cmd/nginx-loadbalancer-kubernetes/main.go | 17 +- internal/observation/handler.go | 141 -------------- internal/observation/handler_test.go | 58 ------ internal/observation/watcher.go | 28 +-- internal/observation/watcher_test.go | 4 +- internal/synchronization/cache.go | 40 ++++ internal/synchronization/synchronizer.go | 179 +++++++++++------- internal/synchronization/synchronizer_test.go | 156 ++++++++++++--- test/mocks/mock_handler.go | 26 --- 9 files changed, 304 insertions(+), 345 deletions(-) delete mode 100644 internal/observation/handler.go delete mode 100644 internal/observation/handler_test.go create mode 100644 internal/synchronization/cache.go delete mode 100644 test/mocks/mock_handler.go diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index b7381f06..501d7c4f 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -49,11 +49,6 @@ func run() error { synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - synchronizer, err := synchronization.NewSynchronizer(settings, synchronizerWorkqueue) - if err != nil { - return fmt.Errorf(`error initializing synchronizer: %w`, err) - } - factory := informers.NewSharedInformerFactoryWithOptions( k8sClient, settings.Watcher.ResyncPeriod, ) @@ -64,12 +59,15 @@ func run() error { nodesInformer := factory.Core().V1().Nodes() nodesLister := nodesInformer.Lister() - handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - translator := translation.NewTranslator(endpointSliceLister, nodesLister) - handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translator) - watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer, nodesInformer) + synchronizer, err := synchronization.NewSynchronizer( + settings, synchronizerWorkqueue, translator, serviceInformer.Lister()) + if err != nil { + return fmt.Errorf(`error initializing synchronizer: %w`, err) + } + + watcher, err := observation.NewWatcher(settings, synchronizer, serviceInformer, endpointSliceInformer, nodesInformer) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } @@ -82,7 +80,6 @@ func run() error { } } - go handler.Run(ctx) go synchronizer.Run(ctx.Done()) probeServer := probation.NewHealthServer() diff --git a/internal/observation/handler.go b/internal/observation/handler.go deleted file mode 100644 index 45ae6c4e..00000000 --- a/internal/observation/handler.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package observation - -import ( - "context" - "fmt" - "log/slog" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/workqueue" -) - -// HandlerInterface is the interface for the event handler -type HandlerInterface interface { - // AddRateLimitedEvent defines the interface for adding an event to the event queue - AddRateLimitedEvent(event *core.Event) - - // Run defines the interface used to start the event handler - Run(ctx context.Context) - - // ShutDown defines the interface used to stop the event handler - ShutDown() -} - -// Handler is responsible for processing events in the "nlk-handler" queue. -// When processing a message the Translation module is used to translate the event into an internal representation. -// The translation process may result in multiple events being generated. This fan-out mainly supports the differences -// in NGINX Plus API calls for creating/updating Upstreams and deleting Upstreams. -type Handler struct { - // eventQueue is the queue used to store events - eventQueue workqueue.RateLimitingInterface - - // settings is the configuration settings - settings configuration.Settings - - // synchronizer is the synchronizer used to synchronize the internal representation with a Border Server - synchronizer synchronization.Interface - - translator Translator -} - -type Translator interface { - Translate(*core.Event) (core.ServerUpdateEvents, error) -} - -// NewHandler creates a new event handler -func NewHandler( - settings configuration.Settings, - synchronizer synchronization.Interface, - eventQueue workqueue.RateLimitingInterface, - translator Translator, -) *Handler { - return &Handler{ - eventQueue: eventQueue, - settings: settings, - synchronizer: synchronizer, - translator: translator, - } -} - -// AddRateLimitedEvent adds an event to the event queue -func (h *Handler) AddRateLimitedEvent(event *core.Event) { - slog.Debug(`Handler::AddRateLimitedEvent`, "event", event) - h.eventQueue.AddRateLimited(event) -} - -// Run starts the event handler, spins up Goroutines to process events, and waits for context to be done -func (h *Handler) Run(ctx context.Context) { - slog.Debug("Handler::Run") - - worker := func() { - for h.handleNextEvent() { - // TODO: Add Telemetry - } - } - - for i := 0; i < h.settings.Handler.Threads; i++ { - go wait.Until(worker, 0, ctx.Done()) - } - - <-ctx.Done() -} - -// ShutDown stops the event handler and shuts down the event queue -func (h *Handler) ShutDown() { - slog.Debug("Handler::ShutDown") - h.eventQueue.ShutDown() -} - -// handleEvent feeds translated events to the synchronizer -func (h *Handler) handleEvent(e *core.Event) error { - slog.Debug("Handler::handleEvent", "event", e) - // TODO: Add Telemetry - - events, err := h.translator.Translate(e) - if err != nil { - return fmt.Errorf(`Handler::handleEvent error translating: %v`, err) - } - - h.synchronizer.AddEvents(events) - - return nil -} - -// handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (h *Handler) handleNextEvent() bool { - evt, quit := h.eventQueue.Get() - slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) - if quit { - return false - } - - defer h.eventQueue.Done(evt) - - event := evt.(*core.Event) - h.withRetry(h.handleEvent(event), event) - - return true -} - -// withRetry handles errors from the event handler and requeues events that fail -func (h *Handler) withRetry(err error, event *core.Event) { - slog.Debug("Handler::withRetry") - if err != nil { - // TODO: Add Telemetry - if h.eventQueue.NumRequeues(event) < h.settings.Handler.RetryCount { - h.eventQueue.AddRateLimited(event) - slog.Info("Handler::withRetry: requeued event", "event", event, "error", err) - } else { - h.eventQueue.Forget(event) - slog.Warn(`Handler::withRetry: event has been dropped due to too many retries`, "event", event) - } - } // TODO: Add error logging -} diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go deleted file mode 100644 index 550b318c..00000000 --- a/internal/observation/handler_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package observation - -import ( - "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - v1 "k8s.io/api/core/v1" -) - -func TestHandler_AddsEventToSynchronizer(t *testing.T) { - t.Parallel() - synchronizer, handler := buildHandler() - - event := &core.Event{ - Type: core.Created, - Service: &v1.Service{ - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "http-back", - }, - }, - }, - }, - } - - handler.AddRateLimitedEvent(event) - - handler.handleNextEvent() - - if len(synchronizer.Events) != 1 { - t.Errorf(`handler.AddRateLimitedEvent did not add the event to the queue`) - } -} - -func buildHandler() ( - *mocks.MockSynchronizer, *Handler, -) { - eventQueue := &mocks.MockRateLimiter{} - synchronizer := &mocks.MockSynchronizer{} - - handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue, &fakeTranslator{}) - - return synchronizer, handler -} - -type fakeTranslator struct{} - -func (t *fakeTranslator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { - return core.ServerUpdateEvents{{}}, nil -} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 83fc37aa..6c8977ef 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -12,6 +12,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -24,8 +25,7 @@ import ( // Particularly, Services in the namespace defined in the WatcherSettings::NginxIngressNamespace setting. // When a change is detected, an Event is generated and added to the Handler's queue. type Watcher struct { - // handler is the event handler - handler HandlerInterface + synchronizer synchronization.Interface // settings is the configuration settings settings configuration.Settings @@ -45,7 +45,7 @@ type Watcher struct { // NewWatcher creates a new Watcher func NewWatcher( settings configuration.Settings, - handler HandlerInterface, + synchronizer synchronization.Interface, serviceInformer coreinformers.ServiceInformer, endpointSliceInformer discoveryinformers.EndpointSliceInformer, nodeInformer coreinformers.NodeInformer, @@ -67,7 +67,7 @@ func NewWatcher( nodesInformer := nodeInformer.Informer() w := &Watcher{ - handler: handler, + synchronizer: synchronizer, settings: settings, servicesInformer: servicesInformer, endpointSliceInformer: endpointSlicesInformer, @@ -92,7 +92,7 @@ func (w *Watcher) Run(ctx context.Context) error { slog.Debug("Watcher::Watch") defer utilruntime.HandleCrash() - defer w.handler.ShutDown() + defer w.synchronizer.ShutDown() <-ctx.Done() return nil @@ -115,7 +115,7 @@ func (w *Watcher) buildNodesEventHandlerForAdd() func(interface{}) { for _, service := range w.register.listServices() { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } } @@ -127,7 +127,7 @@ func (w *Watcher) buildNodesEventHandlerForUpdate() func(interface{}, interface{ for _, service := range w.register.listServices() { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } } @@ -139,7 +139,7 @@ func (w *Watcher) buildNodesEventHandlerForDelete() func(interface{}) { for _, service := range w.register.listServices() { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } } @@ -162,7 +162,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -184,7 +184,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, i var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -206,7 +206,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -224,7 +224,7 @@ func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Created, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -242,7 +242,7 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -267,7 +267,7 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac w.register.addOrUpdateService(service) e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 46184089..b8e8369a 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" "github.com/stretchr/testify/require" ) @@ -20,6 +19,5 @@ func TestWatcher_ErrWithNilInformers(t *testing.T) { } func buildWatcherWithNilInformer() (*Watcher, error) { - handler := &mocks.MockHandler{} - return NewWatcher(configuration.Settings{}, handler, nil, nil, nil) + return NewWatcher(configuration.Settings{}, nil, nil, nil, nil) } diff --git a/internal/synchronization/cache.go b/internal/synchronization/cache.go new file mode 100644 index 00000000..6c3f1644 --- /dev/null +++ b/internal/synchronization/cache.go @@ -0,0 +1,40 @@ +package synchronization + +import ( + "sync" + + v1 "k8s.io/api/core/v1" +) + +// cache contains the most recent definitions for services monitored by NLK. +// We need these so that if a service is deleted from the shared informer cache, the +// caller can access the spec of the deleted service for cleanup. +type cache struct { + mu sync.RWMutex + store map[ServiceKey]*v1.Service +} + +func newCache() *cache { + return &cache{ + store: make(map[ServiceKey]*v1.Service), + } +} + +func (s *cache) get(key ServiceKey) (*v1.Service, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + service, ok := s.store[key] + return service, ok +} + +func (s *cache) add(key ServiceKey, service *v1.Service) { + s.mu.Lock() + defer s.mu.Unlock() + s.store[key] = service +} + +func (s *cache) delete(key ServiceKey) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.store, key) +} diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 20bc99a7..fef97245 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -6,6 +6,7 @@ package synchronization import ( + "errors" "fmt" "log/slog" @@ -14,17 +15,16 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" + corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/util/workqueue" ) // Interface defines the interface needed to implement a synchronizer. type Interface interface { - // AddEvents adds a list of events to the queue. - AddEvents(events core.ServerUpdateEvents) - // AddEvent adds an event to the queue. - AddEvent(event *core.ServerUpdateEvent) + AddEvent(event core.Event) // Run starts the synchronizer. Run(stopCh <-chan struct{}) @@ -33,60 +33,57 @@ type Interface interface { ShutDown() } +type Translator interface { + Translate(*core.Event) (core.ServerUpdateEvents, error) +} + +type ServiceKey struct { + Name string + Namespace string +} + // Synchronizer is responsible for synchronizing the state of the Border Servers. // Operating against the "nlk-synchronizer", it handles events by creating // a Border Client as specified in the Service annotation for the Upstream. // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { - eventQueue workqueue.RateLimitingInterface - settings configuration.Settings + eventQueue workqueue.RateLimitingInterface + settings configuration.Settings + translator Translator + cache *cache + serviceLister corelisters.ServiceLister } // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( settings configuration.Settings, eventQueue workqueue.RateLimitingInterface, + translator Translator, + serviceLister corelisters.ServiceLister, ) (*Synchronizer, error) { synchronizer := Synchronizer{ - eventQueue: eventQueue, - settings: settings, + eventQueue: eventQueue, + settings: settings, + cache: newCache(), + translator: translator, + serviceLister: serviceLister, } return &synchronizer, nil } -// AddEvents adds a list of events to the queue. If no hosts are specified this is a null operation. -// Events will fan out to the number of hosts specified before being added to the queue. -func (s *Synchronizer) AddEvents(events core.ServerUpdateEvents) { - slog.Debug(`Synchronizer::AddEvents adding events`, slog.Int("eventCount", len(events))) +// AddEvent adds an event to the rate-limited queue. If no hosts are specified this is a null operation. +func (s *Synchronizer) AddEvent(event core.Event) { + slog.Debug(`Synchronizer::AddEvent`) if len(s.settings.NginxPlusHosts) == 0 { slog.Warn(`No Nginx Plus hosts were specified. Skipping synchronization.`) return } - updatedEvents := s.fanOutEventToHosts(events) - - for _, event := range updatedEvents { - s.AddEvent(event) - } -} - -// AddEvent adds an event to the queue. If no hosts are specified this is a null operation. -// Events will be added to the queue after a random delay between MinMillisecondsJitter and MaxMillisecondsJitter. -func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { - slog.Debug(`Synchronizer::AddEvent`, "event", event) - - if event.NginxHost == `` { - slog.Warn(`Nginx host was not specified. Skipping synchronization.`) - return - } - - after := RandomMilliseconds( - s.settings.Synchronizer.MinMillisecondsJitter, - s.settings.Synchronizer.MaxMillisecondsJitter, - ) - s.eventQueue.AddAfter(event, after) + key := ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace} + s.cache.add(key, event.Service) + s.eventQueue.AddRateLimited(key) } // Run starts the Synchronizer, spins up Goroutines to process events, and waits for a stop signal. @@ -129,7 +126,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica // fanOutEventToHosts takes a list of events and returns a list of events, one for each Border Server. func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.ServerUpdateEvents { - slog.Debug(`Synchronizer::fanOutEventToHosts`, "event", event) + slog.Debug(`Synchronizer::fanOutEventToHosts`) var events core.ServerUpdateEvents @@ -145,33 +142,77 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se return events } -// handleEvent dispatches an event to the proper handler function. -func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { - slog.Debug(`Synchronizer::handleEvent`, slog.String("eventID", event.ID)) - - var err error +// handleServiceEvent gets the latest state for the service from the shared +// informer cache, translates the service event into server update events and +// dispatches these events to the proper handler function. +func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { + logger := slog.With("service", key) + logger.Debug(`Synchronizer::handleServiceEvent`) + + // if a service exists in the shared informer cache, we can assume that we need to update it + event := core.Event{Type: core.Updated} + + namespaceLister := s.serviceLister.Services(key.Namespace) + k8sService, err := namespaceLister.Get(key.Name) + switch { + // the service has been deleted. We need to rely on the local cache to + // gather the last known state of the service so we can delete its + // upstream servers + case err != nil && apierrors.IsNotFound(err): + service, ok := s.cache.get(key) + if !ok { + logger.Warn(`Synchronizer::handleServiceEvent: no information could be gained about service`) + return nil + } + // no matter what type the cached event has, the service no longer exists, so the type is Deleted + event.Type = core.Deleted + event.Service = service + case err != nil: + return err + default: + event.Service = k8sService + } - switch event.Type { - case core.Created: - fallthrough + events, err := s.translator.Translate(&event) + if err != nil { + return err + } - case core.Updated: - err = s.handleCreatedUpdatedEvent(event) + if len(events) == 0 { + slog.Warn("Synchronizer::handleServiceEvent: no events to process") + return nil + } - case core.Deleted: - err = s.handleDeletedEvent(event) + events = s.fanOutEventToHosts(events) + + for _, evt := range events { + switch event.Type { + case core.Created, core.Updated: + if handleErr := s.handleCreatedUpdatedEvent(evt); handleErr != nil { + err = errors.Join(err, handleErr) + } + case core.Deleted: + if handleErr := s.handleDeletedEvent(evt); handleErr != nil { + err = errors.Join(err, handleErr) + } + default: + slog.Warn(`Synchronizer::handleServiceEvent: unknown event type`, "type", event.Type) + } + } - default: - slog.Warn(`Synchronizer::handleEvent: unknown event type`, "type", event.Type) + if err != nil { + return err } - if err == nil { - slog.Info( - "Synchronizer::handleEvent: successfully handled the event", - "type", event.TypeName(), "upstreamName", event.UpstreamName, "eventID", event.ID) + if event.Type == core.Deleted { + s.cache.delete(ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace}) } - return err + slog.Debug( + "Synchronizer::handleServiceEvent: successfully handled the service change", "service", key, + ) + + return nil } // handleCreatedUpdatedEvent handles events of type Created or Updated. @@ -210,19 +251,25 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv return nil } -// handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (s *Synchronizer) handleNextEvent() bool { - slog.Debug(`Synchronizer::handleNextEvent`) +// handleNextServiceEvent pulls a service from the event queue and feeds it to +// the service event handler with retry logic +func (s *Synchronizer) handleNextServiceEvent() bool { + slog.Debug(`Synchronizer::handleNextServiceEvent`) - evt, quit := s.eventQueue.Get() + svc, quit := s.eventQueue.Get() if quit { return false } - defer s.eventQueue.Done(evt) + defer s.eventQueue.Done(svc) + + key, ok := svc.(ServiceKey) + if !ok { + slog.Warn(`Synchronizer::handleNextServiceEvent: invalid service type`, "service", svc) + return true + } - event := evt.(*core.ServerUpdateEvent) - s.withRetry(s.handleEvent(event), event) + s.withRetry(s.handleServiceEvent(key), key) return true } @@ -230,18 +277,18 @@ func (s *Synchronizer) handleNextEvent() bool { // worker is the main message loop func (s *Synchronizer) worker() { slog.Debug(`Synchronizer::worker`) - for s.handleNextEvent() { + for s.handleNextServiceEvent() { } } // withRetry handles errors from the event handler and requeues events that fail -func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { +func (s *Synchronizer) withRetry(err error, key ServiceKey) { slog.Debug("Synchronizer::withRetry") if err != nil { // TODO: Add Telemetry - s.eventQueue.AddRateLimited(event) - slog.Info(`Synchronizer::withRetry: requeued event`, "eventID", event.ID, "error", err) + s.eventQueue.AddRateLimited(key) + slog.Info(`Synchronizer::withRetry: requeued service update`, "service", key, "error", err) } else { - s.eventQueue.Forget(event) + s.eventQueue.Forget(key) } // TODO: Add error logging } diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index d1710b22..7592f5ca 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -13,6 +13,10 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" ) func TestSynchronizer_NewSynchronizer(t *testing.T) { @@ -20,7 +24,12 @@ func TestSynchronizer_NewSynchronizer(t *testing.T) { rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(configuration.Settings{}, rateLimiter) + synchronizer, err := NewSynchronizer( + configuration.Settings{}, + rateLimiter, + &fakeTranslator{}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -33,17 +42,15 @@ func TestSynchronizer_NewSynchronizer(t *testing.T) { func TestSynchronizer_AddEventNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 - event := &core.ServerUpdateEvent{ - ID: "", - NginxHost: "", - Type: 0, - UpstreamName: "", - UpstreamServers: nil, - } rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(), + rateLimiter, + &fakeTranslator{}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -54,7 +61,7 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { // NOTE: Ideally we have a custom logger that can be mocked to capture the log message // and assert a warning was logged that the NGINX Plus host was not specified. - synchronizer.AddEvent(event) + synchronizer.AddEvent(core.Event{}) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -64,11 +71,16 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { func TestSynchronizer_AddEventOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 1 - events := buildEvents(1) + events := buildServerUpdateEvents(1) rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings("https://localhost:8080"), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -77,7 +89,7 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvent(events[0]) + synchronizer.AddEvent(buildServiceUpdateEvent(1)) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -87,7 +99,7 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { func TestSynchronizer_AddEventManyHosts(t *testing.T) { t.Parallel() const expectedEventCount = 1 - events := buildEvents(1) + events := buildServerUpdateEvents(1) hosts := []string{ "https://localhost:8080", "https://localhost:8081", @@ -96,7 +108,12 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(hosts...), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -105,7 +122,7 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvent(events[0]) + synchronizer.AddEvent(buildServiceUpdateEvent(1)) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -115,10 +132,15 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { func TestSynchronizer_AddEventsNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 - events := buildEvents(4) + events := buildServerUpdateEvents(4) rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -129,7 +151,10 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { // NOTE: Ideally we have a custom logger that can be mocked to capture the log message // and assert a warning was logged that the NGINX Plus host was not specified. - synchronizer.AddEvents(events) + for i := 0; i < 4; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -139,10 +164,15 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 4 - events := buildEvents(4) + events := buildServerUpdateEvents(1) rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings("https://localhost:8080"), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -151,7 +181,10 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvents(events) + for i := 0; i < 4; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -161,7 +194,7 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { func TestSynchronizer_AddEventsManyHosts(t *testing.T) { t.Parallel() const eventCount = 4 - events := buildEvents(eventCount) + events := buildServerUpdateEvents(eventCount) rateLimiter := &mocks.MockRateLimiter{} hosts := []string{ @@ -170,9 +203,14 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { "https://localhost:8082", } - expectedEventCount := eventCount * len(hosts) + expectedEventCount := 4 - synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(hosts...), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -181,14 +219,28 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvents(events) + for i := 0; i < eventCount; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) } } -func buildEvents(count int) core.ServerUpdateEvents { +func buildServiceUpdateEvent(serviceID int) core.Event { + return core.Event{ + Service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-service%d", serviceID), + Namespace: "test-namespace", + }, + }, + } +} + +func buildServerUpdateEvents(count int) core.ServerUpdateEvents { events := make(core.ServerUpdateEvents, count) for i := 0; i < count; i++ { events[i] = &core.ServerUpdateEvent{ @@ -218,3 +270,53 @@ func defaultSettings(nginxHosts ...string) configuration.Settings { }, } } + +type fakeTranslator struct { + events core.ServerUpdateEvents + err error +} + +func (t *fakeTranslator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { + return t.events, t.err +} + +func newFakeServicesLister(list ...*v1.Service) corelisters.ServiceLister { + return &servicesLister{ + list: list, + } +} + +type servicesLister struct { + list []*v1.Service + err error +} + +func (l *servicesLister) List(selector labels.Selector) (ret []*v1.Service, err error) { + return l.list, l.err +} + +func (l *servicesLister) Get(name string) (*v1.Service, error) { + for _, service := range l.list { + if service.Name == name { + return service, nil + } + } + + return nil, nil +} + +func (l *servicesLister) Services(name string) corelisters.ServiceNamespaceLister { + return l +} + +func defaultService() *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-service", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + }, + } +} diff --git a/test/mocks/mock_handler.go b/test/mocks/mock_handler.go deleted file mode 100644 index f144cea0..00000000 --- a/test/mocks/mock_handler.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package mocks - -import ( - "context" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -type MockHandler struct{} - -func (h *MockHandler) AddRateLimitedEvent(_ *core.Event) { -} - -func (h *MockHandler) Initialize() { -} - -func (h *MockHandler) Run(ctx context.Context) { -} - -func (h *MockHandler) ShutDown() { -} From 915724b6b35fe28dd2af0b6cbbe1a58cce4a6b15 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 11 Dec 2024 10:31:48 -0700 Subject: [PATCH 074/110] NLB-5933 Added error group to the main routine Even though this is not really modifying the application's behavior right now (the watcher and synchronizer were only returning when the context was cancelled) it is a nicer framework for handling the errors of modules launched in separate goroutines if we do choose to make any of the NLK modules return errors. --- cmd/nginx-loadbalancer-kubernetes/main.go | 14 +++++++------- go.mod | 1 + go.sum | 2 ++ internal/synchronization/synchronizer.go | 10 ++++++---- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 501d7c4f..5aba57f1 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -17,6 +17,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" + "golang.org/x/sync/errgroup" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -80,18 +81,17 @@ func run() error { } } - go synchronizer.Run(ctx.Done()) + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { return synchronizer.Run(ctx) }) probeServer := probation.NewHealthServer() probeServer.Start() - err = watcher.Run(ctx) - if err != nil { - return fmt.Errorf(`error occurred running watcher: %w`, err) - } + g.Go(func() error { return watcher.Run(ctx) }) - <-ctx.Done() - return nil + err = g.Wait() + return err } func initializeLogger(logLevel string) { diff --git a/go.mod b/go.mod index 94a13195..cc4fe63e 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + golang.org/x/sync v0.6.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 diff --git a/go.sum b/go.sum index 1433805d..bd58f373 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index fef97245..0d70215b 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -6,6 +6,7 @@ package synchronization import ( + "context" "errors" "fmt" "log/slog" @@ -27,7 +28,7 @@ type Interface interface { AddEvent(event core.Event) // Run starts the synchronizer. - Run(stopCh <-chan struct{}) + Run(ctx context.Context) error // ShutDown shuts down the synchronizer. ShutDown() @@ -87,14 +88,15 @@ func (s *Synchronizer) AddEvent(event core.Event) { } // Run starts the Synchronizer, spins up Goroutines to process events, and waits for a stop signal. -func (s *Synchronizer) Run(stopCh <-chan struct{}) { +func (s *Synchronizer) Run(ctx context.Context) error { slog.Debug(`Synchronizer::Run`) for i := 0; i < s.settings.Synchronizer.Threads; i++ { - go wait.Until(s.worker, 0, stopCh) + go wait.Until(s.worker, 0, ctx.Done()) } - <-stopCh + <-ctx.Done() + return nil } // ShutDown stops the Synchronizer and shuts down the event queue From 7ffb943bfe73b42f50b3aa92b155a16b80acf2f5 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 17 Dec 2024 16:24:18 -0700 Subject: [PATCH 075/110] NLB-5933 Remove PreviousService from core event This field was not being used anywhere, so it seems better to get rid of it and avoid the hassle of keeping it accurate. --- internal/core/event.go | 10 +++------- internal/core/event_test.go | 7 +------ internal/observation/watcher.go | 26 +++++++++---------------- internal/translation/translator_test.go | 4 +--- 4 files changed, 14 insertions(+), 33 deletions(-) diff --git a/internal/core/event.go b/internal/core/event.go index 16d5d946..c32511e1 100644 --- a/internal/core/event.go +++ b/internal/core/event.go @@ -29,16 +29,12 @@ type Event struct { // Service represents the service object in its current state Service *v1.Service - - // PreviousService represents the service object in its previous state - PreviousService *v1.Service } // NewEvent factory method to create a new Event -func NewEvent(eventType EventType, service *v1.Service, previousService *v1.Service) Event { +func NewEvent(eventType EventType, service *v1.Service) Event { return Event{ - Type: eventType, - Service: service, - PreviousService: previousService, + Type: eventType, + Service: service, } } diff --git a/internal/core/event_test.go b/internal/core/event_test.go index f0184fbe..09724cfa 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -10,9 +10,8 @@ func TestNewEvent(t *testing.T) { t.Parallel() expectedType := Created expectedService := &v1.Service{} - expectedPreviousService := &v1.Service{} - event := NewEvent(expectedType, expectedService, expectedPreviousService) + event := NewEvent(expectedType, expectedService) if event.Type != expectedType { t.Errorf("expected Created, got %v", event.Type) @@ -21,8 +20,4 @@ func TestNewEvent(t *testing.T) { if event.Service != expectedService { t.Errorf("expected service, got %#v", event.Service) } - - if event.PreviousService != expectedPreviousService { - t.Errorf("expected previous service, got %#v", event.PreviousService) - } } diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 6c8977ef..0df49fe1 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -113,8 +113,7 @@ func (w *Watcher) buildNodesEventHandlerForAdd() func(interface{}) { return func(obj interface{}) { slog.Debug("received node add event") for _, service := range w.register.listServices() { - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -125,8 +124,7 @@ func (w *Watcher) buildNodesEventHandlerForUpdate() func(interface{}, interface{ return func(previous, updated interface{}) { slog.Debug("received node update event") for _, service := range w.register.listServices() { - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -137,8 +135,7 @@ func (w *Watcher) buildNodesEventHandlerForDelete() func(interface{}) { return func(obj interface{}) { slog.Debug("received node delete event") for _, service := range w.register.listServices() { - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -160,8 +157,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { return } - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -182,8 +178,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, i return } - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -204,8 +199,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { return } - var previousService *v1.Service - e := core.NewEvent(core.Deleted, service, previousService) + e := core.NewEvent(core.Deleted, service) w.synchronizer.AddEvent(e) } } @@ -222,8 +216,7 @@ func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { w.register.addOrUpdateService(service) - var previousService *v1.Service - e := core.NewEvent(core.Created, service, previousService) + e := core.NewEvent(core.Created, service) w.synchronizer.AddEvent(e) } } @@ -240,8 +233,7 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { w.register.removeService(service) - var previousService *v1.Service - e := core.NewEvent(core.Deleted, service, previousService) + e := core.NewEvent(core.Deleted, service) w.synchronizer.AddEvent(e) } } @@ -266,7 +258,7 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac w.register.addOrUpdateService(service) - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index c6d42f5f..11fa22ce 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -1324,9 +1324,7 @@ func buildUpdatedEvent(service *v1.Service) core.Event { } func buildEvent(eventType core.EventType, service *v1.Service) core.Event { - previousService := defaultService(service.Spec.Type) - - event := core.NewEvent(eventType, service, previousService) + event := core.NewEvent(eventType, service) event.Service.Name = "default-service" return event } From f29ab7e3aeb89ffe1a79fa475870897cbadef47d Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 19 Dec 2024 10:32:29 -0700 Subject: [PATCH 076/110] NLB-5933 Handle 404 errors when deleting upstream servers If a user removes a upstream server from their nginx configuration and then deletes the service associated with that upstream from their AKS cluster, we should handle the resulting 404s on the upstream servers and forget about the workitem, rather than retrying the updates indefinitely. Checking the error for a substring is not ideal as a way of handling the error, but the go plus client gives us no option here. --- internal/synchronization/synchronizer.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 0d70215b..63b032a4 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" @@ -246,11 +247,17 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv return fmt.Errorf(`error occurred creating the border client: %w`, err) } - if err = borderClient.Delete(serverUpdateEvent); err != nil { + err = borderClient.Update(serverUpdateEvent) + + switch { + case err == nil: + return nil + // checking the string is not ideal, but the plus client gives us no option + case strings.Contains(err.Error(), "status=404"): + return nil + default: return fmt.Errorf(`error occurred deleting the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) } - - return nil } // handleNextServiceEvent pulls a service from the event queue and feeds it to From 88a5b0e429fd6cabf80f72d95ba9bd5d1a14ebac Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 19 Dec 2024 14:58:53 -0700 Subject: [PATCH 077/110] NLB-5933 Bumped version to 0.8.2 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 6f4eebdf..100435be 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.1 +0.8.2 From 5e0228d2abbf2f8a13a2a601cd3baaebcbcdf221 Mon Sep 17 00:00:00 2001 From: sarna Date: Sat, 4 Jan 2025 18:22:29 +0530 Subject: [PATCH 078/110] Upgrade the indirect net dep The golang.org/x/net package has a high CVE. --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index cc4fe63e..502888d7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.10.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 @@ -55,11 +55,11 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index bd58f373..26b1b01c 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= @@ -180,8 +180,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -190,18 +190,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 070af1e45e56203b386ca6bfd567006678540d0f Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 9 Jan 2025 13:27:56 -0700 Subject: [PATCH 079/110] Upgrade nginx-plus-go-client to v2.2.0 This update brings changes that cause the client to apply all upstream server updates and not to return on first error. The upgrade necessitates changing the border client interfaces to match the new method signatures of the plus go client. As a result, I've threaded the context through various methods which did not require it before. --- go.mod | 6 ++-- go.sum | 4 +-- internal/application/border_client.go | 5 +-- .../application/nginx_client_interface.go | 14 ++++++-- .../application/nginx_http_border_client.go | 11 +++--- .../nginx_http_border_client_test.go | 9 ++--- .../application/nginx_stream_border_client.go | 11 +++--- .../nginx_stream_border_client_test.go | 9 ++--- internal/application/null_border_client.go | 5 +-- .../application/null_border_client_test.go | 9 +++-- internal/synchronization/synchronizer.go | 36 +++++++++---------- test/mocks/mock_nginx_plus_client.go | 12 +++++-- 12 files changed, 77 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 502888d7..512dc364 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.21.2 +go 1.22.6 -toolchain go1.21.4 +toolchain go1.23.4 require ( - github.com/nginxinc/nginx-plus-go-client v1.2.2 + github.com/nginx/nginx-plus-go-client/v2 v2.2.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 golang.org/x/sync v0.10.0 diff --git a/go.sum b/go.sum index 26b1b01c..cae00cea 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginxinc/nginx-plus-go-client v1.2.2 h1:sl7HqNDDZq2EVu0eQQVoZ6PKYGa4h2dB/Qr5Ib0YKGw= -github.com/nginxinc/nginx-plus-go-client v1.2.2/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4= +github.com/nginx/nginx-plus-go-client/v2 v2.2.0 h1:qwhx4fF/pq+h72/nE+o+XSH5mZmDU/R8fwim6VcZ8cM= +github.com/nginx/nginx-plus-go-client/v2 v2.2.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= diff --git a/internal/application/border_client.go b/internal/application/border_client.go index e4eded8b..8bca843b 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -6,6 +6,7 @@ package application import ( + "context" "fmt" "log/slog" @@ -14,8 +15,8 @@ import ( // Interface defines the functions required to implement a Border Client. type Interface interface { - Update(*core.ServerUpdateEvent) error - Delete(*core.ServerUpdateEvent) error + Update(context.Context, *core.ServerUpdateEvent) error + Delete(context.Context, *core.ServerUpdateEvent) error } // BorderClient defines any state need by the Border Client. diff --git a/internal/application/nginx_client_interface.go b/internal/application/nginx_client_interface.go index 31a7b946..1a60c5ee 100644 --- a/internal/application/nginx_client_interface.go +++ b/internal/application/nginx_client_interface.go @@ -5,25 +5,33 @@ package application -import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" +import ( + "context" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" +) + +var _ NginxClientInterface = (*nginxClient.NginxClient)(nil) // NginxClientInterface defines the functions used on the NGINX Plus client, // abstracting away the full details of that client. type NginxClientInterface interface { // DeleteStreamServer is used by the NginxStreamBorderClient. - DeleteStreamServer(upstream string, server string) error + DeleteStreamServer(ctx context.Context, upstream string, server string) error // UpdateStreamServers is used by the NginxStreamBorderClient. UpdateStreamServers( + ctx context.Context, upstream string, servers []nginxClient.StreamUpstreamServer, ) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) // DeleteHTTPServer is used by the NginxHTTPBorderClient. - DeleteHTTPServer(upstream string, server string) error + DeleteHTTPServer(ctx context.Context, upstream string, server string) error // UpdateHTTPServers is used by the NginxHTTPBorderClient. UpdateHTTPServers( + ctx context.Context, upstream string, servers []nginxClient.UpstreamServer, ) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index e9e754a6..4de147e5 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -7,10 +7,11 @@ package application import ( + "context" "fmt" + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) // NginxHttpBorderClient implements the BorderClient interface for HTTP upstreams. @@ -32,9 +33,9 @@ func NewNginxHTTPBorderClient(client interface{}) (Interface, error) { } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHTTPBorderClient) Update(event *core.ServerUpdateEvent) error { +func (hbc *NginxHTTPBorderClient) Update(ctx context.Context, event *core.ServerUpdateEvent) error { httpUpstreamServers := asNginxHTTPUpstreamServers(event.UpstreamServers) - _, _, _, err := hbc.nginxClient.UpdateHTTPServers(event.UpstreamName, httpUpstreamServers) + _, _, _, err := hbc.nginxClient.UpdateHTTPServers(ctx, event.UpstreamName, httpUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) } @@ -43,8 +44,8 @@ func (hbc *NginxHTTPBorderClient) Update(event *core.ServerUpdateEvent) error { } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHTTPBorderClient) Delete(event *core.ServerUpdateEvent) error { - err := hbc.nginxClient.DeleteHTTPServer(event.UpstreamName, event.UpstreamServers[0].Host) +func (hbc *NginxHTTPBorderClient) Delete(ctx context.Context, event *core.ServerUpdateEvent) error { + err := hbc.nginxClient.DeleteHTTPServer(ctx, event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) } diff --git a/internal/application/nginx_http_border_client_test.go b/internal/application/nginx_http_border_client_test.go index 039b4ecd..7a972068 100644 --- a/internal/application/nginx_http_border_client_test.go +++ b/internal/application/nginx_http_border_client_test.go @@ -9,6 +9,7 @@ package application import ( + "context" "testing" ) @@ -20,7 +21,7 @@ func TestHttpBorderClient_Delete(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -38,7 +39,7 @@ func TestHttpBorderClient_Update(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -65,7 +66,7 @@ func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) @@ -80,7 +81,7 @@ func TestHttpBorderClient_UpdateReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index 19982b45..238a22b6 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -7,10 +7,11 @@ package application import ( + "context" "fmt" + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) // NginxStreamBorderClient implements the BorderClient interface for stream upstreams. @@ -32,9 +33,9 @@ func NewNginxStreamBorderClient(client interface{}) (Interface, error) { } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (tbc *NginxStreamBorderClient) Update(event *core.ServerUpdateEvent) error { +func (tbc *NginxStreamBorderClient) Update(ctx context.Context, event *core.ServerUpdateEvent) error { streamUpstreamServers := asNginxStreamUpstreamServers(event.UpstreamServers) - _, _, _, err := tbc.nginxClient.UpdateStreamServers(event.UpstreamName, streamUpstreamServers) + _, _, _, err := tbc.nginxClient.UpdateStreamServers(ctx, event.UpstreamName, streamUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) } @@ -43,8 +44,8 @@ func (tbc *NginxStreamBorderClient) Update(event *core.ServerUpdateEvent) error } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (tbc *NginxStreamBorderClient) Delete(event *core.ServerUpdateEvent) error { - err := tbc.nginxClient.DeleteStreamServer(event.UpstreamName, event.UpstreamServers[0].Host) +func (tbc *NginxStreamBorderClient) Delete(ctx context.Context, event *core.ServerUpdateEvent) error { + err := tbc.nginxClient.DeleteStreamServer(ctx, event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) } diff --git a/internal/application/nginx_stream_border_client_test.go b/internal/application/nginx_stream_border_client_test.go index c86a7767..cf4d302d 100644 --- a/internal/application/nginx_stream_border_client_test.go +++ b/internal/application/nginx_stream_border_client_test.go @@ -7,6 +7,7 @@ package application import ( + "context" "testing" ) @@ -18,7 +19,7 @@ func TestTcpBorderClient_Delete(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -36,7 +37,7 @@ func TestTcpBorderClient_Update(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -63,7 +64,7 @@ func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) @@ -78,7 +79,7 @@ func TestTcpBorderClient_UpdateReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) diff --git a/internal/application/null_border_client.go b/internal/application/null_border_client.go index 295ca62f..dc4467d1 100644 --- a/internal/application/null_border_client.go +++ b/internal/application/null_border_client.go @@ -6,6 +6,7 @@ package application import ( + "context" "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" @@ -22,13 +23,13 @@ func NewNullBorderClient() (Interface, error) { } // Update logs a Warning. It is, after all, a NullObject Pattern implementation. -func (nbc *NullBorderClient) Update(_ *core.ServerUpdateEvent) error { +func (nbc *NullBorderClient) Update(_ context.Context, _ *core.ServerUpdateEvent) error { slog.Warn("NullBorderClient.Update called") return nil } // Delete logs a Warning. It is, after all, a NullObject Pattern implementation. -func (nbc *NullBorderClient) Delete(_ *core.ServerUpdateEvent) error { +func (nbc *NullBorderClient) Delete(_ context.Context, _ *core.ServerUpdateEvent) error { slog.Warn("NullBorderClient.Delete called") return nil } diff --git a/internal/application/null_border_client_test.go b/internal/application/null_border_client_test.go index f973949f..01d9fe23 100644 --- a/internal/application/null_border_client_test.go +++ b/internal/application/null_border_client_test.go @@ -5,12 +5,15 @@ package application -import "testing" +import ( + "context" + "testing" +) func TestNullBorderClient_Delete(t *testing.T) { t.Parallel() client := NullBorderClient{} - err := client.Delete(nil) + err := client.Delete(context.Background(), nil) if err != nil { t.Errorf(`expected no error deleting border client, got: %v`, err) } @@ -19,7 +22,7 @@ func TestNullBorderClient_Delete(t *testing.T) { func TestNullBorderClient_Update(t *testing.T) { t.Parallel() client := NullBorderClient{} - err := client.Update(nil) + err := client.Update(context.Background(), nil) if err != nil { t.Errorf(`expected no error updating border client, got: %v`, err) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 63b032a4..d45277b8 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -12,11 +12,11 @@ import ( "log/slog" "strings" + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" corelisters "k8s.io/client-go/listers/core/v1" @@ -92,8 +92,15 @@ func (s *Synchronizer) AddEvent(event core.Event) { func (s *Synchronizer) Run(ctx context.Context) error { slog.Debug(`Synchronizer::Run`) + // worker is the main message loop + worker := func() { + slog.Debug(`Synchronizer::worker`) + for s.handleNextServiceEvent(ctx) { + } + } + for i := 0; i < s.settings.Synchronizer.Threads; i++ { - go wait.Until(s.worker, 0, ctx.Done()) + go wait.Until(worker, 0, ctx.Done()) } <-ctx.Done() @@ -148,7 +155,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se // handleServiceEvent gets the latest state for the service from the shared // informer cache, translates the service event into server update events and // dispatches these events to the proper handler function. -func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { +func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) (err error) { logger := slog.With("service", key) logger.Debug(`Synchronizer::handleServiceEvent`) @@ -191,11 +198,11 @@ func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { for _, evt := range events { switch event.Type { case core.Created, core.Updated: - if handleErr := s.handleCreatedUpdatedEvent(evt); handleErr != nil { + if handleErr := s.handleCreatedUpdatedEvent(ctx, evt); handleErr != nil { err = errors.Join(err, handleErr) } case core.Deleted: - if handleErr := s.handleDeletedEvent(evt); handleErr != nil { + if handleErr := s.handleDeletedEvent(ctx, evt); handleErr != nil { err = errors.Join(err, handleErr) } default: @@ -219,7 +226,7 @@ func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { } // handleCreatedUpdatedEvent handles events of type Created or Updated. -func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { +func (s *Synchronizer) handleCreatedUpdatedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -229,7 +236,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU return fmt.Errorf(`error occurred creating the border client: %w`, err) } - if err = borderClient.Update(serverUpdateEvent); err != nil { + if err = borderClient.Update(ctx, serverUpdateEvent); err != nil { return fmt.Errorf(`error occurred updating the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) } @@ -237,7 +244,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU } // handleDeletedEvent handles events of type Deleted. -func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { +func (s *Synchronizer) handleDeletedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { slog.Debug(`Synchronizer::handleDeletedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -247,7 +254,7 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv return fmt.Errorf(`error occurred creating the border client: %w`, err) } - err = borderClient.Update(serverUpdateEvent) + err = borderClient.Update(ctx, serverUpdateEvent) switch { case err == nil: @@ -262,7 +269,7 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv // handleNextServiceEvent pulls a service from the event queue and feeds it to // the service event handler with retry logic -func (s *Synchronizer) handleNextServiceEvent() bool { +func (s *Synchronizer) handleNextServiceEvent(ctx context.Context) bool { slog.Debug(`Synchronizer::handleNextServiceEvent`) svc, quit := s.eventQueue.Get() @@ -278,18 +285,11 @@ func (s *Synchronizer) handleNextServiceEvent() bool { return true } - s.withRetry(s.handleServiceEvent(key), key) + s.withRetry(s.handleServiceEvent(ctx, key), key) return true } -// worker is the main message loop -func (s *Synchronizer) worker() { - slog.Debug(`Synchronizer::worker`) - for s.handleNextServiceEvent() { - } -} - // withRetry handles errors from the event handler and requeues events that fail func (s *Synchronizer) withRetry(err error, key ServiceKey) { slog.Debug("Synchronizer::withRetry") diff --git a/test/mocks/mock_nginx_plus_client.go b/test/mocks/mock_nginx_plus_client.go index 00b560ea..991ab083 100644 --- a/test/mocks/mock_nginx_plus_client.go +++ b/test/mocks/mock_nginx_plus_client.go @@ -5,7 +5,11 @@ package mocks -import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" +import ( + "context" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" +) type MockNginxClient struct { CalledFunctions map[string]bool @@ -26,7 +30,7 @@ func NewErroringMockClient(err error) *MockNginxClient { } } -func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { +func (m MockNginxClient) DeleteStreamServer(_ context.Context, _ string, _ string) error { m.CalledFunctions["DeleteStreamServer"] = true if m.Error != nil { @@ -37,6 +41,7 @@ func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { } func (m MockNginxClient) UpdateStreamServers( + _ context.Context, _ string, _ []nginxClient.StreamUpstreamServer, ) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { @@ -49,7 +54,7 @@ func (m MockNginxClient) UpdateStreamServers( return nil, nil, nil, nil } -func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { +func (m MockNginxClient) DeleteHTTPServer(_ context.Context, _ string, _ string) error { m.CalledFunctions["DeleteHTTPServer"] = true if m.Error != nil { @@ -60,6 +65,7 @@ func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { } func (m MockNginxClient) UpdateHTTPServers( + _ context.Context, _ string, _ []nginxClient.UpstreamServer, ) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { From e248d0200b51ce8ee085cfbfd74c554a6bd5ae02 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 9 Jan 2025 15:36:19 -0700 Subject: [PATCH 080/110] Bumped version to 0.8.3 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 100435be..ee94dd83 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.2 +0.8.3 From 80ecb6cf97ccfe8bd5f14565b71938a564123680 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 21 Jan 2025 16:09:41 -0700 Subject: [PATCH 081/110] NLB-6070 Removal of service annotation is equivalent to service deletion We interpret the removal of the nginxaas service annotation as a sign from the user that they no longer wish us to route traffic to the upstreams formerly associated with theservice. It is as if they were at the starting point of not yet having configured NLK to monitor a specific kubernetes service. When we see that the nginxaas service annotation has been removed, we delete all upstream servers from the upstreams associated with the service. --- internal/observation/watcher.go | 4 +++- internal/synchronization/cache.go | 21 +++++++++++++++------ internal/synchronization/synchronizer.go | 18 ++++++++++++++---- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 0df49fe1..9711347c 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -243,12 +243,14 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interface{}) { slog.Info("Watcher::buildServiceEventHandlerForUpdate") return func(previous, updated interface{}) { - // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API previousService := previous.(*v1.Service) service := updated.(*v1.Service) if w.isDesiredService(previousService) && !w.isDesiredService(service) { + slog.Info("Watcher::service annotation removed", "serviceName", service.Name) w.register.removeService(previousService) + e := core.NewEvent(core.Deleted, previousService) + w.synchronizer.AddEvent(e) return } diff --git a/internal/synchronization/cache.go b/internal/synchronization/cache.go index 6c3f1644..14effb99 100644 --- a/internal/synchronization/cache.go +++ b/internal/synchronization/cache.go @@ -2,6 +2,7 @@ package synchronization import ( "sync" + "time" v1 "k8s.io/api/core/v1" ) @@ -11,23 +12,31 @@ import ( // caller can access the spec of the deleted service for cleanup. type cache struct { mu sync.RWMutex - store map[ServiceKey]*v1.Service + store map[ServiceKey]service +} + +type service struct { + service *v1.Service + // removedAt indicates when the service was removed from NGINXaaS + // monitoring. A zero time indicates that the service is still actively + // being monitored by NGINXaaS. + removedAt time.Time } func newCache() *cache { return &cache{ - store: make(map[ServiceKey]*v1.Service), + store: make(map[ServiceKey]service), } } -func (s *cache) get(key ServiceKey) (*v1.Service, bool) { +func (s *cache) get(key ServiceKey) (service, bool) { s.mu.RLock() defer s.mu.RUnlock() - service, ok := s.store[key] - return service, ok + svc, ok := s.store[key] + return svc, ok } -func (s *cache) add(key ServiceKey, service *v1.Service) { +func (s *cache) add(key ServiceKey, service service) { s.mu.Lock() defer s.mu.Unlock() s.store[key] = service diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index d45277b8..82b3071f 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -11,6 +11,7 @@ import ( "fmt" "log/slog" "strings" + "time" nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" @@ -84,7 +85,12 @@ func (s *Synchronizer) AddEvent(event core.Event) { } key := ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace} - s.cache.add(key, event.Service) + var deletedAt time.Time + if event.Type == core.Deleted { + deletedAt = time.Now() + } + + s.cache.add(key, service{event.Service, deletedAt}) s.eventQueue.AddRateLimited(key) } @@ -162,6 +168,8 @@ func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) ( // if a service exists in the shared informer cache, we can assume that we need to update it event := core.Event{Type: core.Updated} + cachedService, exists := s.cache.get(key) + namespaceLister := s.serviceLister.Services(key.Namespace) k8sService, err := namespaceLister.Get(key.Name) switch { @@ -169,16 +177,18 @@ func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) ( // gather the last known state of the service so we can delete its // upstream servers case err != nil && apierrors.IsNotFound(err): - service, ok := s.cache.get(key) - if !ok { + if !exists { logger.Warn(`Synchronizer::handleServiceEvent: no information could be gained about service`) return nil } // no matter what type the cached event has, the service no longer exists, so the type is Deleted event.Type = core.Deleted - event.Service = service + event.Service = cachedService.service case err != nil: return err + case exists && !cachedService.removedAt.IsZero(): + event.Type = core.Deleted + event.Service = cachedService.service default: event.Service = k8sService } From 3c8efa537f86de38b38e45149e52084c82b43709 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 21 Jan 2025 16:27:06 -0700 Subject: [PATCH 082/110] NLB-6070 bumped version to 0.8.4 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index ee94dd83..b60d7196 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.3 +0.8.4 From d77cc40cddd71fcb164039f175043494abde9ecd Mon Sep 17 00:00:00 2001 From: Dylan WAY Date: Wed, 29 Jan 2025 14:21:56 -0700 Subject: [PATCH 083/110] NLB-5828: update nginx-plus-go-client version Pulls in the latest nginx-plus-go-client to make use of some optimizations in the UpdateStreamServers and UpdateHTTPServers calls. --- go.mod | 2 +- go.sum | 4 ++-- version | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 512dc364..f7c3dc8a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ go 1.22.6 toolchain go1.23.4 require ( - github.com/nginx/nginx-plus-go-client/v2 v2.2.0 + github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 golang.org/x/sync v0.10.0 diff --git a/go.sum b/go.sum index cae00cea..505b7f64 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginx/nginx-plus-go-client/v2 v2.2.0 h1:qwhx4fF/pq+h72/nE+o+XSH5mZmDU/R8fwim6VcZ8cM= -github.com/nginx/nginx-plus-go-client/v2 v2.2.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= +github.com/nginx/nginx-plus-go-client/v2 v2.3.0 h1:ciKh1lwadNzUaOGjLcKWu/BGigASxU6p7v/6US71fhA= +github.com/nginx/nginx-plus-go-client/v2 v2.3.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= diff --git a/version b/version index b60d7196..7ada0d30 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.4 +0.8.5 From 5cd6a4d9671a24342b2355000a41a0808b9b88c3 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 4 Feb 2025 12:08:07 -0800 Subject: [PATCH 084/110] Release NLK 1.0.0 This marks the official GA release for NLK. The updated semver should help communicate the readiness of the integration and the product to the customers. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 7ada0d30..3eefcb9d 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.5 +1.0.0 From f92c0619c83b1fb14e2736540217135d62e7be7a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 10 Feb 2025 14:43:17 -0700 Subject: [PATCH 085/110] NLB-6230 Handle services of type LoadBalancer NLK listens for changes to any kubernetes service of type LoadBalancer and updates the NGINXaaS upstreams with the external IPs and ports of the LB service. We are using the external IPs and not the node IPs so as to provide seamless plug-and-play functionality with the customer's networking framework. --- internal/translation/translator.go | 42 ++++- internal/translation/translator_test.go | 210 +++++++++++++++++++----- 2 files changed, 210 insertions(+), 42 deletions(-) diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 117c0b91..8491a668 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -56,6 +56,8 @@ func (t *Translator) buildServerUpdateEvents(ports []v1.ServicePort, event *core return t.buildNodeIPEvents(ports, event) case v1.ServiceTypeClusterIP: return t.buildClusterIPEvents(event) + case v1.ServiceTypeLoadBalancer: + return t.buildLoadBalancerEvents(event) default: return events, fmt.Errorf("unsupported service type: %s", event.Service.Spec.Type) } @@ -66,6 +68,38 @@ type upstream struct { name string } +func (t *Translator) buildLoadBalancerEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { + slog.Debug("Translator::buildLoadBalancerEvents", "ports", event.Service.Spec.Ports) + + addresses := make([]string, 0, len(event.Service.Status.LoadBalancer.Ingress)) + for _, ingress := range event.Service.Status.LoadBalancer.Ingress { + addresses = append(addresses, ingress.IP) + } + + for _, port := range event.Service.Spec.Ports { + context, upstreamName, err := getContextAndUpstreamName(port.Name) + if err != nil { + slog.Info("Translator::buildLoadBalancerEvents: ignoring port", "err", err, "name", port.Name) + continue + } + + upstreamServers := buildUpstreamServers(addresses, port.Port) + + switch event.Type { + case core.Created, core.Updated: + events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) + case core.Deleted: + events = append(events, core.NewServerUpdateEvent( + core.Updated, upstreamName, context, nil, + )) + default: + slog.Warn(`Translator::buildLoadBalancerEvents: unknown event type`, "type", event.Type) + } + } + + return events, nil +} + func (t *Translator) buildClusterIPEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { namespace := event.Service.GetObjectMeta().GetNamespace() serviceName := event.Service.Name @@ -153,7 +187,7 @@ func (t *Translator) buildNodeIPEvents(ports []v1.ServicePort, event *core.Event return nil, err } - upstreamServers := buildUpstreamServers(addresses, port) + upstreamServers := buildUpstreamServers(addresses, port.NodePort) switch event.Type { case core.Created: @@ -175,11 +209,11 @@ func (t *Translator) buildNodeIPEvents(ports []v1.ServicePort, event *core.Event return events, nil } -func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamServers { +func buildUpstreamServers(ipAddresses []string, port int32) core.UpstreamServers { var servers core.UpstreamServers - for _, nodeIP := range nodeIPs { - host := fmt.Sprintf("%s:%d", nodeIP, port.NodePort) + for _, ip := range ipAddresses { + host := fmt.Sprintf("%s:%d", ip, port) server := core.NewUpstreamServer(host) servers = append(servers, server) } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 11fa22ce..309a07cb 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -39,8 +39,9 @@ const ( func TestCreatedTranslateNoPorts(t *testing.T) { t.Parallel() testcases := map[string]struct{ serviceType v1.ServiceType }{ - "nodePort": {v1.ServiceTypeNodePort}, - "clusterIP": {v1.ServiceTypeClusterIP}, + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + "loadBalancer": {v1.ServiceTypeLoadBalancer}, } for name, tc := range testcases { @@ -51,7 +52,7 @@ func TestCreatedTranslateNoPorts(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, 0) translator := NewTranslator( NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), @@ -74,8 +75,9 @@ func TestCreatedTranslateNoPorts(t *testing.T) { func TestCreatedTranslateNoInterestingPorts(t *testing.T) { t.Parallel() testcases := map[string]struct{ serviceType v1.ServiceType }{ - "nodePort": {v1.ServiceTypeNodePort}, - "clusterIP": {v1.ServiceTypeClusterIP}, + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + "loadBalancer": {v1.ServiceTypeLoadBalancer}, } for name, tc := range testcases { @@ -88,7 +90,7 @@ func TestCreatedTranslateNoInterestingPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, 0) translator := NewTranslator( NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), @@ -114,6 +116,7 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -127,6 +130,11 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } for name, tc := range testcases { @@ -139,7 +147,7 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -163,6 +171,7 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -176,6 +185,11 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } for name, tc := range testcases { @@ -188,7 +202,7 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -213,6 +227,7 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -226,6 +241,11 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } for name, tc := range testcases { @@ -239,7 +259,7 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -263,6 +283,7 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -276,6 +297,11 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), expectedServerCount: ManyEndpointSlices, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + expectedServerCount: ManyNodes, + }, } for name, tc := range testcases { @@ -288,7 +314,7 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -316,6 +342,7 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -329,6 +356,11 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -338,7 +370,7 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -360,6 +392,7 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -373,6 +406,11 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -384,7 +422,7 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -406,6 +444,7 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -419,6 +458,11 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -430,7 +474,7 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -455,6 +499,7 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -468,6 +513,11 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -479,7 +529,7 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -504,6 +554,7 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -517,6 +568,11 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -529,7 +585,7 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -554,6 +610,7 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -567,6 +624,11 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), expectedServerCount: ManyEndpointSlices, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: ManyNodes, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -579,7 +641,7 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -608,6 +670,7 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -615,6 +678,9 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { "clusterIP": { serviceType: v1.ServiceTypeClusterIP, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -625,7 +691,7 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -650,6 +716,7 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -657,6 +724,9 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { "clusterIP": { serviceType: v1.ServiceTypeClusterIP, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -668,7 +738,7 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -694,6 +764,7 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -702,6 +773,9 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(0, 1, 1), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -714,7 +788,7 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -740,6 +814,7 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -748,6 +823,9 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(0, 4, 4), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -759,7 +837,7 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -784,6 +862,7 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -792,6 +871,9 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(0, 6, 2), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -805,7 +887,7 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -831,6 +913,7 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -840,6 +923,10 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -850,7 +937,7 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -875,6 +962,7 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -884,6 +972,10 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -895,7 +987,7 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -921,6 +1013,7 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -930,6 +1023,10 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -942,7 +1039,7 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -968,6 +1065,7 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -977,6 +1075,10 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -989,7 +1091,7 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1014,6 +1116,7 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1023,6 +1126,10 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -1035,7 +1142,7 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1061,6 +1168,7 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1070,6 +1178,10 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 0, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1079,7 +1191,7 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1104,6 +1216,7 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1113,6 +1226,10 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1125,7 +1242,7 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1151,6 +1268,7 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1160,6 +1278,10 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 1), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1171,7 +1293,7 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1197,6 +1319,7 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1206,6 +1329,10 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 4, 4), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1217,7 +1344,7 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1242,6 +1369,7 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1263,7 +1391,7 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1311,21 +1439,27 @@ func serviceWithPorts(serviceType v1.ServiceType, ports []v1.ServicePort) *v1.Se } } -func buildCreatedEvent(service *v1.Service) core.Event { - return buildEvent(core.Created, service) +func buildCreatedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Created, service, ingressCount) } -func buildDeletedEvent(service *v1.Service) core.Event { - return buildEvent(core.Deleted, service) +func buildDeletedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Deleted, service, ingressCount) } -func buildUpdatedEvent(service *v1.Service) core.Event { - return buildEvent(core.Updated, service) +func buildUpdatedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Updated, service, ingressCount) } -func buildEvent(eventType core.EventType, service *v1.Service) core.Event { +func buildEvent(eventType core.EventType, service *v1.Service, ingressCount int) core.Event { event := core.NewEvent(eventType, service) event.Service.Name = "default-service" + ingresses := make([]v1.LoadBalancerIngress, 0, ingressCount) + for i := range ingressCount { + ingress := v1.LoadBalancerIngress{IP: fmt.Sprintf("ipAddress%d", i)} + ingresses = append(ingresses, ingress) + } + event.Service.Status.LoadBalancer.Ingress = ingresses return event } From 40f2dc8b90f7ee00a561e90485fc15e364a58cf8 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 10 Feb 2025 14:46:31 -0700 Subject: [PATCH 086/110] NLB-6320 Bumped to version 1.1.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 3eefcb9d..9084fa2f 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.0.0 +1.1.0 From 7613acd34884b6a585c991b168424c70daa7597f Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 11:11:04 -0700 Subject: [PATCH 087/110] NLB-6293 Updated k8s libraries used by NLK This will help fix vulnerabilities discoved by mend in k8s.io/apiMAChinery-v0.26.0. --- go.mod | 58 +++++++-------- go.sum | 229 ++++++++++++++++++--------------------------------------- 2 files changed, 100 insertions(+), 187 deletions(-) diff --git a/go.mod b/go.mod index f7c3dc8a..5943bf62 100644 --- a/go.mod +++ b/go.mod @@ -4,40 +4,38 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.22.6 - -toolchain go1.23.4 +go 1.23.0 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.10.0 - k8s.io/api v0.26.0 - k8s.io/apimachinery v0.26.0 - k8s.io/client-go v0.26.0 + golang.org/x/sync v0.11.0 + k8s.io/api v0.32.2 + k8s.io/apimachinery v0.32.2 + k8s.io/client-go v0.32.2 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -52,27 +50,27 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/x448/float16 v0.8.4 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.5.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 505b7f64..e03c1876 100644 --- a/go.sum +++ b/go.sum @@ -1,76 +1,53 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -79,10 +56,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -94,12 +69,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nginx/nginx-plus-go-client/v2 v2.3.0 h1:ciKh1lwadNzUaOGjLcKWu/BGigASxU6p7v/6US71fhA= github.com/nginx/nginx-plus-go-client/v2 v2.3.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= -github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -107,9 +80,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -124,24 +96,23 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -149,131 +120,75 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= -k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= -k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= -k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= -k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= +k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= +k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= +k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= +k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= From 683f71956718398458fde7e824583c9c3f6972f5 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 11:12:09 -0700 Subject: [PATCH 088/110] NLB-6293 Added mutex lock around certification's Certificates type This enables concurrency-safe access by multiple goroutines. --- cmd/certificates-test-harness/main.go | 4 +- cmd/tls-config-factory-test-harness/main.go | 40 +++--- internal/authentication/factory_test.go | 130 ++++++++++---------- internal/certification/certificates.go | 61 +++++---- internal/certification/certificates_test.go | 15 ++- 5 files changed, 136 insertions(+), 114 deletions(-) diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index f3468a9d..898c3a39 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -37,14 +37,14 @@ func run() error { return fmt.Errorf(`error building a Kubernetes client: %w`, err) } - certificates := certification.NewCertificates(ctx, k8sClient) + certificates := certification.NewCertificates(k8sClient, nil) err = certificates.Initialize() if err != nil { return fmt.Errorf(`error occurred initializing certificates: %w`, err) } - go certificates.Run() //nolint:errcheck + go certificates.Run(ctx) //nolint:errcheck <-ctx.Done() return nil diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 7883d65a..638b0bc1 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -89,28 +89,28 @@ func buildConfigMap() map[string]TLSConfiguration { } func ssTLSConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) return configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } } func ssMtlsConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) return configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } } @@ -121,14 +121,14 @@ func caTLSConfig() configuration.Settings { } func caMtlsConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) return configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.CertificateAuthorityMutualTLS, + Certificates: certificates, } } diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go index 6b5fcaf9..47776074 100644 --- a/internal/authentication/factory_test.go +++ b/internal/authentication/factory_test.go @@ -37,16 +37,16 @@ func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } tlsConfig, err := NewTLSConfig(settings) @@ -73,14 +73,16 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -95,16 +97,16 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -119,17 +121,17 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } tlsConfig, err := NewTLSConfig(settings) @@ -156,15 +158,17 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -179,17 +183,17 @@ func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -232,16 +236,16 @@ func TestTlsFactory_CaTlsMode(t *testing.T) { func TestTlsFactory_CaMtlsMode(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.CertificateAuthorityMutualTLS, + Certificates: certificates, } tlsConfig, err := NewTLSConfig(settings) @@ -268,15 +272,17 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.CertificateAuthorityMutualTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go index c59eb0ea..86b93206 100644 --- a/internal/certification/certificates.go +++ b/internal/certification/certificates.go @@ -13,6 +13,7 @@ import ( "context" "fmt" "log/slog" + "sync" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" @@ -34,10 +35,8 @@ const ( ) type Certificates struct { - Certificates map[string]map[string]core.SecretBytes - - // Context is the context used to control the application. - Context context.Context + mu sync.Mutex // guards Certificates + certificates map[string]map[string]core.SecretBytes // CaCertificateSecretKey is the name of the Secret that contains the Certificate Authority certificate. CaCertificateSecretKey string @@ -56,25 +55,32 @@ type Certificates struct { } // NewCertificates factory method that returns a new Certificates object. -func NewCertificates(ctx context.Context, k8sClient kubernetes.Interface) *Certificates { +func NewCertificates( + k8sClient kubernetes.Interface, certificates map[string]map[string]core.SecretBytes, +) *Certificates { return &Certificates{ k8sClient: k8sClient, - Context: ctx, - Certificates: nil, + certificates: certificates, } } // GetCACertificate returns the Certificate Authority certificate. func (c *Certificates) GetCACertificate() core.SecretBytes { - bytes := c.Certificates[c.CaCertificateSecretKey][CertificateKey] + c.mu.Lock() + defer c.mu.Unlock() + + bytes := c.certificates[c.CaCertificateSecretKey][CertificateKey] return bytes } // GetClientCertificate returns the Client certificate and key. func (c *Certificates) GetClientCertificate() (core.SecretBytes, core.SecretBytes) { - keyBytes := c.Certificates[c.ClientCertificateSecretKey][CertificateKeyKey] - certificateBytes := c.Certificates[c.ClientCertificateSecretKey][CertificateKey] + c.mu.Lock() + defer c.mu.Unlock() + + keyBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKeyKey] + certificateBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKey] return keyBytes, certificateBytes } @@ -85,7 +91,9 @@ func (c *Certificates) Initialize() error { var err error - c.Certificates = make(map[string]map[string]core.SecretBytes) + c.mu.Lock() + c.certificates = make(map[string]map[string]core.SecretBytes) + c.mu.Unlock() informer := c.buildInformer() @@ -100,16 +108,16 @@ func (c *Certificates) Initialize() error { } // Run starts the SharedInformer. -func (c *Certificates) Run() error { +func (c *Certificates) Run(ctx context.Context) error { slog.Info("Certificates::Run") if c.informer == nil { return fmt.Errorf(`initialize must be called before Run`) } - c.informer.Run(c.Context.Done()) + c.informer.Run(ctx.Done()) - <-c.Context.Done() + <-ctx.Done() return nil } @@ -152,17 +160,20 @@ func (c *Certificates) handleAddEvent(obj interface{}) { return } - c.Certificates[secret.Name] = map[string]core.SecretBytes{} + c.mu.Lock() + defer c.mu.Unlock() + + c.certificates[secret.Name] = map[string]core.SecretBytes{} // Input from the secret comes in the form // tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVCVEN... // tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0l... // Where the keys are `tls.crt` and `tls.key` and the values are []byte for k, v := range secret.Data { - c.Certificates[secret.Name][k] = core.SecretBytes(v) + c.certificates[secret.Name][k] = core.SecretBytes(v) } - slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.Certificates))) + slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.certificates))) } func (c *Certificates) handleDeleteEvent(obj interface{}) { @@ -174,11 +185,14 @@ func (c *Certificates) handleDeleteEvent(obj interface{}) { return } - if c.Certificates[secret.Name] != nil { - delete(c.Certificates, secret.Name) + c.mu.Lock() + defer c.mu.Unlock() + + if c.certificates[secret.Name] != nil { + delete(c.certificates, secret.Name) } - slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.Certificates))) + slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.certificates))) } func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { @@ -190,9 +204,12 @@ func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { return } + c.mu.Lock() + defer c.mu.Unlock() + for k, v := range secret.Data { - c.Certificates[secret.Name][k] = v + c.certificates[secret.Name][k] = v } - slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.Certificates))) + slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.certificates))) } diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index 901964ac..7d2363b2 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -23,9 +23,8 @@ const ( func TestNewCertificate(t *testing.T) { t.Parallel() - ctx := context.Background() - certificates := NewCertificates(ctx, nil) + certificates := NewCertificates(nil, nil) if certificates == nil { t.Fatalf(`certificates should not be nil`) @@ -34,7 +33,7 @@ func TestNewCertificate(t *testing.T) { func TestCertificates_Initialize(t *testing.T) { t.Parallel() - certificates := NewCertificates(context.Background(), nil) + certificates := NewCertificates(nil, nil) err := certificates.Initialize() if err != nil { @@ -44,9 +43,9 @@ func TestCertificates_Initialize(t *testing.T) { func TestCertificates_RunWithoutInitialize(t *testing.T) { t.Parallel() - certificates := NewCertificates(context.Background(), nil) + certificates := NewCertificates(nil, nil) - err := certificates.Run() + err := certificates.Run(context.Background()) if err == nil { t.Fatalf(`Expected error`) } @@ -58,7 +57,7 @@ func TestCertificates_RunWithoutInitialize(t *testing.T) { func TestCertificates_EmptyCertificates(t *testing.T) { t.Parallel() - certificates := NewCertificates(context.Background(), nil) + certificates := NewCertificates(nil, nil) err := certificates.Initialize() if err != nil { @@ -86,7 +85,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { k8sClient := fake.NewSimpleClientset() - certificates := NewCertificates(ctx, k8sClient) + certificates := NewCertificates(k8sClient, nil) _ = certificates.Initialize() @@ -94,7 +93,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { //nolint:govet,staticcheck go func() { - err := certificates.Run() + err := certificates.Run(context.Background()) assert.NoError(t, err, "expected no error running certificates") }() From 531d26031e372bca86576e693003f9d678d1d630 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 11:20:49 -0700 Subject: [PATCH 089/110] NLB-6293 Upgraded golangci-lint to v1.64.5 and fixed configuration --- .golangci.yml | 18 +++++++----------- cmd/nginx-loadbalancer-kubernetes/main.go | 4 ++++ internal/synchronization/synchronizer.go | 3 +++ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 6cb35877..6f0fddf4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,7 +26,6 @@ linters: - misspell - nakedret - prealloc - - exportloopref - stylecheck - unconvert - unparam @@ -38,20 +37,17 @@ linters: run: # 10 minute timeout for analysis timeout: 10m - skip-dirs-use-default: true - skip-dirs: - - .go/pkg/mod - - pkg/spec/api # Generated code - - vendor - - vendor-fork # Specific linter settings linters-settings: gocyclo: # Minimal code complexity to report min-complexity: 16 govet: - # Report shadowed variables - check-shadowing: true + disable-all: true + enable: + # Report shadowed variables + - shadow + misspell: # Correct spellings using locale preferences for US locale: US @@ -63,8 +59,8 @@ linters-settings: # If this list is empty, all structs are tested. # Default: [] include: - - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/pkg/token.TokenID' - - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest' + - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/pkg/token.TokenID" + - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest" issues: # Exclude configuration diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 5aba57f1..d2d923c8 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -133,9 +133,13 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { return client, nil } +// TODO: NLB-6294 change to use new typed workqueues +// +//nolint:staticcheck //ignore deprecation warnings func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { slog.Debug("Watcher::buildSynchronizerWorkQueue") + //nolint:staticcheck //ignore deprecation warnings rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 82b3071f..3810805a 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -50,6 +50,8 @@ type ServiceKey struct { // a Border Client as specified in the Service annotation for the Upstream. // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { + // TODO: NLB-6294 change to use new typed workqueues + //nolint:staticcheck //ignore deprecation warnings eventQueue workqueue.RateLimitingInterface settings configuration.Settings translator Translator @@ -60,6 +62,7 @@ type Synchronizer struct { // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( settings configuration.Settings, + //nolint:staticcheck //ignore deprecation warnings eventQueue workqueue.RateLimitingInterface, translator Translator, serviceLister corelisters.ServiceLister, From 09e5c021d6aad26c1cc9f3f1abca1099fd2b041f Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 15:19:01 -0700 Subject: [PATCH 090/110] NLB-6294 The synchronizer uses a typed rate-limited workqueue The latest versions of the kubernetes libraries recommend using a typed workqueue and this reduces a bit of boilerplate and error handling, because we no longer have to cast the workitems returned by the queue into the desired types. --- cmd/nginx-loadbalancer-kubernetes/main.go | 13 ++++----- internal/synchronization/synchronizer.go | 15 ++-------- internal/synchronization/synchronizer_test.go | 14 ++++----- test/mocks/mock_ratelimitinginterface.go | 29 +++++++++---------- 4 files changed, 30 insertions(+), 41 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index d2d923c8..43de7940 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -133,13 +133,12 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { return client, nil } -// TODO: NLB-6294 change to use new typed workqueues -// -//nolint:staticcheck //ignore deprecation warnings -func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { +func buildWorkQueue(settings configuration.WorkQueueSettings, +) workqueue.TypedRateLimitingInterface[synchronization.ServiceKey] { slog.Debug("Watcher::buildSynchronizerWorkQueue") - //nolint:staticcheck //ignore deprecation warnings - rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) - return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) + rateLimiter := workqueue.NewTypedItemExponentialFailureRateLimiter[synchronization.ServiceKey]( + settings.RateLimiterBase, settings.RateLimiterMax) + return workqueue.NewTypedRateLimitingQueueWithConfig( + rateLimiter, workqueue.TypedRateLimitingQueueConfig[synchronization.ServiceKey]{Name: settings.Name}) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 3810805a..50717d79 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -50,9 +50,7 @@ type ServiceKey struct { // a Border Client as specified in the Service annotation for the Upstream. // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { - // TODO: NLB-6294 change to use new typed workqueues - //nolint:staticcheck //ignore deprecation warnings - eventQueue workqueue.RateLimitingInterface + eventQueue workqueue.TypedRateLimitingInterface[ServiceKey] settings configuration.Settings translator Translator cache *cache @@ -62,8 +60,7 @@ type Synchronizer struct { // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( settings configuration.Settings, - //nolint:staticcheck //ignore deprecation warnings - eventQueue workqueue.RateLimitingInterface, + eventQueue workqueue.TypedRateLimitingInterface[ServiceKey], translator Translator, serviceLister corelisters.ServiceLister, ) (*Synchronizer, error) { @@ -292,13 +289,7 @@ func (s *Synchronizer) handleNextServiceEvent(ctx context.Context) bool { defer s.eventQueue.Done(svc) - key, ok := svc.(ServiceKey) - if !ok { - slog.Warn(`Synchronizer::handleNextServiceEvent: invalid service type`, "service", svc) - return true - } - - s.withRetry(s.handleServiceEvent(ctx, key), key) + s.withRetry(s.handleServiceEvent(ctx, svc), svc) return true } diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index 7592f5ca..c49c7183 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -22,7 +22,7 @@ import ( func TestSynchronizer_NewSynchronizer(t *testing.T) { t.Parallel() - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( configuration.Settings{}, @@ -43,7 +43,7 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings(), @@ -73,7 +73,7 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { const expectedEventCount = 1 events := buildServerUpdateEvents(1) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings("https://localhost:8080"), @@ -106,7 +106,7 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { "https://localhost:8082", } - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings(hosts...), @@ -133,7 +133,7 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 events := buildServerUpdateEvents(4) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings(), @@ -165,7 +165,7 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 4 events := buildServerUpdateEvents(1) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings("https://localhost:8080"), @@ -195,7 +195,7 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { t.Parallel() const eventCount = 4 events := buildServerUpdateEvents(eventCount) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} hosts := []string{ "https://localhost:8080", diff --git a/test/mocks/mock_ratelimitinginterface.go b/test/mocks/mock_ratelimitinginterface.go index ee3ccd49..d5da3b71 100644 --- a/test/mocks/mock_ratelimitinginterface.go +++ b/test/mocks/mock_ratelimitinginterface.go @@ -7,51 +7,50 @@ package mocks import "time" -type MockRateLimiter struct { - items []interface{} +type MockRateLimiter[T any] struct { + items []T } -func (m *MockRateLimiter) Add(_ interface{}) { +func (m *MockRateLimiter[T]) Add(_ T) { } -func (m *MockRateLimiter) Len() int { +func (m *MockRateLimiter[T]) Len() int { return len(m.items) } -func (m *MockRateLimiter) Get() (item interface{}, shutdown bool) { +func (m *MockRateLimiter[T]) Get() (item T, shutdown bool) { if len(m.items) > 0 { item = m.items[0] m.items = m.items[1:] return item, false } - return nil, false + return item, false } -func (m *MockRateLimiter) Done(_ interface{}) { +func (m *MockRateLimiter[T]) Done(_ T) { } -func (m *MockRateLimiter) ShutDown() { +func (m *MockRateLimiter[T]) ShutDown() { } -func (m *MockRateLimiter) ShutDownWithDrain() { +func (m *MockRateLimiter[T]) ShutDownWithDrain() { } -func (m *MockRateLimiter) ShuttingDown() bool { +func (m *MockRateLimiter[T]) ShuttingDown() bool { return true } -func (m *MockRateLimiter) AddAfter(item interface{}, _ time.Duration) { +func (m *MockRateLimiter[T]) AddAfter(item T, _ time.Duration) { m.items = append(m.items, item) } -func (m *MockRateLimiter) AddRateLimited(item interface{}) { +func (m *MockRateLimiter[T]) AddRateLimited(item T) { m.items = append(m.items, item) } -func (m *MockRateLimiter) Forget(_ interface{}) { - +func (m *MockRateLimiter[T]) Forget(_ T) { } -func (m *MockRateLimiter) NumRequeues(_ interface{}) int { +func (m *MockRateLimiter[T]) NumRequeues(_ T) int { return 0 } From 796beb88d487c009978afe0eede24da256f21a26 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 15:21:53 -0700 Subject: [PATCH 091/110] NLB-6294 Bumped version to 1.1.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 9084fa2f..524cb552 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.0 +1.1.1 From 4cdeb58adacea921956a28f274a64b51eaa29d8a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 28 Feb 2025 09:07:24 -0700 Subject: [PATCH 092/110] NLB-6294 NewTransport clones the default http.DefaultTransport variable Multiple parallel tests were all accessing the same pointer to a single variable for the DefaultTransport in the http package. This was leading to data races in unit tests. --- internal/communication/factory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index eec593b3..cf1bfcbe 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -63,7 +63,7 @@ func NewTLSConfig(settings configuration.Settings) *tls.Config { // NewTransport is a factory method to create a new basic Http Transport. func NewTransport(config *tls.Config) *netHttp.Transport { - transport := netHttp.DefaultTransport.(*netHttp.Transport) + transport := netHttp.DefaultTransport.(*netHttp.Transport).Clone() transport.TLSClientConfig = config return transport From bd9a3dc426caac368434ac6829e3ffa77167b735 Mon Sep 17 00:00:00 2001 From: chrisakker <38477223+chrisakker@users.noreply.github.com> Date: Thu, 25 Jul 2024 13:55:40 -0700 Subject: [PATCH 093/110] prometheus config update (#178) --- docs/http/http-installation-guide.md | 15 +++++++++++---- docs/http/prometheus.conf | 3 +++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/http/http-installation-guide.md b/docs/http/http-installation-guide.md index 8484bfd1..a9545429 100644 --- a/docs/http/http-installation-guide.md +++ b/docs/http/http-installation-guide.md @@ -201,21 +201,28 @@ Note: If you choose a different Application to test with, `the NGINX configurati This can be any standard Linux OS system, based on the Linux Distro and Technical Specs required for NGINX Plus, which can be found here: https://docs.nginx.com/nginx/technical-specs/ - 1. This Solution followed the `Installation of NGINX Plus on Centos/Redhat/Oracle` steps for installing NGINX Plus. +1. This Solution followed the `Installation of NGINX Plus on Centos/Redhat/Oracle` steps for installing NGINX Plus. https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-plus/ >NOTE: This Solution will only work with NGINX Plus, as NGINX OpenSource does not have the API that is used in this Solution. Installation on unsupported Linux Distros is not recommended. - 1. Install the NGINX Javascript module (njs). This is required for exporting Prometheus Metrics from NGINX Plus. + If you need a license for NGINX Plus, a 30-day Trial license is available here: + + https://www.nginx.com/free-trial-request/ + +1. Install the NGINX Javascript module (njs). This is required for exporting Prometheus Metrics from NGINX Plus. ```bash yum install nginx-plus-module-njs ``` - 1. If you need a license for NGINX Plus, a 30-day Trial license is available here: +1. Install Nginx Javascript for Prometheus + + ```bash + yum install nginx-plus-module-prometheus + ``` - https://www.nginx.com/free-trial-request/
diff --git a/docs/http/prometheus.conf b/docs/http/prometheus.conf index 15f53c9c..ad1204ee 100644 --- a/docs/http/prometheus.conf +++ b/docs/http/prometheus.conf @@ -7,6 +7,9 @@ js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; server { + listen 9113: + status_zone prometheus; + location = /metrics { js_content prometheus.metrics; } From ec41eb1a95bc84df2f16ba4db3a5223b61139393 Mon Sep 17 00:00:00 2001 From: chrisakker <38477223+chrisakker@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:05:52 -0700 Subject: [PATCH 094/110] update prometheus files (#179) --- docs/http/clusters.conf | 2 +- docs/http/default-http.conf | 28 ------------- docs/http/http-installation-guide.md | 59 ++++++---------------------- docs/http/prometheus.yml | 2 +- 4 files changed, 14 insertions(+), 77 deletions(-) delete mode 100644 docs/http/default-http.conf diff --git a/docs/http/clusters.conf b/docs/http/clusters.conf index 6c3ab6d8..4a71ef4f 100644 --- a/docs/http/clusters.conf +++ b/docs/http/clusters.conf @@ -9,7 +9,7 @@ # Define Key Value store, backup state file, timeout, and enable sync -keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=30d sync; +keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=365d sync; keyval $host $split_level zone=split; # Main Nginx Server Block for cafe.example.com, with TLS diff --git a/docs/http/default-http.conf b/docs/http/default-http.conf deleted file mode 100644 index b9685533..00000000 --- a/docs/http/default-http.conf +++ /dev/null @@ -1,28 +0,0 @@ -# NGINX Loadbalancer for K8s Solution -# Chris Akker, Apr 2023 -# Example default.conf -# Change default_server to port 8080 -# -server { - listen 8080 default_server; # Changed to 8080 - server_name localhost; - - #access_log /var/log/nginx/host.access.log main; - - location / { - root /usr/share/nginx/html; - index index.html index.htm; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - -### other sections removed for clarity - -} \ No newline at end of file diff --git a/docs/http/http-installation-guide.md b/docs/http/http-installation-guide.md index a9545429..60609258 100644 --- a/docs/http/http-installation-guide.md +++ b/docs/http/http-installation-guide.md @@ -231,20 +231,18 @@ This can be any standard Linux OS system, based on the Linux Distro and Technica
### This is the NGINX configuration required for the NGINX Loadbalancing Server, external to the cluster. It must be configured for the following: - -1. Move the NGINX default Welcome page from port 80 to port 8080. Port 80 will be used by Prometheus in this Solution. -2. The NGINX NJS module is enabled, and configured to export the NGINX Plus statistics. +1. The NGINX NJS module is enabled, and configured to export the NGINX Plus statistics. -3. A self-signed TLS cert/key are used in this example for terminating TLS traffic for the Demo application, https://cafe.example.com. +2. A self-signed TLS cert/key are used in this example for terminating TLS traffic for the Demo application, https://cafe.example.com. -4. Plus API with write access enabled on port 9000. The Plus Dashboard is also enabled, used for testing, monitoring, and visualization of the Solution working. +3. Plus API with write access enabled on port 9000. The Plus Dashboard is also enabled, used for testing, monitoring, and visualization of the Solution working. -5. The `http` context is used for MultiCluster Loadbalancing, for HTTP/S processing, Split Clients ratio. The Plus Key Value Store is configured, to hold the dynamic Split ratio metadata. +4. The `http` context is used for MultiCluster Loadbalancing, for HTTP/S processing, Split Clients ratio. The Plus Key Value Store is configured, to hold the dynamic Split ratio metadata. -6. Enable Prometheus metrics exporting. +5. Enable Prometheus metrics exporting. -7. Plus Zone Sync on Port 9001 is configured, to synchronize the dynamic KeyVal data between multiple NGINX Loadbalancing Servers. +6. Plus Zone Sync on Port 9001 is configured, to synchronize the dynamic KeyVal data between multiple NGINX Loadbalancing Servers.
@@ -266,7 +264,6 @@ etc/ ├── conf.d/ │ ├── clusters.conf.......... MultiCluster Loadbalancing and split clients config │ ├── dashboard.conf......... NGINX Plus API and Dashboard config - │ ├── default-http.conf...... New default.conf config │ └── prometheus.conf........ NGINX Prometheus config ├── nginx.conf................. New nginx.conf └── stream @@ -277,41 +274,6 @@ etc/ After a new installation of NGINX Plus, make the following configuration changes: -1. Change NGINX's http default server to port 8080. See the included `default-http.conf` file. After reloading nginx, the default `Welcome to NGINX` page will be located at http://localhost:8080. - - ```bash - cat /etc/nginx/conf.d/default.conf - # NGINX Loadbalancer for Kubernetes Solution - # Chris Akker, Apr 2023 - # Example default.conf - # Change default_server to port 8080 - # - server { - listen 8080 default_server; # Changed to 8080 - server_name localhost; - - #access_log /var/log/nginx/host.access.log main; - - location / { - root /usr/share/nginx/html; - index index.html index.htm; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - - ### other sections removed for clarity - - } - - ``` - 1. Use the included nginx.conf file, it enables the NGINX NJS module, for exporting the Plus statistics: ```bash @@ -403,7 +365,7 @@ After a new installation of NGINX Plus, make the following configuration changes # Define Key Value store, backup state file, timeout, and enable sync - keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=30d sync; + keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=365d sync; keyval $host $split_level zone=split; # Main NGINX Server Block for cafe.example.com, with TLS @@ -556,6 +518,9 @@ After a new installation of NGINX Plus, make the following configuration changes js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; server { + listen 9113; + status_zone prometheus; + location = /metrics { js_content prometheus.metrics; } @@ -1014,7 +979,7 @@ Here are the instructions to run 2 Docker containers on a Monitor Server, which
-1. Configure your Prometheus server to collect NGINX Plus statistics from the scraper page. Use the prometheus.yml file provided, edit the IP addresses to match your NGINX Loadbalancing Server(s). +1. Configure your Prometheus server to collect NGINX Plus statistics from the scraper page. You can use the prometheus.yml file provided, edit the IP addresses to match your NGINX Loadbalancing Server(s). ```bash cat prometheus.yaml @@ -1033,7 +998,7 @@ Here are the instructions to run 2 Docker containers on a Monitor Server, which scrape_interval: 5s static_configs: - - targets: ['10.1.1.4:80', '10.1.1.5:80'] # NGINX Loadbalancing Servers + - targets: ['10.1.1.4:9113', '10.1.1.5:9113'] # NGINX Loadbalancing Servers ``` 1. Review, edit and place the sample `prometheus.yml` file in /etc/prometheus folder. diff --git a/docs/http/prometheus.yml b/docs/http/prometheus.yml index c8b19cb2..3dee16bc 100644 --- a/docs/http/prometheus.yml +++ b/docs/http/prometheus.yml @@ -10,5 +10,5 @@ scrape_configs: scrape_interval: 5s static_configs: - - targets: ['10.1.1.4:80', '10.1.1.5:80'] + - targets: ['10.1.1.4:9113', '10.1.1.5:9113'] \ No newline at end of file From 90c99d0e85317a4b1022189346883e1c4e38438b Mon Sep 17 00:00:00 2001 From: chrisakker <38477223+chrisakker@users.noreply.github.com> Date: Tue, 13 Aug 2024 07:46:08 -0700 Subject: [PATCH 095/110] fix typo (#181) --- docs/http/prometheus.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/http/prometheus.conf b/docs/http/prometheus.conf index ad1204ee..47d15e92 100644 --- a/docs/http/prometheus.conf +++ b/docs/http/prometheus.conf @@ -7,7 +7,7 @@ js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; server { - listen 9113: + listen 9113; status_zone prometheus; location = /metrics { From f4a556650354b18b52c264b297b07a36aef5a1f7 Mon Sep 17 00:00:00 2001 From: Itay Talmi <42022401+itaytalmi@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:09:15 -0400 Subject: [PATCH 096/110] Fix "listen" directive syntax prometheus.conf (#180) There's a syntax issue in the "listen" directive. Should be "listen 9113;", not "listen 9113:". Using the current file, I got an error upon reloading the NGINX service ("nginx: [emerg] invalid port in "9113:" of the "listen" directive in /etc/nginx/conf.d/prometheus.conf:11") From d3a9b76f9bcf56b85b8a7bf102852316632615f5 Mon Sep 17 00:00:00 2001 From: Steve Wagner Date: Fri, 14 Feb 2025 13:21:33 -0800 Subject: [PATCH 097/110] Corrects the NGINX Plus Client interface There was a change in the API for the NGINX Plus Client that was missed when updating to the latest version. This corrects that. --- internal/application/nginx_http_border_client.go | 2 ++ internal/application/nginx_stream_border_client.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index 4de147e5..91828082 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -18,6 +18,7 @@ import ( type NginxHTTPBorderClient struct { BorderClient nginxClient NginxClientInterface + ctx context.Context } // NewNginxHTTPBorderClient is the Factory function for creating an NewNginxHTTPBorderClient. @@ -29,6 +30,7 @@ func NewNginxHTTPBorderClient(client interface{}) (Interface, error) { return &NginxHTTPBorderClient{ nginxClient: ngxClient, + ctx: context.Background(), }, nil } diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index 238a22b6..a65be866 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -18,6 +18,7 @@ import ( type NginxStreamBorderClient struct { BorderClient nginxClient NginxClientInterface + ctx context.Context } // NewNginxStreamBorderClient is the Factory function for creating an NginxStreamBorderClient. @@ -29,6 +30,7 @@ func NewNginxStreamBorderClient(client interface{}) (Interface, error) { return &NginxStreamBorderClient{ nginxClient: ngxClient, + ctx: context.Background(), }, nil } From 3aff5d608a6578be123a1b1d1eba71843308a6e3 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 13 Mar 2025 16:06:20 -0600 Subject: [PATCH 098/110] Removed synhronization's random functions These functions were being used to create IDs that were not really necessary for the business logic and which were generating security alerts because of weak cryptography techniques. --- internal/core/server_update_event.go | 9 +---- internal/core/server_update_event_test.go | 10 +---- internal/synchronization/rand.go | 39 ------------------- internal/synchronization/synchronizer.go | 11 +++--- internal/synchronization/synchronizer_test.go | 1 - 5 files changed, 8 insertions(+), 62 deletions(-) delete mode 100644 internal/synchronization/rand.go diff --git a/internal/core/server_update_event.go b/internal/core/server_update_event.go index 0bc88680..04d2dc08 100644 --- a/internal/core/server_update_event.go +++ b/internal/core/server_update_event.go @@ -9,14 +9,10 @@ package core // from Events received from the Handler. These are then consumed by the Synchronizer and passed along to // the appropriate BorderClient. type ServerUpdateEvent struct { - // ClientType is the type of BorderClient that should handle this event. This is configured via Service Annotations. // See application_constants.go for the list of supported types. ClientType string - // Id is the unique identifier for this event. - ID string - // NginxHost is the host name of the NGINX Plus instance that should handle this event. NginxHost string @@ -48,11 +44,10 @@ func NewServerUpdateEvent( } } -// ServerUpdateEventWithIDAndHost creates a new ServerUpdateEvent with the specified Id and Host. -func ServerUpdateEventWithIDAndHost(event *ServerUpdateEvent, id string, nginxHost string) *ServerUpdateEvent { +// ServerUpdateEventWithHost creates a new ServerUpdateEvent with the specified Host. +func ServerUpdateEventWithHost(event *ServerUpdateEvent, nginxHost string) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: event.ClientType, - ID: id, NginxHost: nginxHost, Type: event.Type, UpstreamName: event.UpstreamName, diff --git a/internal/core/server_update_event_test.go b/internal/core/server_update_event_test.go index 9f36002c..3be4702c 100644 --- a/internal/core/server_update_event_test.go +++ b/internal/core/server_update_event_test.go @@ -17,19 +17,11 @@ func TestServerUpdateEventWithIdAndHost(t *testing.T) { t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) - if event.ID != "" { - t.Errorf("expected empty ID, got %s", event.ID) - } - if event.NginxHost != "" { t.Errorf("expected empty NginxHost, got %s", event.NginxHost) } - eventWithIDAndHost := ServerUpdateEventWithIDAndHost(event, "id", "host") - - if eventWithIDAndHost.ID != "id" { - t.Errorf("expected Id to be 'id', got %s", eventWithIDAndHost.ID) - } + eventWithIDAndHost := ServerUpdateEventWithHost(event, "host") if eventWithIDAndHost.NginxHost != "host" { t.Errorf("expected NginxHost to be 'host', got %s", eventWithIDAndHost.NginxHost) diff --git a/internal/synchronization/rand.go b/internal/synchronization/rand.go deleted file mode 100644 index 6bf58d1d..00000000 --- a/internal/synchronization/rand.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package synchronization - -import ( - // Try using crpyto if needed. - "math/rand" - "time" -) - -// charset contains all characters that can be used in random string generation -var charset = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -// number contains all numbers that can be used in random string generation -var number = []byte("0123456789") - -// alphaNumeric contains all characters and numbers that can be used in random string generation -var alphaNumeric = append(charset, number...) - -// RandomString where n is the length of random string we want to generate -func RandomString(n int) string { - b := make([]byte, n) - for i := range b { - // randomly select 1 character from given charset - b[i] = alphaNumeric[rand.Intn(len(alphaNumeric))] //nolint:gosec - } - return string(b) -} - -// RandomMilliseconds returns a random duration between min and max milliseconds -func RandomMilliseconds(min, max int) time.Duration { - randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec - random := randomizer.Intn(max-min) + min - - return time.Millisecond * time.Duration(random) -} diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 50717d79..80faaa31 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -146,10 +146,9 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se var events core.ServerUpdateEvents - for hidx, host := range s.settings.NginxPlusHosts { - for eidx, event := range event { - id := fmt.Sprintf(`[%d:%d]-[%s]-[%s]-[%s]`, hidx, eidx, RandomString(12), event.UpstreamName, host) - updatedEvent := core.ServerUpdateEventWithIDAndHost(event, id, host) + for _, host := range s.settings.NginxPlusHosts { + for _, event := range event { + updatedEvent := core.ServerUpdateEventWithHost(event, host) events = append(events, updatedEvent) } @@ -237,7 +236,7 @@ func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) ( // handleCreatedUpdatedEvent handles events of type Created or Updated. func (s *Synchronizer) handleCreatedUpdatedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { - slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`, "eventID", serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`) var err error @@ -255,7 +254,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(ctx context.Context, serverUpda // handleDeletedEvent handles events of type Deleted. func (s *Synchronizer) handleDeletedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { - slog.Debug(`Synchronizer::handleDeletedEvent`, "eventID", serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleDeletedEvent`) var err error diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index c49c7183..ba2253b2 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -244,7 +244,6 @@ func buildServerUpdateEvents(count int) core.ServerUpdateEvents { events := make(core.ServerUpdateEvents, count) for i := 0; i < count; i++ { events[i] = &core.ServerUpdateEvent{ - ID: fmt.Sprintf("id-%v", i), NginxHost: "https://localhost:8080", Type: 0, UpstreamName: "", From 32706211c9889d0950ac8facc9f5438af331aa06 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 21 Mar 2025 14:46:22 -0600 Subject: [PATCH 099/110] NGINX plus http client rejects requests with too many headers The http client is processing requests created by the nginx plus client library, and that library should always include a sensible number of headers. But the lack of change on the number of headers was causing security vulnerability flags to be raised over denial of service resource exhaustion attacks. --- internal/communication/roundtripper.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/communication/roundtripper.go b/internal/communication/roundtripper.go index 58de6f03..1dbaf5b0 100644 --- a/internal/communication/roundtripper.go +++ b/internal/communication/roundtripper.go @@ -6,10 +6,13 @@ package communication import ( + "errors" "net/http" "strings" ) +const maxHeaders = 1000 + // RoundTripper is a simple type that wraps the default net/communication RoundTripper to add additional headers. type RoundTripper struct { Headers []string @@ -26,6 +29,10 @@ func NewRoundTripper(headers []string, transport *http.Transport) *RoundTripper // RoundTrip This simply adds our default headers to the request before passing it on to the default RoundTripper. func (roundTripper *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if len(req.Header) > maxHeaders { + return nil, errors.New("request includes too many headers") + } + newRequest := new(http.Request) *newRequest = *req newRequest.Header = make(http.Header, len(req.Header)) From a39e2dbba29af6ffd4c5dfc4c6a045ef39b98e06 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 21 Mar 2025 14:46:49 -0600 Subject: [PATCH 100/110] Upgraded go to 1.23.8 and golang.org/x/sync to v0.12.0 --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 5943bf62..3fd8dff5 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,13 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.23.0 +go 1.23.8 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.11.0 + golang.org/x/sync v0.12.0 k8s.io/api v0.32.2 k8s.io/apimachinery v0.32.2 k8s.io/client-go v0.32.2 diff --git a/go.sum b/go.sum index e03c1876..91691728 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 217dc34f9f09801fff18a97f2981b95f65e69818 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 21 Mar 2025 14:46:49 -0600 Subject: [PATCH 101/110] Bumped version to 1.1.2 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 524cb552..45a1b3f4 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.1 +1.1.2 From 8109c368e54159070ec818f02f98aeebd98e67b0 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 19 May 2025 16:11:53 -0700 Subject: [PATCH 102/110] Skip go-cache while linting Go cache in the CI is seeded in the project working directory. We should skip the mod cache from lint/formatting as it's upstream code and there are high chances of the linting failing as upstream lint rules != our lint rules. --- .golangci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 6f0fddf4..268e39ff 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -63,6 +63,8 @@ linters-settings: - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest" issues: + exclude-dirs: + - .go/pkg/mod # Exclude configuration exclude-rules: # Exclude gochecknoinits and gosec from running on tests files From 89ab9a99e236aa647016d847252c858eb0e7f9b8 Mon Sep 17 00:00:00 2001 From: Aniruddh Kuthiala Date: Fri, 30 May 2025 10:37:28 -0600 Subject: [PATCH 103/110] unit test flake fix and go version upgrade to 1.24.3 --- go.mod | 35 +++++++++-------- go.sum | 73 +++++++++++++++++++----------------- internal/probation/server.go | 12 +++++- version | 2 +- 4 files changed, 66 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index 3fd8dff5..c09a4b41 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,16 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.23.8 +go 1.24.3 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 golang.org/x/sync v0.12.0 - k8s.io/api v0.32.2 - k8s.io/apimachinery v0.32.2 - k8s.io/client-go v0.32.2 + k8s.io/api v0.33.1 + k8s.io/apimachinery v0.33.1 + k8s.io/client-go v0.33.1 ) require ( @@ -26,10 +26,8 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -54,22 +52,23 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.7.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 91691728..80d0e9d4 100644 --- a/go.sum +++ b/go.sum @@ -25,16 +25,12 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -80,8 +76,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -99,14 +95,16 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -115,6 +113,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -128,10 +128,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -140,16 +140,16 @@ golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -160,8 +160,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -174,21 +174,24 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= -k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= -k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= -k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= -k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= +k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= +k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= +k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= +k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= +k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/probation/server.go b/internal/probation/server.go index 9520fc76..c3328c70 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -8,6 +8,7 @@ package probation import ( "fmt" "log/slog" + "net" "net/http" "time" ) @@ -58,10 +59,17 @@ func (hs *HealthServer) Start() { mux.HandleFunc("/livez", hs.HandleLive) mux.HandleFunc("/readyz", hs.HandleReady) mux.HandleFunc("/startupz", hs.HandleStartup) - hs.httpServer = &http.Server{Addr: address, Handler: mux, ReadTimeout: 2 * time.Second} + + listener, err := net.Listen("tcp", address) + if err != nil { + slog.Error("failed to listen", "error", err) + return + } + + hs.httpServer = &http.Server{Handler: mux, ReadTimeout: 2 * time.Second} go func() { - if err := hs.httpServer.ListenAndServe(); err != nil { + if err := hs.httpServer.Serve(listener); err != nil { slog.Error("unable to start probe listener", "address", hs.httpServer.Addr, "error", err) } }() diff --git a/version b/version index 45a1b3f4..781dcb07 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.2 +1.1.3 From 2197336b5f258be99f161f1d70d525d5f94e1e7b Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 26 Jun 2025 15:32:04 -0600 Subject: [PATCH 104/110] Fixed up helm templating for nginx-hosts param to support multiple nginx hosts In order for the nginx-hosts yaml field to be parsed correctly by viper the template needs to: 1. not put double quotes around the value (this causes viper to interpret it as a string) 2. render it as a JSON array rather than a go representation of a slice. --- charts/nlk/templates/nlk-configmap.yaml | 2 +- internal/configuration/configuration_test.go | 112 +++++++++++++----- .../testdata/multiple-nginx-hosts.yaml | 11 ++ .../testdata/one-nginx-host.yaml | 11 ++ internal/configuration/testdata/test.yaml | 11 -- version | 2 +- 6 files changed, 105 insertions(+), 44 deletions(-) create mode 100644 internal/configuration/testdata/multiple-nginx-hosts.yaml create mode 100644 internal/configuration/testdata/one-nginx-host.yaml delete mode 100644 internal/configuration/testdata/test.yaml diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 38475a9e..0b6db576 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -9,7 +9,7 @@ data: log-level: "{{ . }}" {{- end }} {{- with .Values.nlk.config.nginxHosts }} - nginx-hosts: "{{ . }}" + nginx-hosts: {{ toJson . }} {{- end }} tls-mode: "{{ .Values.nlk.config.tls.mode }}" {{- with .Values.nlk.config.serviceAnnotationMatch }} diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go index 96949891..fc419745 100644 --- a/internal/configuration/configuration_test.go +++ b/internal/configuration/configuration_test.go @@ -12,41 +12,91 @@ import ( func TestConfiguration(t *testing.T) { t.Parallel() - expectedSettings := configuration.Settings{ - LogLevel: "warn", - NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, - TLSMode: configuration.NoTLS, - Certificates: &certification.Certificates{ - CaCertificateSecretKey: "fakeCAKey", - ClientCertificateSecretKey: "fakeCertKey", - }, - Handler: configuration.HandlerSettings{ - RetryCount: 5, - Threads: 1, - WorkQueueSettings: configuration.WorkQueueSettings{ - RateLimiterBase: time.Second * 2, - RateLimiterMax: time.Second * 60, - Name: "nlk-handler", + + tests := map[string]struct { + testFile string + expectedSettings configuration.Settings + }{ + "one nginx plus host": { + testFile: "one-nginx-host", + expectedSettings: configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, + TLSMode: configuration.NoTLS, + Certificates: &certification.Certificates{ + CaCertificateSecretKey: "fakeCAKey", + ClientCertificateSecretKey: "fakeCertKey", + }, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, }, }, - Synchronizer: configuration.SynchronizerSettings{ - MaxMillisecondsJitter: 750, - MinMillisecondsJitter: 250, - RetryCount: 5, - Threads: 1, - WorkQueueSettings: configuration.WorkQueueSettings{ - RateLimiterBase: time.Second * 2, - RateLimiterMax: time.Second * 60, - Name: "nlk-synchronizer", + "multiple nginx plus hosts": { + testFile: "multiple-nginx-hosts", + expectedSettings: configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"}, + TLSMode: configuration.NoTLS, + Certificates: &certification.Certificates{ + CaCertificateSecretKey: "fakeCAKey", + ClientCertificateSecretKey: "fakeCertKey", + }, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, }, }, - Watcher: configuration.WatcherSettings{ - ResyncPeriod: 0, - ServiceAnnotation: "fakeServiceMatch", - }, } - settings, err := configuration.Read("test", "./testdata") - require.NoError(t, err) - require.Equal(t, expectedSettings, settings) + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + settings, err := configuration.Read(tc.testFile, "./testdata") + require.NoError(t, err) + require.Equal(t, tc.expectedSettings, settings) + }) + } } diff --git a/internal/configuration/testdata/multiple-nginx-hosts.yaml b/internal/configuration/testdata/multiple-nginx-hosts.yaml new file mode 100644 index 00000000..2235c1ae --- /dev/null +++ b/internal/configuration/testdata/multiple-nginx-hosts.yaml @@ -0,0 +1,11 @@ +ca-certificate: "fakeCAKey" +client-certificate: "fakeCertKey" +log-level: "warn" +nginx-hosts: ["https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"] +tls-mode: "no-tls" +service-annotation-match: "fakeServiceMatch" +creationTimestamp: "2024-09-04T17:59:20Z" +name: "nlk-config" +namespace: "nlk" +resourceVersion: "5909" +uid: "66d49974-49d6-4ad8-8135-dcebda7b5c9e" diff --git a/internal/configuration/testdata/one-nginx-host.yaml b/internal/configuration/testdata/one-nginx-host.yaml new file mode 100644 index 00000000..f05d81e0 --- /dev/null +++ b/internal/configuration/testdata/one-nginx-host.yaml @@ -0,0 +1,11 @@ +ca-certificate: "fakeCAKey" +client-certificate: "fakeCertKey" +log-level: "warn" +nginx-hosts: "https://10.0.0.1:9000/api" +tls-mode: "no-tls" +service-annotation-match: "fakeServiceMatch" +creationTimestamp: "2024-09-04T17:59:20Z" +name: "nlk-config" +namespace: "nlk" +resourceVersion: "5909" +uid: "66d49974-49d6-4ad8-8135-dcebda7b5c9e" diff --git a/internal/configuration/testdata/test.yaml b/internal/configuration/testdata/test.yaml deleted file mode 100644 index 717dcdbe..00000000 --- a/internal/configuration/testdata/test.yaml +++ /dev/null @@ -1,11 +0,0 @@ -ca-certificate: fakeCAKey -client-certificate: fakeCertKey -log-level: warn -nginx-hosts: https://10.0.0.1:9000/api -tls-mode: no-tls -service-annotation-match: fakeServiceMatch -creationTimestamp: "2024-09-04T17:59:20Z" -name: nlk-config -namespace: nlk -resourceVersion: "5909" -uid: 66d49974-49d6-4ad8-8135-dcebda7b5c9e diff --git a/version b/version index 781dcb07..65087b4f 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.3 +1.1.4 From 79f8c5b9a5e6a03949da1b9c57f84fe9713c6183 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 30 Jun 2025 14:20:12 -0600 Subject: [PATCH 105/110] NLB-6295 Simplified user configuration of TLS modes The biggest change here is to remove most the TLS modes to enable mTLS and self-signed certificates. Product decided that this was too complex and there was not enough user demand for most of these options. We decided to pare down the code and remove tests that were no longer well maintained. The remaining configuration allows users to toggle a single switch: whether to make the http client verify the NGINX host's certificate chain and host name if https is being used. If the user wishes to enable https with self-signed certs they can use the "skip-verify-tls" setting to allow this. The default behavior is to perform this verification. We are maintaining the deprecated "no-tls" and "ca-tls" inputs for NGINXaaS backwards comptability reasons. The "no-tls" setting name was highly misleading, because all it did was disable TLS verification: it DID NOT disable TLS altogether in https mode. Similarly, the "ca-tls" setting did not enable TLS itself. TLS is enabled by default when the URL of the NGINX host includes the https protocol. The user setting merely enforced the verification of the certificate chain and host as described above. --- cmd/certificates-test-harness/doc.go | 11 - cmd/certificates-test-harness/main.go | 81 ---- cmd/tls-config-factory-test-harness/doc.go | 1 - cmd/tls-config-factory-test-harness/main.go | 236 ---------- internal/authentication/doc.go | 10 - internal/authentication/factory.go | 123 ----- internal/authentication/factory_test.go | 442 ------------------ internal/certification/certificates.go | 215 --------- internal/certification/certificates_test.go | 232 --------- internal/certification/doc.go | 10 - internal/communication/factory.go | 33 +- internal/communication/factory_test.go | 15 +- internal/communication/roundtripper_test.go | 12 +- internal/configuration/configuration_test.go | 60 ++- internal/configuration/settings.go | 39 +- .../testdata/one-nginx-host.yaml | 1 - internal/configuration/tlsmodes.go | 48 +- internal/configuration/tlsmodes_test.go | 76 --- internal/core/secret_bytes.go | 21 - internal/core/secret_bytes_test.go | 33 -- internal/synchronization/synchronizer.go | 3 +- 21 files changed, 95 insertions(+), 1607 deletions(-) delete mode 100644 cmd/certificates-test-harness/doc.go delete mode 100644 cmd/certificates-test-harness/main.go delete mode 100644 cmd/tls-config-factory-test-harness/doc.go delete mode 100644 cmd/tls-config-factory-test-harness/main.go delete mode 100644 internal/authentication/doc.go delete mode 100644 internal/authentication/factory.go delete mode 100644 internal/authentication/factory_test.go delete mode 100644 internal/certification/certificates.go delete mode 100644 internal/certification/certificates_test.go delete mode 100644 internal/certification/doc.go delete mode 100644 internal/configuration/tlsmodes_test.go delete mode 100644 internal/core/secret_bytes.go delete mode 100644 internal/core/secret_bytes_test.go diff --git a/cmd/certificates-test-harness/doc.go b/cmd/certificates-test-harness/doc.go deleted file mode 100644 index 2d76fd59..00000000 --- a/cmd/certificates-test-harness/doc.go +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package certificates_test_harness includes functionality boostrap -and test the certification.Certificates implplementation. -*/ - -package main diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go deleted file mode 100644 index 898c3a39..00000000 --- a/cmd/certificates-test-harness/main.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" -) - -func main() { - handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) - logger := slog.New(handler) - slog.SetDefault(logger) - err := run() - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } -} - -func run() error { - slog.Info("certificates-test-harness::run") - - ctx := context.Background() - var err error - - k8sClient, err := buildKubernetesClient() - if err != nil { - return fmt.Errorf(`error building a Kubernetes client: %w`, err) - } - - certificates := certification.NewCertificates(k8sClient, nil) - - err = certificates.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing certificates: %w`, err) - } - - go certificates.Run(ctx) //nolint:errcheck - - <-ctx.Done() - return nil -} - -func buildKubernetesClient() (*kubernetes.Clientset, error) { - slog.Debug("Watcher::buildKubernetesClient") - - var kubeconfig *string - var k8sConfig *rest.Config - - k8sConfig, err := rest.InClusterConfig() - if errors.Is(err, rest.ErrNotInCluster) { - if home := homedir.HomeDir(); home != "" { - path := filepath.Join(home, ".kube", "config") - kubeconfig = &path - - k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - return nil, fmt.Errorf(`error occurred building the kubeconfig: %w`, err) - } - } else { - return nil, fmt.Errorf(`not running in a Cluster: %w`, err) - } - } else if err != nil { - return nil, fmt.Errorf(`error occurred getting the Cluster config: %w`, err) - } - - client, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - return nil, fmt.Errorf(`error occurred creating a client: %w`, err) - } - return client, nil -} diff --git a/cmd/tls-config-factory-test-harness/doc.go b/cmd/tls-config-factory-test-harness/doc.go deleted file mode 100644 index 06ab7d0f..00000000 --- a/cmd/tls-config-factory-test-harness/doc.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go deleted file mode 100644 index 638b0bc1..00000000 --- a/cmd/tls-config-factory-test-harness/main.go +++ /dev/null @@ -1,236 +0,0 @@ -package main - -import ( - "bufio" - "log/slog" - "os" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" - ClientCertificateSecretKey = "nlk-tls-client-secret" -) - -type TLSConfiguration struct { - Description string - Settings configuration.Settings -} - -func main() { - handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) - logger := slog.New(handler) - slog.SetDefault(logger) - - configurations := buildConfigMap() - - for name, settings := range configurations { - slog.Info("\033[H\033[2J") - - slog.Info("\n\n\t*** Building TLS config\n\n", "name", name) - - tlsConfig, err := authentication.NewTLSConfig(settings.Settings) - if err != nil { - panic(err) - } - - rootCaCount := 0 - certificateCount := 0 - - if tlsConfig.RootCAs != nil { - rootCaCount = len(tlsConfig.RootCAs.Subjects()) //nolint:staticcheck - } - - if tlsConfig.Certificates != nil { - certificateCount = len(tlsConfig.Certificates) - } - - slog.Info("Successfully built TLS config", - "description", settings.Description, - "rootCaCount", rootCaCount, - "certificateCount", certificateCount, - ) - - _, _ = bufio.NewReader(os.Stdin).ReadBytes('\n') - } - - slog.Info("\033[H\033[2J") - slog.Info("\n\n\t*** All done! ***\n\n") -} - -func buildConfigMap() map[string]TLSConfiguration { - configurations := make(map[string]TLSConfiguration) - - configurations["ss-tls"] = TLSConfiguration{ - Description: "Self-signed TLS requires just a CA certificate", - Settings: ssTLSConfig(), - } - - configurations["ss-mtls"] = TLSConfiguration{ - Description: "Self-signed mTLS requires a CA certificate and a client certificate", - Settings: ssMtlsConfig(), - } - - configurations["ca-tls"] = TLSConfiguration{ - Description: "CA TLS requires no certificates", - Settings: caTLSConfig(), - } - - configurations["ca-mtls"] = TLSConfiguration{ - Description: "CA mTLS requires a client certificate", - Settings: caMtlsConfig(), - } - - return configurations -} - -func ssTLSConfig() configuration.Settings { - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - - return configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } -} - -func ssMtlsConfig() configuration.Settings { - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - - return configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } -} - -func caTLSConfig() configuration.Settings { - return configuration.Settings{ - TLSMode: configuration.CertificateAuthorityTLS, - } -} - -func caMtlsConfig() configuration.Settings { - certs := make(map[string]map[string]core.SecretBytes) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - - return configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: certificates, - } -} - -func caCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIDTzCCAjcCFA4Zdj3E9TdjOP48eBRDGRLfkj7CMA0GCSqGSIb3DQEBCwUAMGQx -CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0 -dGxlMQ4wDAYDVQQKDAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFu -Y2VzMB4XDTIzMDkyOTE3MTY1MVoXDTIzMTAyOTE3MTY1MVowZDELMAkGA1UEBhMC -VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDjAMBgNV -BAoMBU5HSU5YMR4wHAYDVQQLDBVDb21tdW5pdHkgJiBBbGxpYW5jZXMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwlI4ZvJ/6hvqULFVL+1ZSRDTPQ48P -umehJhPz6xPhC9UkeTe2FZxm2Rsi1I5QXm/bTG2OcX775jgXzae9NQjctxwrz4Ks -LOWUvRkkfhQR67xk0Noux76/9GWGnB+Fapn54tlWql6uHQfOu1y7MCRkZ27zHbkk -lq4Oa2RmX8rIyECWgbTyL0kETBVJU8bYORQ5JjhRlz08inq3PggY8blrehIetrWN -dw+gzcqdvAI2uSCodHTHM/77KipnYmPiSiDjSDRlXdxTG8JnyIB78IoH/sw6RyBm -CvVa3ytvKziXAvbBoXq5On5WmMRF97p/MmBc53ExMuDZjA4fisnViS0PAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAJeoa2P59zopLjBInx/DnWn1N1CmFLb0ejKxG2jh -cOw15Sx40O0XrtrAto38iu4R/bkBeNCSUILlT+A3uYDila92Dayvls58WyIT3meD -G6+Sx/QDF69+4AXpVy9mQ+hxcofpFA32+GOMXwmk2OrAcdSkkGSBhZXgvTpQ64dl -xSiQ5EQW/K8LoBoEOXfjIZJNPORgKn5MI09AY7/47ycKDKTUU2yO8AtIHYKttw0x -kfIg7QOdo1F9IXVpGjJI7ynyrgsCEYxMoDyH42Dq84eKgrUFLEXemEz8hgdFgK41 -0eUYhAtzWHbRPBp+U/34CQoZ5ChNFp2YipvtXrzKE8KLkuM= ------END CERTIFICATE----- -` -} - -func clientCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// clientKeyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientKeyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} - -func buildClientCertificateEntry(keyPEM, certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - certification.CertificateKeyKey: core.SecretBytes([]byte(keyPEM)), - } -} - -func buildCaCertificateEntry(certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - } -} diff --git a/internal/authentication/doc.go b/internal/authentication/doc.go deleted file mode 100644 index 109255ed..00000000 --- a/internal/authentication/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package authentication includes functionality to secure communications between NLK and NGINX Plus hosts. -*/ - -package authentication diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go deleted file mode 100644 index c0664f65..00000000 --- a/internal/authentication/factory.go +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - * - * Factory for creating tls.Config objects based on the provided `tls-mode`. - */ - -package authentication - -import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" - "log/slog" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" -) - -func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { - slog.Debug("authentication::NewTLSConfig Creating TLS config", "mode", settings.TLSMode) - switch settings.TLSMode { - - case configuration.NoTLS: - return buildBasicTLSConfig(true), nil - - case configuration.SelfSignedTLS: // needs ca cert - return buildSelfSignedTLSConfig(settings.Certificates) - - case configuration.SelfSignedMutualTLS: // needs ca cert and client cert - return buildSelfSignedMtlsConfig(settings.Certificates) - - case configuration.CertificateAuthorityTLS: // needs nothing - return buildBasicTLSConfig(false), nil - - case configuration.CertificateAuthorityMutualTLS: // needs client cert - return buildCATLSConfig(settings.Certificates) - - default: - return nil, fmt.Errorf("unknown TLS mode: %s", settings.TLSMode) - } -} - -func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - slog.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") - certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) - if err != nil { - return nil, err - } - - //nolint:gosec - return &tls.Config{ - InsecureSkipVerify: false, - RootCAs: certPool, - }, nil -} - -func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - slog.Debug("authentication::buildSelfSignedMtlsConfig Building self-signed mTLS config") - certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) - if err != nil { - return nil, err - } - - certificate, err := buildCertificates(certificates.GetClientCertificate()) - if err != nil { - return nil, err - } - slog.Debug("buildSelfSignedMtlsConfig Certificate", "certificate", certificate) - - //nolint:gosec - return &tls.Config{ - InsecureSkipVerify: false, - RootCAs: certPool, - ClientAuth: tls.RequireAndVerifyClientCert, - Certificates: []tls.Certificate{certificate}, - }, nil -} - -func buildBasicTLSConfig(skipVerify bool) *tls.Config { - slog.Debug("authentication::buildBasicTLSConfig", slog.Bool("skipVerify", skipVerify)) - return &tls.Config{ - InsecureSkipVerify: skipVerify, //nolint:gosec - } -} - -func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - slog.Debug("authentication::buildCATLSConfig") - certificate, err := buildCertificates(certificates.GetClientCertificate()) - if err != nil { - return nil, err - } - - //nolint:gosec - return &tls.Config{ - InsecureSkipVerify: false, - Certificates: []tls.Certificate{certificate}, - }, nil -} - -func buildCertificates(privateKeyPEM []byte, certificatePEM []byte) (tls.Certificate, error) { - slog.Debug("authentication::buildCertificates") - return tls.X509KeyPair(certificatePEM, privateKeyPEM) -} - -func buildCaCertificatePool(caCert []byte) (*x509.CertPool, error) { - slog.Debug("authentication::buildCaCertificatePool") - block, _ := pem.Decode(caCert) - if block == nil { - return nil, fmt.Errorf("failed to decode PEM block containing CA certificate") - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("error parsing certificate: %w", err) - } - - caCertPool := x509.NewCertPool() - caCertPool.AddCert(cert) - - return caCertPool, nil -} diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go deleted file mode 100644 index 47776074..00000000 --- a/internal/authentication/factory_test.go +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package authentication - -import ( - "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" - ClientCertificateSecretKey = "nlk-tls-client-secret" -) - -func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { - t.Parallel() - - tlsConfig, err := NewTLSConfig(configuration.Settings{}) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != true { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be true`) - } -} - -func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) != 0 { - t.Fatalf(`tlsConfig.Certificates should be empty`) - } - - if tlsConfig.RootCAs == nil { - t.Fatalf(`tlsConfig.RootCAs should not be nil`) - } -} - -func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "failed to decode PEM block containing CA certificate" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "error parsing certificate: x509: inner and outer signature algorithm identifiers don't match" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) == 0 { - t.Fatalf(`tlsConfig.Certificates should not be empty`) - } - - if tlsConfig.RootCAs == nil { - t.Fatalf(`tlsConfig.RootCAs should not be nil`) - } -} - -func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "failed to decode PEM block containing CA certificate" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "tls: failed to find any PEM data in certificate input" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_CaTlsMode(t *testing.T) { - t.Parallel() - settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityTLS, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) != 0 { - t.Fatalf(`tlsConfig.Certificates should be empty`) - } - - if tlsConfig.RootCAs != nil { - t.Fatalf(`tlsConfig.RootCAs should be nil`) - } -} - -func TestTlsFactory_CaMtlsMode(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: certificates, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) == 0 { - t.Fatalf(`tlsConfig.Certificates should not be empty`) - } - - if tlsConfig.RootCAs != nil { - t.Fatalf(`tlsConfig.RootCAs should be nil`) - } -} - -func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "tls: failed to find any PEM data in certificate input" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -// caCertificatePEM returns a PEM-encoded CA certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func caCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIDTzCCAjcCFA4Zdj3E9TdjOP48eBRDGRLfkj7CMA0GCSqGSIb3DQEBCwUAMGQx -CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0 -dGxlMQ4wDAYDVQQKDAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFu -Y2VzMB4XDTIzMDkyOTE3MTY1MVoXDTIzMTAyOTE3MTY1MVowZDELMAkGA1UEBhMC -VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDjAMBgNV -BAoMBU5HSU5YMR4wHAYDVQQLDBVDb21tdW5pdHkgJiBBbGxpYW5jZXMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwlI4ZvJ/6hvqULFVL+1ZSRDTPQ48P -umehJhPz6xPhC9UkeTe2FZxm2Rsi1I5QXm/bTG2OcX775jgXzae9NQjctxwrz4Ks -LOWUvRkkfhQR67xk0Noux76/9GWGnB+Fapn54tlWql6uHQfOu1y7MCRkZ27zHbkk -lq4Oa2RmX8rIyECWgbTyL0kETBVJU8bYORQ5JjhRlz08inq3PggY8blrehIetrWN -dw+gzcqdvAI2uSCodHTHM/77KipnYmPiSiDjSDRlXdxTG8JnyIB78IoH/sw6RyBm -CvVa3ytvKziXAvbBoXq5On5WmMRF97p/MmBc53ExMuDZjA4fisnViS0PAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAJeoa2P59zopLjBInx/DnWn1N1CmFLb0ejKxG2jh -cOw15Sx40O0XrtrAto38iu4R/bkBeNCSUILlT+A3uYDila92Dayvls58WyIT3meD -G6+Sx/QDF69+4AXpVy9mQ+hxcofpFA32+GOMXwmk2OrAcdSkkGSBhZXgvTpQ64dl -xSiQ5EQW/K8LoBoEOXfjIZJNPORgKn5MI09AY7/47ycKDKTUU2yO8AtIHYKttw0x -kfIg7QOdo1F9IXVpGjJI7ynyrgsCEYxMoDyH42Dq84eKgrUFLEXemEz8hgdFgK41 -0eUYhAtzWHbRPBp+U/34CQoZ5ChNFp2YipvtXrzKE8KLkuM= ------END CERTIFICATE----- -` -} - -func invalidCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIClzCCAX+gAwIBAgIJAIfPhC0RG6CwMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNV -BAMMDm9pbCBhdXRob3JpdHkwHhcNMjAwNDA3MTUwOTU1WhcNMjEwNDA2MTUwOTU1 -WjBMMSAwHgYDVQQLDBd5b3VuZy1jaGFsbGVuZ2UgdGVzdCBjb25zdW1lczEfMB0G -A1UECwwWc28wMS5jb3Jwb3JhdGlvbnNvY2lhbDEhMB8GA1UEAwwYc29tMS5jb3Jw -b3JhdGlvbnNvY2lhbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB -AQDGRX31uzy+yLUOz7wOJHHm2dzrDgUbC6RZDjURvZxyt2Zi5wYWsEB5r5YhN7L0 -y1R9f+MGwNITIz9nYZuU/PLFOvzF5qX7A8TbdgjZEqvXe2NZ9J2z3iWvYQLN8Py3 -nv/Y6wadgXEBRCNNuIg/bQ9XuOr9tfB6j4Ut1GLU0eIlV/L3Rf9Y6SgrAl+58ITj -Wrg3Js/Wz3J2JU4qBD8U4I3XvUyfnX2SAG8Llm4KBuYz7g63Iu05s6RnmG+Xhu2T -5f2DWZUeATWbAlUW/M4NLO1+5H0gOr0TGulETQ6uElMchT7s/H6Rv1CV+CNCCgEI -adRjWJq9yQ+KrE+urSMCXu8XAgMBAAGjUzBRMB0GA1UdDgQWBBRb40pKGU4lNvqB -1f5Mz3t0N/K3hzAfBgNVHSMEGDAWgBRb40pKGU4lNvqB1f5Mz3t0N/K3hzAPBgNV -HREECDAGhwQAAAAAAAAwCgYIKoZIzj0EAwIDSAAwRQIhAP3ST/mXyRXsU2ciRoE -gE6trllODFY+9FgT6UbF2TwzAiAAuaUxtbk6uXLqtD5NtXqOQf0Ckg8GQxc5V1G2 -9PqTXQ== ------END CERTIFICATE----- -` -} - -// Yoinked from https://cs.opensource.google/go/go/+/refs/tags/go1.21.1:src/crypto/x509/x509_test.go, line 3385 -// This allows the `buildCaCertificatePool(...)` --> `x509.ParseCertificate(...)` call error branch to be covered. -func invalidCertificateDataPEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIBBzCBrqADAgECAgEAMAoGCCqGSM49BAMCMAAwIhgPMDAwMTAxMDEwMDAwMDBa -GA8wMDAxMDEwMTAwMDAwMFowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABOqV -EDuVXxwZgIU3+dOwv1SsMu0xuV48hf7xmK8n7sAMYgllB+96DnPqBeboJj4snYnx -0AcE0PDVQ1l4Z3YXsQWjFTATMBEGA1UdEQEB/wQHMAWCA2FzZDAKBggqhkjOPQQD -AwNIADBFAiBi1jz/T2HT5nAfrD7zsgR+68qh7Erc6Q4qlxYBOgKG4QIhAOtjIn+Q -tA+bq+55P3ntxTOVRq0nv1mwnkjwt9cQR9Fn ------END CERTIFICATE----- -` -} - -// clientCertificatePEM returns a PEM-encoded client certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// clientKeyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientKeyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} - -func buildClientCertificateEntry(keyPEM, certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - certification.CertificateKeyKey: core.SecretBytes([]byte(keyPEM)), - } -} - -func buildCaCertificateEntry(certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - } -} diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go deleted file mode 100644 index 86b93206..00000000 --- a/internal/certification/certificates.go +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - * - * Establishes a Watcher for the Kubernetes Secrets that contain the various certificates - * and keys used to generate a tls.Config object; - * exposes the certificates and keys. - */ - -package certification - -import ( - "context" - "fmt" - "log/slog" - "sync" - - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - // SecretsNamespace is the value used to filter the Secrets Resource in the Informer. - SecretsNamespace = "nlk" - - // CertificateKey is the key for the certificate in the Secret. - CertificateKey = "tls.crt" - - // CertificateKeyKey is the key for the certificate key in the Secret. - CertificateKeyKey = "tls.key" -) - -type Certificates struct { - mu sync.Mutex // guards Certificates - certificates map[string]map[string]core.SecretBytes - - // CaCertificateSecretKey is the name of the Secret that contains the Certificate Authority certificate. - CaCertificateSecretKey string - - // ClientCertificateSecretKey is the name of the Secret that contains the Client certificate. - ClientCertificateSecretKey string - - // informer is the SharedInformer used to watch for changes to the Secrets . - informer cache.SharedInformer - - // K8sClient is the Kubernetes client used to communicate with the Kubernetes API. - k8sClient kubernetes.Interface - - // eventHandlerRegistration is the object used to track the event handlers with the SharedInformer. - eventHandlerRegistration cache.ResourceEventHandlerRegistration -} - -// NewCertificates factory method that returns a new Certificates object. -func NewCertificates( - k8sClient kubernetes.Interface, certificates map[string]map[string]core.SecretBytes, -) *Certificates { - return &Certificates{ - k8sClient: k8sClient, - certificates: certificates, - } -} - -// GetCACertificate returns the Certificate Authority certificate. -func (c *Certificates) GetCACertificate() core.SecretBytes { - c.mu.Lock() - defer c.mu.Unlock() - - bytes := c.certificates[c.CaCertificateSecretKey][CertificateKey] - - return bytes -} - -// GetClientCertificate returns the Client certificate and key. -func (c *Certificates) GetClientCertificate() (core.SecretBytes, core.SecretBytes) { - c.mu.Lock() - defer c.mu.Unlock() - - keyBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKeyKey] - certificateBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKey] - - return keyBytes, certificateBytes -} - -// Initialize initializes the Certificates object. Sets up a SharedInformer for the Secrets Resource. -func (c *Certificates) Initialize() error { - slog.Info("Certificates::Initialize") - - var err error - - c.mu.Lock() - c.certificates = make(map[string]map[string]core.SecretBytes) - c.mu.Unlock() - - informer := c.buildInformer() - - c.informer = informer - - err = c.initializeEventHandlers() - if err != nil { - return fmt.Errorf(`error occurred initializing event handlers: %w`, err) - } - - return nil -} - -// Run starts the SharedInformer. -func (c *Certificates) Run(ctx context.Context) error { - slog.Info("Certificates::Run") - - if c.informer == nil { - return fmt.Errorf(`initialize must be called before Run`) - } - - c.informer.Run(ctx.Done()) - - <-ctx.Done() - - return nil -} - -func (c *Certificates) buildInformer() cache.SharedInformer { - slog.Debug("Certificates::buildInformer") - - options := informers.WithNamespace(SecretsNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(c.k8sClient, 0, options) - informer := factory.Core().V1().Secrets().Informer() - - return informer -} - -func (c *Certificates) initializeEventHandlers() error { - slog.Debug("Certificates::initializeEventHandlers") - - var err error - - handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: c.handleAddEvent, - DeleteFunc: c.handleDeleteEvent, - UpdateFunc: c.handleUpdateEvent, - } - - c.eventHandlerRegistration, err = c.informer.AddEventHandler(handlers) - if err != nil { - return fmt.Errorf(`error occurred registering event handlers: %w`, err) - } - - return nil -} - -func (c *Certificates) handleAddEvent(obj interface{}) { - slog.Debug("Certificates::handleAddEvent") - - secret, ok := obj.(*corev1.Secret) - if !ok { - slog.Error("Certificates::handleAddEvent: unable to cast object to Secret") - return - } - - c.mu.Lock() - defer c.mu.Unlock() - - c.certificates[secret.Name] = map[string]core.SecretBytes{} - - // Input from the secret comes in the form - // tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVCVEN... - // tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0l... - // Where the keys are `tls.crt` and `tls.key` and the values are []byte - for k, v := range secret.Data { - c.certificates[secret.Name][k] = core.SecretBytes(v) - } - - slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.certificates))) -} - -func (c *Certificates) handleDeleteEvent(obj interface{}) { - slog.Debug("Certificates::handleDeleteEvent") - - secret, ok := obj.(*corev1.Secret) - if !ok { - slog.Error("Certificates::handleDeleteEvent: unable to cast object to Secret") - return - } - - c.mu.Lock() - defer c.mu.Unlock() - - if c.certificates[secret.Name] != nil { - delete(c.certificates, secret.Name) - } - - slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.certificates))) -} - -func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { - slog.Debug("Certificates::handleUpdateEvent") - - secret, ok := newValue.(*corev1.Secret) - if !ok { - slog.Error("Certificates::handleUpdateEvent: unable to cast object to Secret") - return - } - - c.mu.Lock() - defer c.mu.Unlock() - - for k, v := range secret.Data { - c.certificates[secret.Name][k] = v - } - - slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.certificates))) -} diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go deleted file mode 100644 index 7d2363b2..00000000 --- a/internal/certification/certificates_test.go +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package certification - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" -) - -func TestNewCertificate(t *testing.T) { - t.Parallel() - - certificates := NewCertificates(nil, nil) - - if certificates == nil { - t.Fatalf(`certificates should not be nil`) - } -} - -func TestCertificates_Initialize(t *testing.T) { - t.Parallel() - certificates := NewCertificates(nil, nil) - - err := certificates.Initialize() - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } -} - -func TestCertificates_RunWithoutInitialize(t *testing.T) { - t.Parallel() - certificates := NewCertificates(nil, nil) - - err := certificates.Run(context.Background()) - if err == nil { - t.Fatalf(`Expected error`) - } - - if err.Error() != `initialize must be called before Run` { - t.Fatalf(`Unexpected error: %v`, err) - } -} - -func TestCertificates_EmptyCertificates(t *testing.T) { - t.Parallel() - certificates := NewCertificates(nil, nil) - - err := certificates.Initialize() - if err != nil { - t.Fatalf(`error Initializing Certificates: %v`, err) - } - - caBytes := certificates.GetCACertificate() - if caBytes != nil { - t.Fatalf(`Expected nil CA certificate`) - } - - clientKey, clientCert := certificates.GetClientCertificate() - if clientKey != nil { - t.Fatalf(`Expected nil client key`) - } - if clientCert != nil { - t.Fatalf(`Expected nil client certificate`) - } -} - -func TestCertificates_ExerciseHandlers(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - k8sClient := fake.NewSimpleClientset() - - certificates := NewCertificates(k8sClient, nil) - - _ = certificates.Initialize() - - certificates.CaCertificateSecretKey = CaCertificateSecretKey - - //nolint:govet,staticcheck - go func() { - err := certificates.Run(context.Background()) - assert.NoError(t, err, "expected no error running certificates") - }() - - cache.WaitForCacheSync(ctx.Done(), certificates.informer.HasSynced) - - secret := buildSecret() - - /* -- Test Create -- */ - - created, err := k8sClient.CoreV1().Secrets(SecretsNamespace).Create(ctx, secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf(`error creating the Secret: %v`, err) - } - - if created.Name != secret.Name { - t.Fatalf(`Expected name %v, got %v`, secret.Name, created.Name) - } - - time.Sleep(2 * time.Second) - - caBytes := certificates.GetCACertificate() - if caBytes == nil { - t.Fatalf(`Expected non-nil CA certificate`) - } - - /* -- Test Update -- */ - - secret.Labels = map[string]string{"updated": "true"} - _, err = k8sClient.CoreV1().Secrets(SecretsNamespace).Update(ctx, secret, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf(`error updating the Secret: %v`, err) - } - - time.Sleep(2 * time.Second) - - caBytes = certificates.GetCACertificate() - if caBytes == nil { - t.Fatalf(`Expected non-nil CA certificate`) - } - - /* -- Test Delete -- */ - - err = k8sClient.CoreV1().Secrets(SecretsNamespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf(`error deleting the Secret: %v`, err) - } - - time.Sleep(2 * time.Second) - - caBytes = certificates.GetCACertificate() - if caBytes != nil { - t.Fatalf(`Expected nil CA certificate, got: %v`, caBytes) - } -} - -func buildSecret() *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: CaCertificateSecretKey, - Namespace: SecretsNamespace, - }, - Data: map[string][]byte{ - CertificateKey: []byte(certificatePEM()), - CertificateKeyKey: []byte(keyPEM()), - }, - Type: corev1.SecretTypeTLS, - } -} - -// certificatePEM returns a PEM-encoded client certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func certificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// keyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func keyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} diff --git a/internal/certification/doc.go b/internal/certification/doc.go deleted file mode 100644 index 3388ea0a..00000000 --- a/internal/certification/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package certification includes functionality to access the Secrets containing the TLS Certificates. -*/ - -package certification diff --git a/internal/communication/factory.go b/internal/communication/factory.go index cf1bfcbe..2084d5cc 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -6,24 +6,19 @@ package communication import ( - "crypto/tls" "fmt" - "log/slog" netHttp "net/http" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" ) -// NewHTTPClient is a factory method to create a new Http Client with a default configuration. -// RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, -// the Headers are configured for JSON. -func NewHTTPClient(settings configuration.Settings) (*netHttp.Client, error) { - headers := NewHeaders(settings.APIKey) - tlsConfig := NewTLSConfig(settings) - transport := NewTransport(tlsConfig) +// NewHTTPClient is a factory method to create a new Http Client configured for +// working with NGINXaaS or the N+ api. If skipVerify is set to true, the http +// transport will skip TLS certificate verification. +func NewHTTPClient(apiKey string, skipVerify bool) (*netHttp.Client, error) { + headers := NewHeaders(apiKey) + transport := NewTransport(skipVerify) roundTripper := NewRoundTripper(headers, transport) return &netHttp.Client{ @@ -49,22 +44,10 @@ func NewHeaders(apiKey string) []string { return headers } -// NewTLSConfig is a factory method to create a new basic Tls Config. -// More attention should be given to the use of `InsecureSkipVerify: true`, as it is not recommended for production use. -func NewTLSConfig(settings configuration.Settings) *tls.Config { - tlsConfig, err := authentication.NewTLSConfig(settings) - if err != nil { - slog.Warn("Failed to create TLS config", "error", err) - return &tls.Config{InsecureSkipVerify: true} //nolint:gosec - } - - return tlsConfig -} - // NewTransport is a factory method to create a new basic Http Transport. -func NewTransport(config *tls.Config) *netHttp.Transport { +func NewTransport(skipVerify bool) *netHttp.Transport { transport := netHttp.DefaultTransport.(*netHttp.Transport).Clone() - transport.TLSClientConfig = config + transport.TLSClientConfig.InsecureSkipVerify = skipVerify return transport } diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 7562484f..8c637a8a 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -7,12 +7,14 @@ package communication import ( "testing" + + "github.com/stretchr/testify/require" ) func TestNewHTTPClient(t *testing.T) { t.Parallel() - client, err := NewHTTPClient(defaultSettings()) + client, err := NewHTTPClient("fakeKey", true) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -80,8 +82,7 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { func TestNewTransport(t *testing.T) { t.Parallel() - config := NewTLSConfig(defaultSettings()) - transport := NewTransport(config) + transport := NewTransport(false) if transport == nil { t.Fatalf(`transport should not be nil`) @@ -91,11 +92,5 @@ func TestNewTransport(t *testing.T) { t.Fatalf(`transport.TLSClientConfig should not be nil`) } - if transport.TLSClientConfig != config { - t.Fatalf(`transport.TLSClientConfig should be the same as config`) - } - - if !transport.TLSClientConfig.InsecureSkipVerify { - t.Fatalf(`transport.TLSClientConfig.InsecureSkipVerify should be true`) - } + require.False(t, transport.TLSClientConfig.InsecureSkipVerify) } diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index 55dea883..d00af173 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -9,15 +9,13 @@ import ( "bytes" netHttp "net/http" "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" ) func TestNewRoundTripper(t *testing.T) { t.Parallel() headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(defaultSettings())) + transport := NewTransport(true) roundTripper := NewRoundTripper(headers, transport) if roundTripper == nil { @@ -57,7 +55,7 @@ func TestRoundTripperRoundTrip(t *testing.T) { t.Parallel() headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(defaultSettings())) + transport := NewTransport(true) roundTripper := NewRoundTripper(headers, transport) request, err := NewRequest("GET", "http://example.com", nil) @@ -92,9 +90,3 @@ func NewRequest(method string, url string, body []byte) (*netHttp.Request, error return request, nil } - -func defaultSettings() configuration.Settings { - return configuration.Settings{ - TLSMode: configuration.NoTLS, - } -} diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go index fc419745..45764d6e 100644 --- a/internal/configuration/configuration_test.go +++ b/internal/configuration/configuration_test.go @@ -4,13 +4,12 @@ import ( "testing" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/stretchr/testify/require" ) -func TestConfiguration(t *testing.T) { +func TestConfiguration_Read(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -22,11 +21,7 @@ func TestConfiguration(t *testing.T) { expectedSettings: configuration.Settings{ LogLevel: "warn", NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, - TLSMode: configuration.NoTLS, - Certificates: &certification.Certificates{ - CaCertificateSecretKey: "fakeCAKey", - ClientCertificateSecretKey: "fakeCertKey", - }, + SkipVerifyTLS: false, Handler: configuration.HandlerSettings{ RetryCount: 5, Threads: 1, @@ -58,11 +53,7 @@ func TestConfiguration(t *testing.T) { expectedSettings: configuration.Settings{ LogLevel: "warn", NginxPlusHosts: []string{"https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"}, - TLSMode: configuration.NoTLS, - Certificates: &certification.Certificates{ - CaCertificateSecretKey: "fakeCAKey", - ClientCertificateSecretKey: "fakeCertKey", - }, + SkipVerifyTLS: true, Handler: configuration.HandlerSettings{ RetryCount: 5, Threads: 1, @@ -100,3 +91,48 @@ func TestConfiguration(t *testing.T) { }) } } + +func TestConfiguration_TLS(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + tlsMode string + expectedSkipVerifyTLS bool + expectedErr bool + }{ + "no input": { + tlsMode: "", + expectedSkipVerifyTLS: false, + }, + "no tls": { + tlsMode: "no-tls", + expectedSkipVerifyTLS: true, + }, + "skip verify tls": { + tlsMode: "skip-verify-tls", + expectedSkipVerifyTLS: true, + }, + "ca tls": { + tlsMode: "ca-tls", + expectedSkipVerifyTLS: false, + }, + "unexpected input": { + tlsMode: "unexpected-tls-mode", + expectedErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + skipVerifyTLS, err := configuration.ValidateTLSMode(tc.tlsMode) + if tc.expectedErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, tc.expectedSkipVerifyTLS, skipVerifyTLS) + }) + } +} diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index ebeacc46..75cec2e5 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -11,8 +11,6 @@ import ( "log/slog" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/spf13/viper" ) @@ -110,16 +108,12 @@ type Settings struct { // NginxPlusHosts is a list of Nginx Plus hosts that will be used to update the Border Servers. NginxPlusHosts []string - // TlsMode is the value used to determine which of the five TLS modes will be used to communicate - // with the Border Servers (see: ../../docs/tls/README.md). - TLSMode TLSMode + // SkipVerifyTLS determines whether the http client will skip TLS verification or not. + SkipVerifyTLS bool // APIKey is the api key used to authenticate with the dataplane API. APIKey string - // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. - Certificates *certification.Certificates - // Handler contains the configuration values needed by the Handler. Handler HandlerSettings @@ -144,11 +138,13 @@ func Read(configName, configPath string) (s Settings, err error) { return s, err } - tlsMode := NoTLS - if t, err := validateTLSMode(v.GetString("tls-mode")); err != nil { + skipVerifyTLS, err := ValidateTLSMode(v.GetString("tls-mode")) + if err != nil { slog.Error("could not validate tls mode", "error", err) - } else { - tlsMode = t + } + + if skipVerifyTLS { + slog.Warn("skipping TLS verification for NGINX hosts") } serviceAnnotation := DefaultServiceAnnotation @@ -159,12 +155,8 @@ func Read(configName, configPath string) (s Settings, err error) { return Settings{ LogLevel: v.GetString("log-level"), NginxPlusHosts: v.GetStringSlice("nginx-hosts"), - TLSMode: tlsMode, + SkipVerifyTLS: skipVerifyTLS, APIKey: base64.StdEncoding.EncodeToString([]byte(v.GetString("NGINXAAS_DATAPLANE_API_KEY"))), - Certificates: &certification.Certificates{ - CaCertificateSecretKey: v.GetString("ca-certificate"), - ClientCertificateSecretKey: v.GetString("client-certificate"), - }, Handler: HandlerSettings{ RetryCount: 5, Threads: 1, @@ -192,10 +184,15 @@ func Read(configName, configPath string) (s Settings, err error) { }, nil } -func validateTLSMode(tlsConfigMode string) (TLSMode, error) { - if tlsMode, tlsModeFound := TLSModeMap[tlsConfigMode]; tlsModeFound { - return tlsMode, nil +func ValidateTLSMode(tlsConfigMode string) (skipVerify bool, err error) { + if tlsConfigMode == "" { + return false, nil + } + + var tlsModeFound bool + if skipVerify, tlsModeFound = tlsModeMap[tlsConfigMode]; tlsModeFound { + return skipVerify, nil } - return NoTLS, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) + return false, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) } diff --git a/internal/configuration/testdata/one-nginx-host.yaml b/internal/configuration/testdata/one-nginx-host.yaml index f05d81e0..45e55eff 100644 --- a/internal/configuration/testdata/one-nginx-host.yaml +++ b/internal/configuration/testdata/one-nginx-host.yaml @@ -2,7 +2,6 @@ ca-certificate: "fakeCAKey" client-certificate: "fakeCertKey" log-level: "warn" nginx-hosts: "https://10.0.0.1:9000/api" -tls-mode: "no-tls" service-annotation-match: "fakeServiceMatch" creationTimestamp: "2024-09-04T17:59:20Z" name: "nlk-config" diff --git a/internal/configuration/tlsmodes.go b/internal/configuration/tlsmodes.go index 2f7271f2..e329047e 100644 --- a/internal/configuration/tlsmodes.go +++ b/internal/configuration/tlsmodes.go @@ -6,41 +6,19 @@ package configuration const ( - NoTLS TLSMode = iota - CertificateAuthorityTLS - CertificateAuthorityMutualTLS - SelfSignedTLS - SelfSignedMutualTLS + // NoTLS is deprecated as misleading. It is the same as SkipVerifyTLS. + NoTLS = "no-tls" + // SkipVerifyTLS causes the http client to skip verification of the NGINX + // host's certificate chain and host name. + SkipVerifyTLS = "skip-verify-tls" + // CertificateAuthorityTLS is deprecated as misleading. This is the same as + // the default behavior which is to verify the NGINX hosts's certificate + // chain and host name, if https is used. + CertificateAuthorityTLS = "ca-tls" ) -const ( - NoTLSString = "no-tls" - CertificateAuthorityTLSString = "ca-tls" - CertificateAuthorityMutualTLSString = "ca-mtls" - SelfSignedTLSString = "ss-tls" - SelfSignedMutualTLSString = "ss-mtls" -) - -type TLSMode int - -var TLSModeMap = map[string]TLSMode{ - NoTLSString: NoTLS, - CertificateAuthorityTLSString: CertificateAuthorityTLS, - CertificateAuthorityMutualTLSString: CertificateAuthorityMutualTLS, - SelfSignedTLSString: SelfSignedTLS, - SelfSignedMutualTLSString: SelfSignedMutualTLS, -} - -func (t TLSMode) String() string { - modes := []string{ - NoTLSString, - CertificateAuthorityTLSString, - CertificateAuthorityMutualTLSString, - SelfSignedTLSString, - SelfSignedMutualTLSString, - } - if t < NoTLS || t > SelfSignedMutualTLS { - return "" - } - return modes[t] +var tlsModeMap = map[string]bool{ + NoTLS: true, + SkipVerifyTLS: true, + CertificateAuthorityTLS: false, } diff --git a/internal/configuration/tlsmodes_test.go b/internal/configuration/tlsmodes_test.go deleted file mode 100644 index d849cd97..00000000 --- a/internal/configuration/tlsmodes_test.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package configuration - -import ( - "testing" -) - -func Test_String(t *testing.T) { - t.Parallel() - mode := NoTLS.String() - if mode != "no-tls" { - t.Errorf("Expected TLSModeNoTLS to be 'no-tls', got '%s'", mode) - } - - mode = CertificateAuthorityTLS.String() - if mode != "ca-tls" { - t.Errorf("Expected TLSModeCaTLS to be 'ca-tls', got '%s'", mode) - } - - mode = CertificateAuthorityMutualTLS.String() - if mode != "ca-mtls" { - t.Errorf("Expected TLSModeCaMTLS to be 'ca-mtls', got '%s'", mode) - } - - mode = SelfSignedTLS.String() - if mode != "ss-tls" { - t.Errorf("Expected TLSModeSsTLS to be 'ss-tls', got '%s'", mode) - } - - mode = SelfSignedMutualTLS.String() - if mode != "ss-mtls" { - t.Errorf("Expected TLSModeSsMTLS to be 'ss-mtls', got '%s',", mode) - } - - mode = TLSMode(5).String() - if mode != "" { - t.Errorf("Expected TLSMode(5) to be '', got '%s'", mode) - } -} - -func Test_TLSModeMap(t *testing.T) { - t.Parallel() - mode := TLSModeMap["no-tls"] - if mode != NoTLS { - t.Errorf("Expected TLSModeMap['no-tls'] to be TLSModeNoTLS, got '%d'", mode) - } - - mode = TLSModeMap["ca-tls"] - if mode != CertificateAuthorityTLS { - t.Errorf("Expected TLSModeMap['ca-tls'] to be TLSModeCaTLS, got '%d'", mode) - } - - mode = TLSModeMap["ca-mtls"] - if mode != CertificateAuthorityMutualTLS { - t.Errorf("Expected TLSModeMap['ca-mtls'] to be TLSModeCaMTLS, got '%d'", mode) - } - - mode = TLSModeMap["ss-tls"] - if mode != SelfSignedTLS { - t.Errorf("Expected TLSModeMap['ss-tls'] to be TLSModeSsTLS, got '%d'", mode) - } - - mode = TLSModeMap["ss-mtls"] - if mode != SelfSignedMutualTLS { - t.Errorf("Expected TLSModeMap['ss-mtls'] to be TLSModeSsMTLS, got '%d'", mode) - } - - mode = TLSModeMap["invalid"] - if mode != TLSMode(0) { - t.Errorf("Expected TLSModeMap['invalid'] to be TLSMode(0), got '%d'", mode) - } -} diff --git a/internal/core/secret_bytes.go b/internal/core/secret_bytes.go deleted file mode 100644 index 0bbc3bf1..00000000 --- a/internal/core/secret_bytes.go +++ /dev/null @@ -1,21 +0,0 @@ -package core - -import ( - "encoding/json" -) - -// Wraps byte slices which potentially could contain -// sensitive data that should not be output to the logs. -// This will output [REDACTED] if attempts are made -// to print this type in logs, serialize to JSON, or -// otherwise convert it to a string. -// Usage: core.SecretBytes(myByteSlice) -type SecretBytes []byte - -func (sb SecretBytes) String() string { - return "[REDACTED]" -} - -func (sb SecretBytes) MarshalJSON() ([]byte, error) { - return json.Marshal("[REDACTED]") -} diff --git a/internal/core/secret_bytes_test.go b/internal/core/secret_bytes_test.go deleted file mode 100644 index 5e6bc3ff..00000000 --- a/internal/core/secret_bytes_test.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package core - -import ( - "encoding/json" - "fmt" - "testing" -) - -func TestSecretBytesToString(t *testing.T) { - t.Parallel() - sensitive := SecretBytes([]byte("If you can see this we have a problem")) - - expected := "foo [REDACTED] bar" - result := fmt.Sprintf("foo %v bar", sensitive) - if result != expected { - t.Errorf("Expected %s, got %s", expected, result) - } -} - -func TestSecretBytesToJSON(t *testing.T) { - t.Parallel() - sensitive, _ := json.Marshal(SecretBytes([]byte("If you can see this we have a problem"))) - expected := `foo "[REDACTED]" bar` - result := fmt.Sprintf("foo %v bar", string(sensitive)) - if result != expected { - t.Errorf("Expected %s, got %s", expected, result) - } -} diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 80faaa31..bbae5e1d 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -126,8 +126,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica slog.Debug(`Synchronizer::buildBorderClient`) var err error - - httpClient, err := communication.NewHTTPClient(s.settings) + httpClient, err := communication.NewHTTPClient(s.settings.APIKey, s.settings.SkipVerifyTLS) if err != nil { return nil, fmt.Errorf(`error creating HTTP client: %v`, err) } From ab3886378e9b173785edf272484fedfeaf67551c Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 13 Mar 2025 16:30:52 -0600 Subject: [PATCH 106/110] NLB-6295 Bumped version to 1.2.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 65087b4f..26aaba0e 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.4 +1.2.0 From e431b7b0fceb9a7a7707e8525fb4ac6d05cd2ae7 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 1 Jul 2025 11:51:34 -0600 Subject: [PATCH 107/110] Updated go version to 1.24.4 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c09a4b41..b49ed5c2 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.24.3 +go 1.24.4 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 From d9b8e1ef74ee7618638125fbc1247feb2c90d5a7 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 2 Jul 2025 14:35:47 -0600 Subject: [PATCH 108/110] NLB-6754 When deleting upstream servers handle upstream not found error Now that the plus go client allows users to check the http status code of the error, handle the upstream not found case by doing nothing. --- go.mod | 4 ++-- go.sum | 8 ++++---- internal/synchronization/synchronizer.go | 15 ++++++++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b49ed5c2..6a78fe2f 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ module github.com/nginxinc/kubernetes-nginx-ingress go 1.24.4 require ( - github.com/nginx/nginx-plus-go-client/v2 v2.3.0 + github.com/nginx/nginx-plus-go-client/v2 v2.4.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/sync v0.12.0 + golang.org/x/sync v0.13.0 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 diff --git a/go.sum b/go.sum index 80d0e9d4..8cf5ed86 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginx/nginx-plus-go-client/v2 v2.3.0 h1:ciKh1lwadNzUaOGjLcKWu/BGigASxU6p7v/6US71fhA= -github.com/nginx/nginx-plus-go-client/v2 v2.3.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= +github.com/nginx/nginx-plus-go-client/v2 v2.4.0 h1:4c7V57CLCZUOxQCUcS9G8a5MClzdmxByBm+f4zKMzAY= +github.com/nginx/nginx-plus-go-client/v2 v2.4.0/go.mod h1:P+dIP2oKYzFoyf/zlLWQa8Sf+fHb+CclOKzxAjxpvug= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= @@ -135,8 +135,8 @@ golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index bbae5e1d..8f0fff60 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -10,7 +10,7 @@ import ( "errors" "fmt" "log/slog" - "strings" + "net/http" "time" nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" @@ -36,6 +36,13 @@ type Interface interface { ShutDown() } +// StatusError is a wrapper for errors from the go plus client that contain http +// status codes. +type StatusError interface { + Status() int + Code() string +} + type Translator interface { Translate(*core.Event) (core.ServerUpdateEvents, error) } @@ -264,11 +271,13 @@ func (s *Synchronizer) handleDeletedEvent(ctx context.Context, serverUpdateEvent err = borderClient.Update(ctx, serverUpdateEvent) + var se StatusError switch { case err == nil: return nil - // checking the string is not ideal, but the plus client gives us no option - case strings.Contains(err.Error(), "status=404"): + case errors.As(err, &se) && se.Status() == http.StatusNotFound: + // if the user has already removed the upstream from their NGINX + // configuration there is nothing left to do return nil default: return fmt.Errorf(`error occurred deleting the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) From 81184f5bb7892308cd3a19dfb703de53f2267a01 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 2 Jul 2025 14:57:04 -0600 Subject: [PATCH 109/110] NLB-6754 Bumped version to 1.2.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 26aaba0e..6085e946 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.2.0 +1.2.1 From 24a3c17aae705a0c1e875919aa0a6776fd7708dc Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 8 Jul 2025 16:27:20 -0600 Subject: [PATCH 110/110] Removed context as a field within the nginx stream border client This is a go anti-pattern --- internal/application/nginx_http_border_client.go | 2 -- internal/application/nginx_stream_border_client.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index 91828082..4de147e5 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -18,7 +18,6 @@ import ( type NginxHTTPBorderClient struct { BorderClient nginxClient NginxClientInterface - ctx context.Context } // NewNginxHTTPBorderClient is the Factory function for creating an NewNginxHTTPBorderClient. @@ -30,7 +29,6 @@ func NewNginxHTTPBorderClient(client interface{}) (Interface, error) { return &NginxHTTPBorderClient{ nginxClient: ngxClient, - ctx: context.Background(), }, nil } diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index a65be866..238a22b6 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -18,7 +18,6 @@ import ( type NginxStreamBorderClient struct { BorderClient nginxClient NginxClientInterface - ctx context.Context } // NewNginxStreamBorderClient is the Factory function for creating an NginxStreamBorderClient. @@ -30,7 +29,6 @@ func NewNginxStreamBorderClient(client interface{}) (Interface, error) { return &NginxStreamBorderClient{ nginxClient: ngxClient, - ctx: context.Background(), }, nil }