diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index f330be63..00000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,3 +0,0 @@ -# Main global owner # -##################### -* @ciroque @chrisakker diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index cc6c1d26..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' ---- -### Describe the bug - -A clear and concise description of what the bug is. - -### To reproduce - -Steps to reproduce the behavior: - -1. Deploy nginx_loadbalancer_kubernetes using -2. View output/logs/configuration on '...' -3. See error - -### Expected behavior - -A clear and concise description of what you expected to happen. - -### Your environment - -- Version of the nginx_loadbalancer_kubernetes or specific commit - -- Target deployment platform - -### Additional context - -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index d27aba8e..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' ---- -### Is your feature request related to a problem? Please describe - -A clear and concise description of what the problem is. Ex. I'm always frustrated when ... - -### Describe the solution you'd like - -A clear and concise description of what you want to happen. - -### Describe alternatives you've considered - -A clear and concise description of any alternative solutions or features you've considered. - -### Additional context - -Add any other context or screenshots about the feature request here. diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 4450376b..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -version: 2 -updates: - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - day: monday - time: "00:00" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index fad5aa1e..00000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,12 +0,0 @@ -### Proposed changes - -Describe the use case and detail of the change. If this PR addresses an issue on GitHub, make sure to include a link to that issue using one of the [supported keywords](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue) here in this description (not in the title of the PR). - -### Checklist - -Before creating a PR, run through this checklist and mark each as complete. - -- [ ] I have read the [`CONTRIBUTING`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/CONTRIBUTING.md) document -- [ ] If applicable, I have added tests that prove my fix is effective or that my feature works -- [ ] If applicable, I have checked that any relevant tests pass after adding my changes -- [ ] I have updated any relevant documentation ([`README.md`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/README.md) and [`CHANGELOG.md`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/CHANGELOG.md)) diff --git a/.github/workflows/build-and-sign-image.yml b/.github/workflows/build-and-sign-image.yml deleted file mode 100644 index 2fbf227c..00000000 --- a/.github/workflows/build-and-sign-image.yml +++ /dev/null @@ -1,98 +0,0 @@ -# This workflow will build and push a signed Docker image - -name: Build and sign image - -on: - push: - tags: - - "v[0-9]+.[0-9]+.[0-9]+" -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build_and_sign_image: - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - id-token: write - security-events: write - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: anchore/sbom-action@v0 - with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - output-file: ./nginx-loadbalancer-kubernetes-${{env.GITHUB_REF_NAME}}.spdx.json - registry-username: ${{ github.actor }} - registry-password: ${{ secrets.GITHUB_TOKEN }} - - - name: Install cosign - uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 #v3.0.2 - with: - cosign-release: 'v1.13.1' - - - name: Log into registry ${{ env.REGISTRY }} for ${{ github.actor }} - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build Docker Image - id: docker-build-and-push - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{github.run_number}} - - - name: Sign the published Docker images - env: - COSIGN_EXPERIMENTAL: "true" - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: cosign sign "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.docker-build-and-push.outputs.digest }}" - - # NOTE: This runs statically against the latest tag in Docker Hub which was not produced by this workflow - # This should be updated once this workflow is fully implemented - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@91713af97dc80187565512baba96e4364e983601 # 0.16.0 - continue-on-error: true - with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - format: 'sarif' - output: 'trivy-results-${{ inputs.image }}.sarif' - ignore-unfixed: 'true' - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v2.2.11 - continue-on-error: true - with: - sarif_file: 'trivy-results-${{ inputs.image }}.sarif' - sha: ${{ github.sha }} - ref: ${{ github.ref }} - - - name: Generate Release - uses: ncipollo/release-action@v1 - with: - artifacts: | - trivy-results-${{ inputs.image }}.sarif - ./nginx-loadbalancer-kubernetes-${{env.GITHUB_REF_NAME}}.spdx.json - body: | - # Release ${{env.GITHUB_REF_NAME}} - ## Changelog - ${{ steps.meta.outputs.changelog }} - generateReleaseNotes: true - makeLatest: false - name: "${{env.GITHUB_REF_NAME}}" diff --git a/.github/workflows/run-scorecard.yml b/.github/workflows/run-scorecard.yml deleted file mode 100644 index 3bbad843..00000000 --- a/.github/workflows/run-scorecard.yml +++ /dev/null @@ -1,72 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: '15 14 * * 3' - push: - branches: [ "main" ] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@v4 # v3.1.0 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecard on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 - with: - sarif_file: results.sarif diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index 454c7169..00000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,32 +0,0 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - -name: Run tests - -on: - branch_protection_rule: - types: - - created - - push: - branches: - - main - - * - -jobs: - - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.19 - - - name: Build - run: go build -v ./... - - - name: Test - run: go test -v ./... diff --git a/.gitignore b/.gitignore index cb5c33a6..e69f5389 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,69 @@ cover* tmp/ docs/tls/DESIGN.md :q -qqq \ No newline at end of file +qqq.env +.env* +!.env.example +!.allowed_clients.json +!.env.example.auth +*.db +priv/certs +priv/nginx-agent/* +!priv/nginx-agent/nginx-agent.conf.example +key-data.json +nginx-instance-manager.tar.gz +vendor/ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Output from debugger +__debug_bin + +code-quality.json +coverage/* + +# vim +*~ +*.swp + +### VisualStudioCode (from https://gitignore.io/api/VisualStudioCode) ### +.vscode/* +!.vscode/tasks.example.json +!.vscode/launch.example.json +!.vscode/extensions.json +!.vscode/KubernetesLocalProcessConfig*.yaml +*.code-workspace + +### Goland +.idea/* + +# bridge to kubernetes artifact +/KubernetesLocalProcessConfig.yaml + + +# output directory for build artifacts +build + +# output directory for test artifacts (eg. coverage report, junit xml) +results + +# devops-utils repo +.devops-utils/ + +# Ignore golang cache in CI +.go/pkg/mod + +.go-build + +nginxaas-loadbalancer-kubernetes-* diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..268e39ff --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,92 @@ +# GolangCI-Lint settings + +# Disable all linters and enable the required ones +linters: + disable-all: true + + # Supported linters: https://golangci-lint.run/usage/linters/ + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + - bodyclose + - dupl + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - gosec + - lll + - misspell + - nakedret + - prealloc + - stylecheck + - unconvert + - unparam + - paralleltest + - forbidigo + fast: false + +# Run options +run: + # 10 minute timeout for analysis + timeout: 10m +# Specific linter settings +linters-settings: + gocyclo: + # Minimal code complexity to report + min-complexity: 16 + govet: + disable-all: true + enable: + # Report shadowed variables + - shadow + + misspell: + # Correct spellings using locale preferences for US + locale: US + goimports: + # Put imports beginning with prefix after 3rd-party packages + local-prefixes: gitswarm.f5net.com/indigo,gitlab.com/f5 + exhaustruct: + # List of regular expressions to match struct packages and names. + # If this list is empty, all structs are tested. + # Default: [] + include: + - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/pkg/token.TokenID" + - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest" + +issues: + exclude-dirs: + - .go/pkg/mod + # Exclude configuration + exclude-rules: + # Exclude gochecknoinits and gosec from running on tests files + - path: _test\.go + linters: + - gochecknoinits + - gosec + - path: test/* + linters: + - gochecknoinits + - gosec + # Exclude lll issues for long lines with go:generate + - linters: + - lll + source: "^//go:generate " + # Exclude false positive paralleltest error : Range statement for test case does not use range value in test Run + - linters: + - paralleltest + text: "does not use range value in test Run" + + # Disable maximum issues count per one linter + max-issues-per-linter: 0 + + # Disable maximum count of issues with the same text + max-same-issues: 0 diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 09548d5e..00000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -golang 1.19.13 diff --git a/Dockerfile b/Dockerfile index 9aa6b8cb..e53aef48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,11 @@ -# Copyright 2023 f5 Inc. All rights reserved. -# Use of this source code is governed by the Apache -# license that can be found in the LICENSE file. +FROM alpine:3.14.1 AS base-certs +RUN apk update && apk add --no-cache ca-certificates -FROM golang:1.19.5-alpine3.16 AS builder +FROM scratch AS base +COPY docker-user /etc/passwd +USER 101 +COPY --from=base-certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -WORKDIR /app - -COPY go.mod go.sum ./ - -RUN go mod download - -COPY . . - -RUN go build -o nginx-loadbalancer-kubernetes ./cmd/nginx-loadbalancer-kubernetes/main.go - -FROM alpine:3.16 - -WORKDIR /opt/nginx-loadbalancer-kubernetes - -RUN adduser -u 11115 -D -H nlk - -USER nlk - -COPY --from=builder /app/nginx-loadbalancer-kubernetes . - -ENTRYPOINT ["/opt/nginx-loadbalancer-kubernetes/nginx-loadbalancer-kubernetes"] +FROM base as nginxaas-loadbalancer-kubernetes +ENTRYPOINT ["/nginxaas-loadbalancer-kubernetes"] +COPY build/nginxaas-loadbalancer-kubernetes / diff --git a/charts/armTemplate.json b/charts/armTemplate.json new file mode 100644 index 00000000..06c37031 --- /dev/null +++ b/charts/armTemplate.json @@ -0,0 +1,256 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "extensionResourceName": { + "type": "string", + "metadata": { + "description": "The name of the extension." + } + }, + "extensionNamespace": { + "type": "string", + "defaultValue": "nlk" + }, + "clusterResourceName": { + "type": "String", + "metadata": { + "description": "The name of the Managed Cluster resource." + } + }, + "createNewCluster": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "When set to 'true', creates new AKS cluster. Otherwise, an existing cluster is used." + } + }, + "location": { + "type": "String", + "metadata": { + "description": "The location of AKS resource." + } + }, + "extensionAutoUpgrade": { + "defaultValue": false, + "metadata": { + "description": "Allow auto upgrade of minor version for the extension." + }, + "type": "bool" + }, + "nginxaasDataplaneApiKey": { + "type": "String" + }, + "nginxaasDataplaneApiEndpoint": { + "type": "String" + }, + "vmSize": { + "type": "String", + "defaultValue": "Standard_DS2_v2", + "metadata": { + "description": "VM size" + } + }, + "vmEnableAutoScale": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "enable auto scaling" + } + }, + "vmCount": { + "type": "Int", + "defaultValue": 3, + "metadata": { + "description": "VM count" + } + }, + "dnsPrefix": { + "defaultValue": "[concat(parameters('clusterResourceName'),'-dns')]", + "type": "String", + "metadata": { + "description": "Optional DNS prefix to use with hosted Kubernetes API server FQDN." + } + }, + "osDiskSizeGB": { + "defaultValue": 0, + "minValue": 0, + "maxValue": 1023, + "type": "Int", + "metadata": { + "description": "Disk size (in GiB) to provision for each of the agent pool nodes. This value ranges from 0 to 1023. Specifying 0 will apply the default disk size for that agentVMSize." + } + }, + "kubernetesVersion": { + "type": "String", + "defaultValue": "1.26.3", + "metadata": { + "description": "The version of Kubernetes." + } + }, + "networkPlugin": { + "defaultValue": "kubenet", + "allowedValues": [ + "azure", + "kubenet" + ], + "type": "String", + "metadata": { + "description": "Network plugin used for building Kubernetes network." + } + }, + "enableRBAC": { + "defaultValue": true, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off of RBAC." + } + }, + "enablePrivateCluster": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Enable private network access to the Kubernetes cluster." + } + }, + "enableHttpApplicationRouting": { + "defaultValue": true, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off http application routing." + } + }, + "enableAzurePolicy": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off Azure Policy addon." + } + }, + "enableSecretStoreCSIDriver": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off secret store CSI driver." + } + }, + "osSKU": { + "type": "string", + "defaultValue": "AzureLinux", + "allowedValues": [ + "AzureLinux", + "Ubuntu" + ], + "metadata": { + "description": "The Linux SKU to use." + } + }, + "enableFIPS": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "Enable FIPS. https://learn.microsoft.com/en-us/azure/aks/create-node-pools#fips-enabled-node-pools" + } + } + }, + "variables": { + "plan-name": "DONOTMODIFY", + "plan-publisher": "DONOTMODIFY", + "plan-offerID": "DONOTMODIFY", + "releaseTrain": "DONOTMODIFY", + "clusterExtensionTypeName": "DONOTMODIFY" + }, + "resources": [ + { + "type": "Microsoft.ContainerService/managedClusters", + "condition": "[parameters('createNewCluster')]", + "apiVersion": "2023-11-01", + "name": "[parameters('clusterResourceName')]", + "location": "[parameters('location')]", + "dependsOn": [], + "tags": {}, + "sku": { + "name": "Basic", + "tier": "Free" + }, + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "kubernetesVersion": "[parameters('kubernetesVersion')]", + "enableRBAC": "[parameters('enableRBAC')]", + "dnsPrefix": "[parameters('dnsPrefix')]", + "agentPoolProfiles": [ + { + "name": "agentpool", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "count": "[parameters('vmCount')]", + "enableAutoScaling": "[parameters('vmEnableAutoScale')]", + "enableFIPS": "[parameters('enableFIPS')]", + "minCount": "[if(parameters('vmEnableAutoScale'), 1, json('null'))]", + "maxCount": "[if(parameters('vmEnableAutoScale'), 10, json('null'))]", + "vmSize": "[parameters('vmSize')]", + "osType": "Linux", + "osSKU": "[parameters('osSKU')]", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "mode": "System", + "maxPods": 110, + "availabilityZones": [], + "enableNodePublicIP": false, + "tags": {} + } + ], + "networkProfile": { + "loadBalancerSku": "standard", + "networkPlugin": "[parameters('networkPlugin')]" + }, + "apiServerAccessProfile": { + "enablePrivateCluster": "[parameters('enablePrivateCluster')]" + }, + "addonProfiles": { + "httpApplicationRouting": { + "enabled": "[parameters('enableHttpApplicationRouting')]" + }, + "azurepolicy": { + "enabled": "[parameters('enableAzurePolicy')]" + }, + "azureKeyvaultSecretsProvider": { + "enabled": "[parameters('enableSecretStoreCSIDriver')]" + } + } + } + }, + { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2023-05-01", + "name": "[parameters('extensionResourceName')]", + "properties": { + "extensionType": "[variables('clusterExtensionTypeName')]", + "autoUpgradeMinorVersion": "[parameters('extensionAutoUpgrade')]", + "releaseTrain": "[variables('releaseTrain')]", + "configurationSettings": { + "nlk.dataplaneApiKey": "[parameters('nginxaasDataplaneApiKey')]", + "nlk.config.nginxHosts": "[parameters('nginxaasDataplaneApiEndpoint')]" + }, + "configurationProtectedSettings": {}, + "scope": { + "cluster": { + "releaseNamespace": "[parameters('extensionNamespace')]" + } + } + }, + "plan": { + "name": "[variables('plan-name')]", + "publisher": "[variables('plan-publisher')]", + "product": "[variables('plan-offerID')]" + }, + "scope": "[concat('Microsoft.ContainerService/managedClusters/', parameters('clusterResourceName'))]", + "dependsOn": [ + "[resourceId('Microsoft.ContainerService/managedClusters/', parameters('clusterResourceName'))]" + ] + } + ], + "outputs": { + } +} diff --git a/charts/createUIDefinition.json b/charts/createUIDefinition.json new file mode 100644 index 00000000..c7566d24 --- /dev/null +++ b/charts/createUIDefinition.json @@ -0,0 +1,264 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#", + "handler": "Microsoft.Azure.CreateUIDef", + "version": "0.1.2-preview", + "parameters": { + "config": { + "isWizard": false, + "basics": { + "location": { + "visible": "[basics('createNewCluster')]", + "resourceTypes": ["Nginx.NginxPlus/nginxDeployments"] + }, + "resourceGroup": { + "allowExisting": true + } + } + }, + "basics": [ + { + "name": "createNewCluster", + "type": "Microsoft.Common.OptionsGroup", + "label": "Create new AKS cluster", + "defaultValue": "No", + "toolTip": "Create a new AKS cluster to install the extension.", + "constraints": { + "allowedValues": [ + { + "label": "Yes", + "value": true + }, + { + "label": "No", + "value": false + } + ], + "required": true + }, + "visible": true + } + ], + "steps": [ + { + "name": "clusterDetails", + "label": "Cluster Details", + "elements": [ + { + "name": "existingClusterSection", + "type": "Microsoft.Common.Section", + "elements": [ + { + "name": "clusterLookupControl", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id, '/resourcegroups/', resourceGroup().name, '/providers/Microsoft.ContainerService/managedClusters?api-version=2022-03-01')]" + } + }, + { + "name": "existingClusterResourceName", + "type": "Microsoft.Common.DropDown", + "label": "AKS Cluster Name", + "toolTip": "The resource name of the existing AKS cluster.", + "constraints": { + "allowedValues": "[map(steps('clusterDetails').existingClusterSection.clusterLookupControl.value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.name, '\"}')))]", + "required": true + } + } + ], + "visible": "[equals(basics('createNewCluster'), false)]" + }, + { + "name": "newClusterSection", + "type": "Microsoft.Common.Section", + "elements": [ + { + "name": "aksVersionLookupControl", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id, '/providers/Microsoft.ContainerService/locations/', location(), '/orchestrators?api-version=2019-04-01&resource-type=managedClusters')]" + } + }, + { + "name": "newClusterResourceName", + "type": "Microsoft.Common.TextBox", + "label": "AKS cluster name", + "defaultValue": "", + "toolTip": "The resource name of the new AKS cluster. Use only allowed characters", + "constraints": { + "required": true, + "regex": "^[a-z0-9A-Z]{6,30}$", + "validationMessage": "Only alphanumeric characters are allowed, and the value must be 6-30 characters long." + } + }, + { + "name": "kubernetesVersion", + "type": "Microsoft.Common.DropDown", + "label": "Kubernetes version", + "toolTip": "The version of Kubernetes that should be used for this cluster. You will be able to upgrade this version after creating the cluster.", + "constraints": { + "allowedValues": "[map(steps('clusterDetails').newClusterSection.aksVersionLookupControl.properties.orchestrators, (item) => parse(concat('{\"label\":\"', item.orchestratorVersion, '\",\"value\":\"', item.orchestratorVersion, '\"}')))]", + "required": true + } + }, + { + "name": "vmSize", + "type": "Microsoft.Compute.SizeSelector", + "label": "VM size", + "toolTip": "The size of virtual machine of AKS worker nodes.", + "recommendedSizes": [ + "Standard_B4ms", + "Standard_DS2_v2", + "Standard_D4s_v3" + ], + "constraints": { + "allowedSizes": [ + "Standard_B4ms", + "Standard_DS2_v2", + "Standard_D4s_v3" + ], + "excludedSizes": [] + }, + "osPlatform": "Linux" + }, + { + "name": "osSKU", + "type": "Microsoft.Common.DropDown", + "label": "OS SKU", + "toolTip": "The SKU of Linux OS for VM.", + "defaultValue": "Ubuntu", + "constraints": { + "allowedValues": [ + { + "label": "Ubuntu", + "value": "Ubuntu" + }, + { + "label": "AzureLinux", + "value": "AzureLinux" + } + ], + "required": true + } + }, + { + "name": "enableAutoScaling", + "type": "Microsoft.Common.CheckBox", + "label": "Enable auto scaling", + "toolTip": "Enable auto scaling", + "defaultValue": true + }, + { + "name": "vmCount", + "type": "Microsoft.Common.Slider", + "min": 1, + "max": 10, + "label": "Number of AKS worker nodes", + "subLabel": "", + "defaultValue": 1, + "showStepMarkers": false, + "toolTip": "Specify the number of AKS worker nodes.", + "constraints": { + "required": false + }, + "visible": true + } + ], + "visible": "[basics('createNewCluster')]" + } + ] + }, + { + "name": "applicationDetails", + "label": "Application Details", + "elements": [ + { + "name": "extensionResourceName", + "type": "Microsoft.Common.TextBox", + "label": "Cluster extension resource name", + "defaultValue": "", + "toolTip": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long.", + "constraints": { + "required": true, + "regex": "^[a-z0-9]{6,30}$", + "validationMessage": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long." + }, + "visible": true + }, + { + "name": "extensionNamespace", + "type": "Microsoft.Common.TextBox", + "label": "Installation namespace", + "defaultValue": "nlk", + "toolTip": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long.", + "constraints": { + "required": true, + "regex": "^[a-z0-9]{3,30}$", + "validationMessage": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long." + }, + "visible": true + }, + { + "name": "extensionAutoUpgrade", + "type": "Microsoft.Common.CheckBox", + "label": "Allow minor version upgrades of extension", + "toolTip": "Allow exntension to be upgraded automatically to latest minor version.", + "visible": true + }, + { + "name": "nginxaasDataplaneApiKey", + "type": "Microsoft.Common.TextBox", + "label": "NGINXaaS Dataplane API Key", + "defaultValue": "", + "toolTip": "The Dataplane API Key for your NGINXaaS for Azure deployment.", + "constraints": { + "required": false, + "regex": ".*", + "validationMessage": "Use the dataplane API key for your deployment." + }, + "visible": true + }, + { + "name": "nginxaasDataplaneApiEndpoint", + "type": "Microsoft.Common.TextBox", + "label": "NGINXaaS Dataplane API Endpoint", + "defaultValue": "", + "toolTip": "The Dataplane API Endpoint for your NGINXaaS for Azure deployment.", + "constraints": { + "required": false, + "regex": ".*", + "validationMessage": "Retreive the dataplane API endpoint from your deployment." + }, + "visible": true + }, + { + "name": "additionalProductInfo", + "type": "Microsoft.Common.InfoBox", + "visible": true, + "options": { + "icon": "Info", + "text": "Learn more about NGINXaaS for Azure.", + "uri": "https://docs.nginx.com/nginxaas/azure/" + } + } + ] + } + ], + "outputs": { + "location": "[location()]", + "createNewCluster": "[basics('createNewCluster')]", + "clusterResourceName": "[if(basics('createNewCluster'), steps('clusterDetails').newClusterSection.newClusterResourceName, steps('clusterDetails').existingClusterSection.existingClusterResourceName)]", + "kubernetesVersion": "[steps('clusterDetails').newClusterSection.kubernetesVersion]", + "vmSize": "[steps('clusterDetails').newClusterSection.vmSize]", + "osSKU": "[steps('clusterDetails').newClusterSection.osSKU]", + "vmEnableAutoScale": "[steps('clusterDetails').newClusterSection.enableAutoScaling]", + "vmCount": "[steps('clusterDetails').newClusterSection.vmCount]", + "extensionResourceName": "[steps('applicationDetails').extensionResourceName]", + "extensionAutoUpgrade": "[steps('applicationDetails').extensionAutoUpgrade]", + "extensionNamespace": "[steps('applicationDetails').extensionNamespace]", + "nginxaasDataplaneApiKey": "[steps('applicationDetails').nginxaasDataplaneApiKey]", + "nginxaasDataplaneApiEndpoint": "[steps('applicationDetails').nginxaasDataplaneApiEndpoint]" + } + } +} diff --git a/charts/manifest.yaml b/charts/manifest.yaml new file mode 100644 index 00000000..6883ea96 --- /dev/null +++ b/charts/manifest.yaml @@ -0,0 +1,11 @@ +applicationName: "marketplace/nginxaas-loadbalancer-kubernetes" +publisher: "F5, Inc." +description: "A component that manages NGINXaaS for Azure deployment and makes it act as Load Balancer for kubernetes workloads." +version: 0.4.0 +helmChart: "./nlk" +clusterArmTemplate: "./armTemplate.json" +uiDefinition: "./createUIDefinition.json" +registryServer: "nlbmarketplaceacrprod.azurecr.io" +extensionRegistrationParameters: + defaultScope: "cluster" + namespace: "nlk" diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index c11d8853..b11ab862 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,19 +1,16 @@ --- apiVersion: v2 -appVersion: 0.1.0 -description: NGINX LoadBalancer for Kubernetes -name: nginx-loadbalancer-kubernetes -home: https://github.com/nginxinc/nginx-loadbalancer-kubernetes -icon: https://raw.githubusercontent.com/nginxinc/nginx-loadbalancer-kubernetes/main/nlk-logo.svg +appVersion: 0.8.0 +description: NGINXaaS LoadBalancer for Kubernetes +name: nginxaas-loadbalancer-kubernetes keywords: -- nginx -- loadbalancer -- ingress + - nginx + - nginxaas + - loadbalancer kubeVersion: '>= 1.22.0-0' maintainers: -- name: "@ciroque" -- name: "@chrisakker" -- name: "@abdennour" - + - name: "@ciroque" + - name: "@chrisakker" + - name: "@abdennour" type: application -version: 0.0.1 +version: 0.8.0 diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 17a64051..27dbe94a 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -48,6 +48,10 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} +{{- define "nlk.apikeyname" -}} +{{- printf "%s-nginxaas-api-key" (include "nlk.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + {{/* Common labels */}} @@ -76,11 +80,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} Expand the name of the configmap. */}} {{- define "nlk.configName" -}} -{{- if .Values.nlk.customConfigMap -}} -{{ .Values.nlk.customConfigMap }} -{{- else -}} -{{- default (include "nlk.fullname" .) .Values.nlk.config.name -}} -{{- end -}} +{{- printf "%s-nlk-config" (include "nlk.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} {{/* @@ -91,14 +91,20 @@ Expand service account name. {{- end -}} {{- define "nlk.tag" -}} +{{- if .Values.global.azure -}} +{{- printf "%s" .Values.global.azure.images.nlk.tag -}} +{{- else -}} {{- default .Chart.AppVersion .Values.nlk.image.tag -}} {{- end -}} +{{- end -}} {{/* Expand image name. */}} {{- define "nlk.image" -}} -{{- if .Values.nlk.image.digest -}} +{{- if .Values.global.azure -}} +{{- printf "%s/%s:%s" .Values.global.azure.images.nlk.registry .Values.global.azure.images.nlk.image (include "nlk.tag" .) -}} +{{- else if .Values.nlk.image.digest -}} {{- printf "%s/%s@%s" .Values.nlk.image.registry .Values.nlk.image.repository .Values.nlk.image.digest -}} {{- else -}} {{- printf "%s/%s:%s" .Values.nlk.image.registry .Values.nlk.image.repository (include "nlk.tag" .) -}} diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index 4164475e..47f95dd4 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -7,12 +7,18 @@ rules: - apiGroups: - "" resources: - - configmaps - nodes - - secrets - services verbs: - get - list - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch {{- end }} diff --git a/charts/nlk/templates/clusterrolebinding.yaml b/charts/nlk/templates/clusterrolebinding.yaml index 0ccd4551..8503a242 100644 --- a/charts/nlk/templates/clusterrolebinding.yaml +++ b/charts/nlk/templates/clusterrolebinding.yaml @@ -6,7 +6,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: {{ .Release.Namespace }}-{{ include "nlk.fullname" . }} diff --git a/charts/nlk/templates/dataplaneApiKey.yaml b/charts/nlk/templates/dataplaneApiKey.yaml new file mode 100644 index 00000000..20511385 --- /dev/null +++ b/charts/nlk/templates/dataplaneApiKey.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "nlk.apikeyname" . }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + nginxaasApiKey: {{ .Values.nlk.dataplaneApiKey | toString | b64enc }} diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 482b8cbf..0b6db576 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -1,14 +1,17 @@ apiVersion: v1 kind: ConfigMap metadata: - name: nlk-config - namespace: nlk + name: {{ include "nlk.configName" . }} + namespace: {{ .Release.Namespace }} data: -{{- if .Values.nlk.config.entries.hosts }} - nginx-hosts: "{{ .Values.nlk.config.entries.hosts }}" + config.yaml: | +{{- with .Values.nlk.config.logLevel }} + log-level: "{{ . }}" +{{- end }} +{{- with .Values.nlk.config.nginxHosts }} + nginx-hosts: {{ toJson . }} +{{- end }} + tls-mode: "{{ .Values.nlk.config.tls.mode }}" +{{- with .Values.nlk.config.serviceAnnotationMatch }} + service-annotation-match: "{{ . }}" {{- end }} - tls-mode: "{{ index .Values.nlk.defaultTLS "tls-mode" }}" - ca-certificate: "{{ index .Values.nlk.defaultTLS "ca-certificate" }}" - client-certificate: "{{ index .Values.nlk.defaultTLS "client-certificate" }}" - log-level: "{{ .Values.nlk.logLevel }}" - diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index fb55d77c..93fcd383 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} labels: app: nlk spec: @@ -14,7 +14,16 @@ spec: metadata: labels: app: nlk +{{- if .Values.global.azure }} + azure-extensions-usage-release-identifier: {{ .Release.Name }} +{{- end }} + annotations: + checksum: {{ tpl (toYaml .Values.nlk) . | sha256sum }} spec: + {{- with .Values.nlk.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} image: {{ include "nlk.image" .}} @@ -41,4 +50,17 @@ spec: initialDelaySeconds: {{ .Values.nlk.readyStatus.initialDelaySeconds }} periodSeconds: {{ .Values.nlk.readyStatus.periodSeconds }} {{- end }} + env: + - name: NGINXAAS_DATAPLANE_API_KEY + valueFrom: + secretKeyRef: + name: {{ include "nlk.apikeyname" . }} + key: nginxaasApiKey + volumeMounts: + - name: config + mountPath: /etc/nginxaas-loadbalancer-kubernetes serviceAccountName: {{ include "nlk.fullname" . }} + volumes: + - name: config + configMap: + name: {{ include "nlk.configName" . }} diff --git a/charts/nlk/templates/nlk-secret.yaml b/charts/nlk/templates/nlk-secret.yaml index ff7d7ff7..cb964866 100644 --- a/charts/nlk/templates/nlk-secret.yaml +++ b/charts/nlk/templates/nlk-secret.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} annotations: kubernetes.io/service-account.name: {{ include "nlk.fullname" . }} type: kubernetes.io/service-account-token diff --git a/charts/nlk/templates/nlk-serviceaccount.yaml b/charts/nlk/templates/nlk-serviceaccount.yaml index 5bdca4f7..d2cd8e42 100644 --- a/charts/nlk/templates/nlk-serviceaccount.yaml +++ b/charts/nlk/templates/nlk-serviceaccount.yaml @@ -3,5 +3,5 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} {{- end }} diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 394bc1fb..1f2f1f2f 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -1,124 +1,59 @@ +##################################### +# Global Azure Marketplace configuration for AKS integration. +# DO NOT REMOVE +global: + azure: + # images: + # nlk: + # registry: registry-1.docker.io + # image: nginx/nginxaas-loadbalancer-kubernetes + # tag: 0.4.0 +##################################### nlk: - name: nginx-loadbalancer-kubernetes - + name: nginxaas-loadbalancer-kubernetes kind: deployment - replicaCount: 1 - image: - registry: ghcr.io - repository: nginxinc/nginx-loadbalancer-kubernetes + registry: registry-1.docker.io + repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: latest - + ## Overrides the image tag whose default is the chart appVersion. + # tag: 0.4.0 imagePullSecrets: [] nameOverride: "" fullnameOverride: "" - serviceAccount: - # Specifies whether a service account should be created + ## Specifies whether a service account should be created create: true - # Automatically mount a ServiceAccount's API credentials? + ## Automatically mount a ServiceAccount's API credentials? automount: true - # Annotations to add to the service account - annotations: {} - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - ingress: - enabled: false - className: "" + ## Annotations to add to the service account annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - - resources: - requests: - cpu: 100m - memory: 128Mi - # limits: - # cpu: 100m - # memory: 128Mi - - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 3 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - - # Additional volumes on the output Deployment definition. - volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - nodeSelector: {} - - tolerations: [] - - affinity: {} - config: - entries: - hosts: - "http://10.1.1.4:9000/api,http://10.1.1.5:9000/api" - - defaultTLS: - tls-mode: "no-tls" - ca-certificate: "" - client-certificate: "" - - logLevel: "warn" - + ## trace,debug,info,warn,error,fatal,panic + logLevel: "info" + + ## the nginx hosts (comma-separated) to send upstream updates to + nginxHosts: "" + ## Sets the annotation value that NLK is looking for to watch a Service + # serviceAnnotationMatch: nginxaas + tls: + ## can also be set to "no-tls" to disable server cert verification + mode: "ca-tls" + ## Override with your own NGINXaaS dataplane API Key. + dataplaneApiKey: "test" containerPort: http: 51031 - liveStatus: enable: true port: 51031 initialDelaySeconds: 5 periodSeconds: 2 - readyStatus: enable: true port: 51031 initialDelaySeconds: 5 periodSeconds: 2 - rbac: ## Configures RBAC. create: true diff --git a/cmd/certificates-test-harness/doc.go b/cmd/certificates-test-harness/doc.go deleted file mode 100644 index 538bed98..00000000 --- a/cmd/certificates-test-harness/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package certificates_test_harness includes functionality boostrap and test the certification.Certificates implplementation. -*/ - -package main diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go deleted file mode 100644 index 44d4a4e6..00000000 --- a/cmd/certificates-test-harness/main.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/sirupsen/logrus" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" - "path/filepath" -) - -func main() { - logrus.SetLevel(logrus.DebugLevel) - err := run() - if err != nil { - logrus.Fatal(err) - } -} - -func run() error { - logrus.Info("certificates-test-harness::run") - - ctx := context.Background() - var err error - - k8sClient, err := buildKubernetesClient() - if err != nil { - return fmt.Errorf(`error building a Kubernetes client: %w`, err) - } - - certificates := certification.NewCertificates(ctx, k8sClient) - - err = certificates.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing certificates: %w`, err) - } - - go certificates.Run() - - <-ctx.Done() - return nil -} - -func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") - - var kubeconfig *string - var k8sConfig *rest.Config - - k8sConfig, err := rest.InClusterConfig() - if errors.Is(err, rest.ErrNotInCluster) { - if home := homedir.HomeDir(); home != "" { - path := filepath.Join(home, ".kube", "config") - kubeconfig = &path - - k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - return nil, fmt.Errorf(`error occurred building the kubeconfig: %w`, err) - } - } else { - return nil, fmt.Errorf(`not running in a Cluster: %w`, err) - } - } else if err != nil { - return nil, fmt.Errorf(`error occurred getting the Cluster config: %w`, err) - } - - client, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - return nil, fmt.Errorf(`error occurred creating a client: %w`, err) - } - return client, nil -} diff --git a/cmd/configuration-test-harness/doc.go b/cmd/configuration-test-harness/doc.go deleted file mode 100644 index 06ab7d0f..00000000 --- a/cmd/configuration-test-harness/doc.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/cmd/configuration-test-harness/main.go b/cmd/configuration-test-harness/main.go deleted file mode 100644 index 56e8b5dd..00000000 --- a/cmd/configuration-test-harness/main.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - configuration2 "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" - "path/filepath" -) - -func main() { - logrus.SetLevel(logrus.DebugLevel) - err := run() - if err != nil { - logrus.Fatal(err) - } -} - -func run() error { - logrus.Info("configuration-test-harness::run") - - ctx := context.Background() - var err error - - k8sClient, err := buildKubernetesClient() - if err != nil { - return fmt.Errorf(`error building a Kubernetes client: %w`, err) - } - - configuration, err := configuration2.NewSettings(ctx, k8sClient) - if err != nil { - return fmt.Errorf(`error occurred creating configuration: %w`, err) - } - - err = configuration.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing configuration: %w`, err) - } - - go configuration.Run() - - <-ctx.Done() - - return err -} - -func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") - - var kubeconfig *string - var k8sConfig *rest.Config - - k8sConfig, err := rest.InClusterConfig() - if errors.Is(err, rest.ErrNotInCluster) { - if home := homedir.HomeDir(); home != "" { - path := filepath.Join(home, ".kube", "config") - kubeconfig = &path - - k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - return nil, fmt.Errorf(`error occurred building the kubeconfig: %w`, err) - } - } else { - return nil, fmt.Errorf(`not running in a Cluster: %w`, err) - } - } else if err != nil { - return nil, fmt.Errorf(`error occurred getting the Cluster config: %w`, err) - } - - client, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - return nil, fmt.Errorf(`error occurred creating a client: %w`, err) - } - return client, nil -} diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 69365579..43de7940 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -8,11 +8,17 @@ package main import ( "context" "fmt" + "log/slog" + "os" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/observation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/probation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "github.com/sirupsen/logrus" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" + "golang.org/x/sync/errgroup" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" @@ -21,7 +27,8 @@ import ( func main() { err := run() if err != nil { - logrus.Fatal(err) + slog.Error(err.Error()) + os.Exit(1) } } @@ -34,62 +41,83 @@ func run() error { return fmt.Errorf(`error building a Kubernetes client: %w`, err) } - settings, err := configuration.NewSettings(ctx, k8sClient) + settings, err := configuration.Read("config.yaml", "/etc/nginxaas-loadbalancer-kubernetes") if err != nil { - return fmt.Errorf(`error occurred creating settings: %w`, err) + return fmt.Errorf(`error occurred accessing configuration: %w`, err) } - err = settings.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing settings: %w`, err) - } + initializeLogger(settings.LogLevel) - go settings.Run() + synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - synchronizerWorkqueue, err := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - if err != nil { - return fmt.Errorf(`error occurred building a workqueue: %w`, err) - } + factory := informers.NewSharedInformerFactoryWithOptions( + k8sClient, settings.Watcher.ResyncPeriod, + ) - synchronizer, err := synchronization.NewSynchronizer(settings, synchronizerWorkqueue) - if err != nil { - return fmt.Errorf(`error initializing synchronizer: %w`, err) - } + serviceInformer := factory.Core().V1().Services() + endpointSliceInformer := factory.Discovery().V1().EndpointSlices() + endpointSliceLister := endpointSliceInformer.Lister() + nodesInformer := factory.Core().V1().Nodes() + nodesLister := nodesInformer.Lister() + + translator := translation.NewTranslator(endpointSliceLister, nodesLister) - handlerWorkqueue, err := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) + synchronizer, err := synchronization.NewSynchronizer( + settings, synchronizerWorkqueue, translator, serviceInformer.Lister()) if err != nil { - return fmt.Errorf(`error occurred building a workqueue: %w`, err) + return fmt.Errorf(`error initializing synchronizer: %w`, err) } - handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue) - - watcher, err := observation.NewWatcher(settings, handler) + watcher, err := observation.NewWatcher(settings, synchronizer, serviceInformer, endpointSliceInformer, nodesInformer) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } - err = watcher.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing the watcher: %w`, err) + factory.Start(ctx.Done()) + results := factory.WaitForCacheSync(ctx.Done()) + for name, success := range results { + if !success { + return fmt.Errorf(`error occurred waiting for cache sync for %s`, name) + } } - go handler.Run(ctx.Done()) - go synchronizer.Run(ctx.Done()) + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { return synchronizer.Run(ctx) }) probeServer := probation.NewHealthServer() probeServer.Start() - err = watcher.Watch() - if err != nil { - return fmt.Errorf(`error occurred watching for events: %w`, err) + g.Go(func() error { return watcher.Run(ctx) }) + + err = g.Wait() + return err +} + +func initializeLogger(logLevel string) { + programLevel := new(slog.LevelVar) + + switch logLevel { + case "error": + programLevel.Set(slog.LevelError) + case "warn": + programLevel.Set(slog.LevelWarn) + case "info": + programLevel.Set(slog.LevelInfo) + case "debug": + programLevel.Set(slog.LevelDebug) + default: + programLevel.Set(slog.LevelWarn) } - <-ctx.Done() - return nil + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: programLevel}) + logger := slog.New(handler).With("version", buildinfo.SemVer()) + slog.SetDefault(logger) + slog.Debug("Settings::setLogLevel", slog.String("level", logLevel)) } func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") + slog.Debug("Watcher::buildKubernetesClient") k8sConfig, err := rest.InClusterConfig() if err == rest.ErrNotInCluster { return nil, fmt.Errorf(`not running in a Cluster: %w`, err) @@ -105,9 +133,12 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { return client, nil } -func buildWorkQueue(settings configuration.WorkQueueSettings) (workqueue.RateLimitingInterface, error) { - logrus.Debug("Watcher::buildSynchronizerWorkQueue") +func buildWorkQueue(settings configuration.WorkQueueSettings, +) workqueue.TypedRateLimitingInterface[synchronization.ServiceKey] { + slog.Debug("Watcher::buildSynchronizerWorkQueue") - rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) - return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name), nil + rateLimiter := workqueue.NewTypedItemExponentialFailureRateLimiter[synchronization.ServiceKey]( + settings.RateLimiterBase, settings.RateLimiterMax) + return workqueue.NewTypedRateLimitingQueueWithConfig( + rateLimiter, workqueue.TypedRateLimitingQueueConfig[synchronization.ServiceKey]{Name: settings.Name}) } diff --git a/cmd/tls-config-factory-test-harness/doc.go b/cmd/tls-config-factory-test-harness/doc.go deleted file mode 100644 index 06ab7d0f..00000000 --- a/cmd/tls-config-factory-test-harness/doc.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go deleted file mode 100644 index 3f46d4fd..00000000 --- a/cmd/tls-config-factory-test-harness/main.go +++ /dev/null @@ -1,230 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" - "os" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" - ClientCertificateSecretKey = "nlk-tls-client-secret" -) - -type TlsConfiguration struct { - Description string - Settings configuration.Settings -} - -func main() { - logrus.SetLevel(logrus.DebugLevel) - - configurations := buildConfigMap() - - for name, settings := range configurations { - fmt.Print("\033[H\033[2J") - - logrus.Infof("\n\n\t*** Building TLS config for <<< %s >>>\n\n", name) - - tlsConfig, err := authentication.NewTlsConfig(&settings.Settings) - if err != nil { - panic(err) - } - - rootCaCount := 0 - certificateCount := 0 - - if tlsConfig.RootCAs != nil { - rootCaCount = len(tlsConfig.RootCAs.Subjects()) - } - - if tlsConfig.Certificates != nil { - certificateCount = len(tlsConfig.Certificates) - } - - logrus.Infof("Successfully built TLS config: \n\tDescription: %s \n\tRootCA count: %v\n\tCertificate count: %v", settings.Description, rootCaCount, certificateCount) - - bufio.NewReader(os.Stdin).ReadBytes('\n') - } - - fmt.Print("\033[H\033[2J") - logrus.Infof("\n\n\t*** All done! ***\n\n") -} - -func buildConfigMap() map[string]TlsConfiguration { - configurations := make(map[string]TlsConfiguration) - - configurations["ss-tls"] = TlsConfiguration{ - Description: "Self-signed TLS requires just a CA certificate", - Settings: ssTlsConfig(), - } - - configurations["ss-mtls"] = TlsConfiguration{ - Description: "Self-signed mTLS requires a CA certificate and a client certificate", - Settings: ssMtlsConfig(), - } - - configurations["ca-tls"] = TlsConfiguration{ - Description: "CA TLS requires no certificates", - Settings: caTlsConfig(), - } - - configurations["ca-mtls"] = TlsConfiguration{ - Description: "CA mTLS requires a client certificate", - Settings: caMtlsConfig(), - } - - return configurations -} - -func ssTlsConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - return configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, - } -} - -func ssMtlsConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - return configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, - } -} - -func caTlsConfig() configuration.Settings { - return configuration.Settings{ - TlsMode: configuration.CertificateAuthorityTLS, - } -} - -func caMtlsConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - return configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, - } -} - -func caCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIDTzCCAjcCFA4Zdj3E9TdjOP48eBRDGRLfkj7CMA0GCSqGSIb3DQEBCwUAMGQx -CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0 -dGxlMQ4wDAYDVQQKDAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFu -Y2VzMB4XDTIzMDkyOTE3MTY1MVoXDTIzMTAyOTE3MTY1MVowZDELMAkGA1UEBhMC -VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDjAMBgNV -BAoMBU5HSU5YMR4wHAYDVQQLDBVDb21tdW5pdHkgJiBBbGxpYW5jZXMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwlI4ZvJ/6hvqULFVL+1ZSRDTPQ48P -umehJhPz6xPhC9UkeTe2FZxm2Rsi1I5QXm/bTG2OcX775jgXzae9NQjctxwrz4Ks -LOWUvRkkfhQR67xk0Noux76/9GWGnB+Fapn54tlWql6uHQfOu1y7MCRkZ27zHbkk -lq4Oa2RmX8rIyECWgbTyL0kETBVJU8bYORQ5JjhRlz08inq3PggY8blrehIetrWN -dw+gzcqdvAI2uSCodHTHM/77KipnYmPiSiDjSDRlXdxTG8JnyIB78IoH/sw6RyBm -CvVa3ytvKziXAvbBoXq5On5WmMRF97p/MmBc53ExMuDZjA4fisnViS0PAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAJeoa2P59zopLjBInx/DnWn1N1CmFLb0ejKxG2jh -cOw15Sx40O0XrtrAto38iu4R/bkBeNCSUILlT+A3uYDila92Dayvls58WyIT3meD -G6+Sx/QDF69+4AXpVy9mQ+hxcofpFA32+GOMXwmk2OrAcdSkkGSBhZXgvTpQ64dl -xSiQ5EQW/K8LoBoEOXfjIZJNPORgKn5MI09AY7/47ycKDKTUU2yO8AtIHYKttw0x -kfIg7QOdo1F9IXVpGjJI7ynyrgsCEYxMoDyH42Dq84eKgrUFLEXemEz8hgdFgK41 -0eUYhAtzWHbRPBp+U/34CQoZ5ChNFp2YipvtXrzKE8KLkuM= ------END CERTIFICATE----- -` -} - -func clientCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// clientKeyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientKeyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} - -func buildClientCertificateEntry(keyPEM, certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - certification.CertificateKeyKey: core.SecretBytes([]byte(keyPEM)), - } -} - -func buildCaCertificateEntry(certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - } -} diff --git a/deployments/checks/sample-ingress-mod.yaml b/deployments/checks/sample-ingress-mod.yaml deleted file mode 100644 index cd793055..00000000 --- a/deployments/checks/sample-ingress-mod.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: example-ingress - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - rules: - - host: hello-world.net - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: web - port: - number: 8080 \ No newline at end of file diff --git a/deployments/checks/sample-ingress.yaml b/deployments/checks/sample-ingress.yaml deleted file mode 100644 index 7ff7fc5f..00000000 --- a/deployments/checks/sample-ingress.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: example-ingress - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - rules: - - host: hello-world.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: web - port: - number: 8080 \ No newline at end of file diff --git a/deployments/deployment/configmap.yaml b/deployments/deployment/configmap.yaml deleted file mode 100644 index fd30dbe4..00000000 --- a/deployments/deployment/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - nginx-hosts: "https://10.0.0.1:9000/api" - tls-mode: "no-tls" - ca-certificate: "" - client-certificate: "" - log-level: "warn" -metadata: - name: nlk-config - namespace: nlk diff --git a/deployments/deployment/deployment.yaml b/deployments/deployment/deployment.yaml deleted file mode 100644 index 4c871c21..00000000 --- a/deployments/deployment/deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nlk-deployment - namespace: nlk - labels: - app: nlk -spec: - replicas: 1 - selector: - matchLabels: - app: nlk - template: - metadata: - labels: - app: nlk - spec: - containers: - - name: nginx-loadbalancer-kubernetes - image: ghcr.io/nginxinc/nginx-loadbalancer-kubernetes:latest - imagePullPolicy: Always - ports: - - name: http - containerPort: 51031 - protocol: TCP - livenessProbe: - httpGet: - path: /livez - port: 51031 - initialDelaySeconds: 5 - periodSeconds: 2 - readinessProbe: - httpGet: - path: /readyz - port: 51031 - initialDelaySeconds: 5 - periodSeconds: 2 - serviceAccountName: nginx-loadbalancer-kubernetes diff --git a/deployments/deployment/namespace.yaml b/deployments/deployment/namespace.yaml deleted file mode 100644 index 8f9e3822..00000000 --- a/deployments/deployment/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: nlk - labels: - name: nlk diff --git a/deployments/rbac/apply.sh b/deployments/rbac/apply.sh deleted file mode 100755 index 58248da1..00000000 --- a/deployments/rbac/apply.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -pushd "$(dirname "$0")" - -echo "Applying all RBAC resources..." - -kubectl apply -f serviceaccount.yaml -kubectl apply -f clusterrole.yaml -kubectl apply -f clusterrolebinding.yaml -kubectl apply -f secret.yaml - -popd diff --git a/deployments/rbac/clusterrole.yaml b/deployments/rbac/clusterrole.yaml deleted file mode 100644 index c50bed85..00000000 --- a/deployments/rbac/clusterrole.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: resource-get-watch-list - namespace: nlk -rules: - - apiGroups: - - "" - resources: ["services", "nodes", "configmaps", "secrets"] - verbs: ["get", "watch", "list"] diff --git a/deployments/rbac/clusterrolebinding.yaml b/deployments/rbac/clusterrolebinding.yaml deleted file mode 100644 index d48ffb84..00000000 --- a/deployments/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: "nginx-loadbalancer-kubernetes:resource-get-watch-list" - namespace: nlk -subjects: - - kind: ServiceAccount - name: nginx-loadbalancer-kubernetes - namespace: nlk -roleRef: - kind: ClusterRole - name: resource-get-watch-list - apiGroup: rbac.authorization.k8s.io diff --git a/deployments/rbac/secret.yaml b/deployments/rbac/secret.yaml deleted file mode 100644 index 71576bfb..00000000 --- a/deployments/rbac/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: nginx-loadbalancer-kubernetes-secret - namespace: nlk - annotations: - kubernetes.io/service-account.name: nginx-loadbalancer-kubernetes -type: kubernetes.io/service-account-token diff --git a/deployments/rbac/serviceaccount.yaml b/deployments/rbac/serviceaccount.yaml deleted file mode 100644 index 76f238c0..00000000 --- a/deployments/rbac/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-loadbalancer-kubernetes - namespace: nlk diff --git a/deployments/rbac/unapply.sh b/deployments/rbac/unapply.sh deleted file mode 100755 index f29f90df..00000000 --- a/deployments/rbac/unapply.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -echo "Unapplying all RBAC resources..." - -kubectl delete -f serviceaccount.yaml -kubectl delete -f clusterrole.yaml -kubectl delete -f clusterrolebinding.yaml -kubectl delete -f secret.yaml diff --git a/doc.go b/doc.go index 7c97bd20..1034f144 100644 --- a/doc.go +++ b/doc.go @@ -3,4 +3,4 @@ * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ -package kubernetes_nginx_ingress +package kubernetesnginxingress diff --git a/docker-user b/docker-user new file mode 100644 index 00000000..65be48a5 --- /dev/null +++ b/docker-user @@ -0,0 +1 @@ +nginx:x:101:101:nginx:/var/cache/nginx:/sbin/nologin \ No newline at end of file diff --git a/docs/http/clusters.conf b/docs/http/clusters.conf index 6c3ab6d8..4a71ef4f 100644 --- a/docs/http/clusters.conf +++ b/docs/http/clusters.conf @@ -9,7 +9,7 @@ # Define Key Value store, backup state file, timeout, and enable sync -keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=30d sync; +keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=365d sync; keyval $host $split_level zone=split; # Main Nginx Server Block for cafe.example.com, with TLS diff --git a/docs/http/default-http.conf b/docs/http/default-http.conf deleted file mode 100644 index b9685533..00000000 --- a/docs/http/default-http.conf +++ /dev/null @@ -1,28 +0,0 @@ -# NGINX Loadbalancer for K8s Solution -# Chris Akker, Apr 2023 -# Example default.conf -# Change default_server to port 8080 -# -server { - listen 8080 default_server; # Changed to 8080 - server_name localhost; - - #access_log /var/log/nginx/host.access.log main; - - location / { - root /usr/share/nginx/html; - index index.html index.htm; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - -### other sections removed for clarity - -} \ No newline at end of file diff --git a/docs/http/http-installation-guide.md b/docs/http/http-installation-guide.md index 8484bfd1..60609258 100644 --- a/docs/http/http-installation-guide.md +++ b/docs/http/http-installation-guide.md @@ -201,21 +201,28 @@ Note: If you choose a different Application to test with, `the NGINX configurati This can be any standard Linux OS system, based on the Linux Distro and Technical Specs required for NGINX Plus, which can be found here: https://docs.nginx.com/nginx/technical-specs/ - 1. This Solution followed the `Installation of NGINX Plus on Centos/Redhat/Oracle` steps for installing NGINX Plus. +1. This Solution followed the `Installation of NGINX Plus on Centos/Redhat/Oracle` steps for installing NGINX Plus. https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-plus/ >NOTE: This Solution will only work with NGINX Plus, as NGINX OpenSource does not have the API that is used in this Solution. Installation on unsupported Linux Distros is not recommended. - 1. Install the NGINX Javascript module (njs). This is required for exporting Prometheus Metrics from NGINX Plus. + If you need a license for NGINX Plus, a 30-day Trial license is available here: + + https://www.nginx.com/free-trial-request/ + +1. Install the NGINX Javascript module (njs). This is required for exporting Prometheus Metrics from NGINX Plus. ```bash yum install nginx-plus-module-njs ``` - 1. If you need a license for NGINX Plus, a 30-day Trial license is available here: +1. Install Nginx Javascript for Prometheus + + ```bash + yum install nginx-plus-module-prometheus + ``` - https://www.nginx.com/free-trial-request/
@@ -224,20 +231,18 @@ This can be any standard Linux OS system, based on the Linux Distro and Technica
### This is the NGINX configuration required for the NGINX Loadbalancing Server, external to the cluster. It must be configured for the following: - -1. Move the NGINX default Welcome page from port 80 to port 8080. Port 80 will be used by Prometheus in this Solution. -2. The NGINX NJS module is enabled, and configured to export the NGINX Plus statistics. +1. The NGINX NJS module is enabled, and configured to export the NGINX Plus statistics. -3. A self-signed TLS cert/key are used in this example for terminating TLS traffic for the Demo application, https://cafe.example.com. +2. A self-signed TLS cert/key are used in this example for terminating TLS traffic for the Demo application, https://cafe.example.com. -4. Plus API with write access enabled on port 9000. The Plus Dashboard is also enabled, used for testing, monitoring, and visualization of the Solution working. +3. Plus API with write access enabled on port 9000. The Plus Dashboard is also enabled, used for testing, monitoring, and visualization of the Solution working. -5. The `http` context is used for MultiCluster Loadbalancing, for HTTP/S processing, Split Clients ratio. The Plus Key Value Store is configured, to hold the dynamic Split ratio metadata. +4. The `http` context is used for MultiCluster Loadbalancing, for HTTP/S processing, Split Clients ratio. The Plus Key Value Store is configured, to hold the dynamic Split ratio metadata. -6. Enable Prometheus metrics exporting. +5. Enable Prometheus metrics exporting. -7. Plus Zone Sync on Port 9001 is configured, to synchronize the dynamic KeyVal data between multiple NGINX Loadbalancing Servers. +6. Plus Zone Sync on Port 9001 is configured, to synchronize the dynamic KeyVal data between multiple NGINX Loadbalancing Servers.
@@ -259,7 +264,6 @@ etc/ ├── conf.d/ │ ├── clusters.conf.......... MultiCluster Loadbalancing and split clients config │ ├── dashboard.conf......... NGINX Plus API and Dashboard config - │ ├── default-http.conf...... New default.conf config │ └── prometheus.conf........ NGINX Prometheus config ├── nginx.conf................. New nginx.conf └── stream @@ -270,41 +274,6 @@ etc/ After a new installation of NGINX Plus, make the following configuration changes: -1. Change NGINX's http default server to port 8080. See the included `default-http.conf` file. After reloading nginx, the default `Welcome to NGINX` page will be located at http://localhost:8080. - - ```bash - cat /etc/nginx/conf.d/default.conf - # NGINX Loadbalancer for Kubernetes Solution - # Chris Akker, Apr 2023 - # Example default.conf - # Change default_server to port 8080 - # - server { - listen 8080 default_server; # Changed to 8080 - server_name localhost; - - #access_log /var/log/nginx/host.access.log main; - - location / { - root /usr/share/nginx/html; - index index.html index.htm; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - - ### other sections removed for clarity - - } - - ``` - 1. Use the included nginx.conf file, it enables the NGINX NJS module, for exporting the Plus statistics: ```bash @@ -396,7 +365,7 @@ After a new installation of NGINX Plus, make the following configuration changes # Define Key Value store, backup state file, timeout, and enable sync - keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=30d sync; + keyval_zone zone=split:1m state=/var/lib/nginx/state/split.keyval timeout=365d sync; keyval $host $split_level zone=split; # Main NGINX Server Block for cafe.example.com, with TLS @@ -549,6 +518,9 @@ After a new installation of NGINX Plus, make the following configuration changes js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; server { + listen 9113; + status_zone prometheus; + location = /metrics { js_content prometheus.metrics; } @@ -1007,7 +979,7 @@ Here are the instructions to run 2 Docker containers on a Monitor Server, which
-1. Configure your Prometheus server to collect NGINX Plus statistics from the scraper page. Use the prometheus.yml file provided, edit the IP addresses to match your NGINX Loadbalancing Server(s). +1. Configure your Prometheus server to collect NGINX Plus statistics from the scraper page. You can use the prometheus.yml file provided, edit the IP addresses to match your NGINX Loadbalancing Server(s). ```bash cat prometheus.yaml @@ -1026,7 +998,7 @@ Here are the instructions to run 2 Docker containers on a Monitor Server, which scrape_interval: 5s static_configs: - - targets: ['10.1.1.4:80', '10.1.1.5:80'] # NGINX Loadbalancing Servers + - targets: ['10.1.1.4:9113', '10.1.1.5:9113'] # NGINX Loadbalancing Servers ``` 1. Review, edit and place the sample `prometheus.yml` file in /etc/prometheus folder. diff --git a/docs/http/prometheus.conf b/docs/http/prometheus.conf index 15f53c9c..47d15e92 100644 --- a/docs/http/prometheus.conf +++ b/docs/http/prometheus.conf @@ -7,6 +7,9 @@ js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; server { + listen 9113; + status_zone prometheus; + location = /metrics { js_content prometheus.metrics; } diff --git a/docs/http/prometheus.yml b/docs/http/prometheus.yml index c8b19cb2..3dee16bc 100644 --- a/docs/http/prometheus.yml +++ b/docs/http/prometheus.yml @@ -10,5 +10,5 @@ scrape_configs: scrape_interval: 5s static_configs: - - targets: ['10.1.1.4:80', '10.1.1.5:80'] + - targets: ['10.1.1.4:9113', '10.1.1.5:9113'] \ No newline at end of file diff --git a/go.mod b/go.mod index 38f6adb0..6a78fe2f 100644 --- a/go.mod +++ b/go.mod @@ -4,55 +4,72 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.19 +go 1.24.4 require ( - github.com/nginxinc/nginx-plus-go-client v0.10.0 - github.com/sirupsen/logrus v1.9.0 - k8s.io/api v0.26.0 - k8s.io/apimachinery v0.26.0 - k8s.io/client-go v0.26.0 + github.com/nginx/nginx-plus-go-client/v2 v2.4.0 + github.com/spf13/viper v1.19.0 + github.com/stretchr/testify v1.10.0 + golang.org/x/sync v0.13.0 + k8s.io/api v0.33.1 + k8s.io/apimachinery v0.33.1 + k8s.io/client-go v0.33.1 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 867f71f1..8cf5ed86 100644 --- a/go.sum +++ b/go.sum @@ -1,153 +1,61 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -155,334 +63,135 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginxinc/nginx-plus-go-client v0.10.0 h1:3zsMMkPvRDo8D7ZSprXtbAEW/SDmezZWzxdyS+6oAlc= -github.com/nginxinc/nginx-plus-go-client v0.10.0/go.mod h1:0v3RsQCvRn/IyrMtW+DK6CNkz+PxEsXDJPjQ3yUMBF0= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/nginx/nginx-plus-go-client/v2 v2.4.0 h1:4c7V57CLCZUOxQCUcS9G8a5MClzdmxByBm+f4zKMzAY= +github.com/nginx/nginx-plus-go-client/v2 v2.4.0/go.mod h1:P+dIP2oKYzFoyf/zlLWQa8Sf+fHb+CclOKzxAjxpvug= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= -k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= -k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= -k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= -k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= +k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= +k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= +k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= +k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/application/application_common_test.go b/internal/application/application_common_test.go index e963d03d..c42bc04c 100644 --- a/internal/application/application_common_test.go +++ b/internal/application/application_common_test.go @@ -7,6 +7,7 @@ package application import ( "errors" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" ) @@ -18,11 +19,11 @@ const ( server = "server" ) -func buildTerrorizingBorderClient(clientType string) (Interface, *mocks.MockNginxClient, error) { +func buildTerrorizingBorderClient(clientType string) (Interface, error) { nginxClient := mocks.NewErroringMockClient(errors.New(`something went horribly horribly wrong`)) bc, err := NewBorderClient(clientType, nginxClient) - return bc, nginxClient, err + return bc, err } func buildBorderClient(clientType string) (Interface, *mocks.MockNginxClient, error) { diff --git a/internal/application/application_constants.go b/internal/application/application_constants.go index 0ec18264..4cb23a54 100644 --- a/internal/application/application_constants.go +++ b/internal/application/application_constants.go @@ -12,7 +12,8 @@ package application // annotations: // nginxinc.io/nlk-: // -// where is the name of the upstream in the NGINX Plus configuration and is one of the constants below. +// where is the name of the upstream in the NGINX Plus configuration +// and is one of the constants below. // // Note, this is an extensibility point. To add a Border Server client... // 1. Create a module that implements the BorderClient interface; @@ -23,6 +24,6 @@ const ( // ClientTypeNginxStream creates a NginxStreamBorderClient that uses the Stream* methods of the NGINX Plus client. ClientTypeNginxStream = "stream" - // ClientTypeNginxHttp creates an NginxHttpBorderClient that uses the HTTP* methods of the NGINX Plus client. - ClientTypeNginxHttp = "http" + // ClientTypeNginxHTTP creates an NginxHTTPBorderClient that uses the HTTP* methods of the NGINX Plus client. + ClientTypeNginxHTTP = "http" ) diff --git a/internal/application/border_client.go b/internal/application/border_client.go index a5cc93e0..8bca843b 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -6,20 +6,21 @@ package application import ( + "context" "fmt" + "log/slog" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) // Interface defines the functions required to implement a Border Client. type Interface interface { - Update(*core.ServerUpdateEvent) error - Delete(*core.ServerUpdateEvent) error + Update(context.Context, *core.ServerUpdateEvent) error + Delete(context.Context, *core.ServerUpdateEvent) error } // BorderClient defines any state need by the Border Client. -type BorderClient struct { -} +type BorderClient struct{} // NewBorderClient is the Factory function for creating a Border Client. // @@ -28,14 +29,14 @@ type BorderClient struct { // 2. Add a new constant in application_constants.go that acts as a key for selecting the client; // 3. Update the NewBorderClient factory method in border_client.go that returns the client; func NewBorderClient(clientType string, borderClient interface{}) (Interface, error) { - logrus.Debugf(`NewBorderClient for type: %s`, clientType) + slog.Debug("NewBorderClient", slog.String("client", clientType)) switch clientType { case ClientTypeNginxStream: return NewNginxStreamBorderClient(borderClient) - case ClientTypeNginxHttp: - return NewNginxHttpBorderClient(borderClient) + case ClientTypeNginxHTTP: + return NewNginxHTTPBorderClient(borderClient) default: borderClient, _ := NewNullBorderClient() diff --git a/internal/application/border_client_test.go b/internal/application/border_client_test.go index 0b8105ec..8960eee7 100644 --- a/internal/application/border_client_test.go +++ b/internal/application/border_client_test.go @@ -6,23 +6,26 @@ package application import ( - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" ) func TestBorderClient_CreatesHttpBorderClient(t *testing.T) { + t.Parallel() borderClient := mocks.MockNginxClient{} client, err := NewBorderClient("http", borderClient) if err != nil { t.Errorf(`error creating border client: %v`, err) } - if _, ok := client.(*NginxHttpBorderClient); !ok { - t.Errorf(`expected client to be of type NginxHttpBorderClient`) + if _, ok := client.(*NginxHTTPBorderClient); !ok { + t.Errorf(`expected client to be of type NginxHTTPBorderClient`) } } func TestBorderClient_CreatesTcpBorderClient(t *testing.T) { + t.Parallel() borderClient := mocks.MockNginxClient{} client, err := NewBorderClient("stream", borderClient) if err != nil { @@ -35,6 +38,7 @@ func TestBorderClient_CreatesTcpBorderClient(t *testing.T) { } func TestBorderClient_UnknownClientType(t *testing.T) { + t.Parallel() unknownClientType := "unknown" borderClient := mocks.MockNginxClient{} client, err := NewBorderClient(unknownClientType, borderClient) diff --git a/internal/application/doc.go b/internal/application/doc.go index 34c27d0e..296cb67c 100644 --- a/internal/application/doc.go +++ b/internal/application/doc.go @@ -17,7 +17,7 @@ To add a Border Server client... At this time the only supported Border Servers are NGINX Plus servers. The two Border Server clients for NGINX Plus are: -- NginxHttpBorderClient: updates NGINX Plus servers using HTTP Upstream methods on the NGINX Plus API. +- NginxHTTPBorderClient: updates NGINX Plus servers using HTTP Upstream methods on the NGINX Plus API. - NginxStreamBorderClient: updates NGINX Plus servers using Stream Upstream methods on the NGINX Plus API. Both of these implementations use the NGINX Plus client module to communicate with the NGINX Plus server. @@ -27,7 +27,8 @@ Selection of the appropriate client is based on the Annotations present on the S annotations: nginxinc.io/nlk-: -where is the name of the upstream in the NGINX Plus configuration and is one of the constants in application_constants.go. +where is the name of the upstream in the NGINX Plus configuration +and is one of the constants in application_constants.go. */ package application diff --git a/internal/application/nginx_client_interface.go b/internal/application/nginx_client_interface.go index 728db1e3..1a60c5ee 100644 --- a/internal/application/nginx_client_interface.go +++ b/internal/application/nginx_client_interface.go @@ -5,19 +5,34 @@ package application -import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" +import ( + "context" -// NginxClientInterface defines the functions used on the NGINX Plus client, abstracting away the full details of that client. + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" +) + +var _ NginxClientInterface = (*nginxClient.NginxClient)(nil) + +// NginxClientInterface defines the functions used on the NGINX Plus client, +// abstracting away the full details of that client. type NginxClientInterface interface { // DeleteStreamServer is used by the NginxStreamBorderClient. - DeleteStreamServer(upstream string, server string) error + DeleteStreamServer(ctx context.Context, upstream string, server string) error // UpdateStreamServers is used by the NginxStreamBorderClient. - UpdateStreamServers(upstream string, servers []nginxClient.StreamUpstreamServer) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) + UpdateStreamServers( + ctx context.Context, + upstream string, + servers []nginxClient.StreamUpstreamServer, + ) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) - // DeleteHTTPServer is used by the NginxHttpBorderClient. - DeleteHTTPServer(upstream string, server string) error + // DeleteHTTPServer is used by the NginxHTTPBorderClient. + DeleteHTTPServer(ctx context.Context, upstream string, server string) error - // UpdateHTTPServers is used by the NginxHttpBorderClient. - UpdateHTTPServers(upstream string, servers []nginxClient.UpstreamServer) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) + // UpdateHTTPServers is used by the NginxHTTPBorderClient. + UpdateHTTPServers( + ctx context.Context, + upstream string, + servers []nginxClient.UpstreamServer, + ) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) } diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index b7657c5a..4de147e5 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -2,37 +2,40 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_stream_border_client.go +//nolint:dupl package application import ( + "context" "fmt" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) // NginxHttpBorderClient implements the BorderClient interface for HTTP upstreams. -type NginxHttpBorderClient struct { +type NginxHTTPBorderClient struct { BorderClient nginxClient NginxClientInterface } -// NewNginxHttpBorderClient is the Factory function for creating an NginxHttpBorderClient. -func NewNginxHttpBorderClient(client interface{}) (Interface, error) { +// NewNginxHTTPBorderClient is the Factory function for creating an NewNginxHTTPBorderClient. +func NewNginxHTTPBorderClient(client interface{}) (Interface, error) { ngxClient, ok := client.(NginxClientInterface) if !ok { return nil, fmt.Errorf(`expected a NginxClientInterface, got a %v`, client) } - return &NginxHttpBorderClient{ + return &NginxHTTPBorderClient{ nginxClient: ngxClient, }, nil } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHttpBorderClient) Update(event *core.ServerUpdateEvent) error { - httpUpstreamServers := asNginxHttpUpstreamServers(event.UpstreamServers) - _, _, _, err := hbc.nginxClient.UpdateHTTPServers(event.UpstreamName, httpUpstreamServers) +func (hbc *NginxHTTPBorderClient) Update(ctx context.Context, event *core.ServerUpdateEvent) error { + httpUpstreamServers := asNginxHTTPUpstreamServers(event.UpstreamServers) + _, _, _, err := hbc.nginxClient.UpdateHTTPServers(ctx, event.UpstreamName, httpUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) } @@ -41,8 +44,8 @@ func (hbc *NginxHttpBorderClient) Update(event *core.ServerUpdateEvent) error { } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHttpBorderClient) Delete(event *core.ServerUpdateEvent) error { - err := hbc.nginxClient.DeleteHTTPServer(event.UpstreamName, event.UpstreamServers[0].Host) +func (hbc *NginxHTTPBorderClient) Delete(ctx context.Context, event *core.ServerUpdateEvent) error { + err := hbc.nginxClient.DeleteHTTPServer(ctx, event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) } @@ -51,18 +54,18 @@ func (hbc *NginxHttpBorderClient) Delete(event *core.ServerUpdateEvent) error { } // asNginxHttpUpstreamServer converts a core.UpstreamServer to a nginxClient.UpstreamServer. -func asNginxHttpUpstreamServer(server *core.UpstreamServer) nginxClient.UpstreamServer { +func asNginxHTTPUpstreamServer(server *core.UpstreamServer) nginxClient.UpstreamServer { return nginxClient.UpstreamServer{ Server: server.Host, } } -// asNginxHttpUpstreamServers converts a core.UpstreamServers to a []nginxClient.UpstreamServer. -func asNginxHttpUpstreamServers(servers core.UpstreamServers) []nginxClient.UpstreamServer { - var upstreamServers []nginxClient.UpstreamServer +// asNginxHTTPUpstreamServers converts a core.UpstreamServers to a []nginxClient.UpstreamServer. +func asNginxHTTPUpstreamServers(servers core.UpstreamServers) []nginxClient.UpstreamServer { + upstreamServers := []nginxClient.UpstreamServer{} for _, server := range servers { - upstreamServers = append(upstreamServers, asNginxHttpUpstreamServer(server)) + upstreamServers = append(upstreamServers, asNginxHTTPUpstreamServer(server)) } return upstreamServers diff --git a/internal/application/nginx_http_border_client_test.go b/internal/application/nginx_http_border_client_test.go index defc2ef8..7a972068 100644 --- a/internal/application/nginx_http_border_client_test.go +++ b/internal/application/nginx_http_border_client_test.go @@ -3,20 +3,25 @@ * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ +// dupl complains about duplicates with nginx_stream_border_client_test.go +// +//nolint:dupl package application import ( + "context" "testing" ) func TestHttpBorderClient_Delete(t *testing.T) { - event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHttp) - borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHTTP) + borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -27,13 +32,14 @@ func TestHttpBorderClient_Delete(t *testing.T) { } func TestHttpBorderClient_Update(t *testing.T) { - event := buildServerUpdateEvent(createEventType, ClientTypeNginxHttp) - borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(createEventType, ClientTypeNginxHTTP) + borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -44,21 +50,23 @@ func TestHttpBorderClient_Update(t *testing.T) { } func TestHttpBorderClient_BadNginxClient(t *testing.T) { + t.Parallel() var emptyInterface interface{} - _, err := NewBorderClient(ClientTypeNginxHttp, emptyInterface) + _, err := NewBorderClient(ClientTypeNginxHTTP, emptyInterface) if err == nil { t.Fatalf(`expected an error to occur when creating a new border client`) } } func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { - event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHttp) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHTTP) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) @@ -66,13 +74,14 @@ func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { } func TestHttpBorderClient_UpdateReturnsError(t *testing.T) { - event := buildServerUpdateEvent(createEventType, ClientTypeNginxHttp) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(createEventType, ClientTypeNginxHTTP) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index 46cd4985..238a22b6 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -2,13 +2,16 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_http_border_client.go +//nolint:dupl package application import ( + "context" "fmt" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) // NginxStreamBorderClient implements the BorderClient interface for stream upstreams. @@ -30,9 +33,9 @@ func NewNginxStreamBorderClient(client interface{}) (Interface, error) { } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (tbc *NginxStreamBorderClient) Update(event *core.ServerUpdateEvent) error { +func (tbc *NginxStreamBorderClient) Update(ctx context.Context, event *core.ServerUpdateEvent) error { streamUpstreamServers := asNginxStreamUpstreamServers(event.UpstreamServers) - _, _, _, err := tbc.nginxClient.UpdateStreamServers(event.UpstreamName, streamUpstreamServers) + _, _, _, err := tbc.nginxClient.UpdateStreamServers(ctx, event.UpstreamName, streamUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) } @@ -41,8 +44,8 @@ func (tbc *NginxStreamBorderClient) Update(event *core.ServerUpdateEvent) error } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (tbc *NginxStreamBorderClient) Delete(event *core.ServerUpdateEvent) error { - err := tbc.nginxClient.DeleteStreamServer(event.UpstreamName, event.UpstreamServers[0].Host) +func (tbc *NginxStreamBorderClient) Delete(ctx context.Context, event *core.ServerUpdateEvent) error { + err := tbc.nginxClient.DeleteStreamServer(ctx, event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) } @@ -57,7 +60,7 @@ func asNginxStreamUpstreamServer(server *core.UpstreamServer) nginxClient.Stream } func asNginxStreamUpstreamServers(servers core.UpstreamServers) []nginxClient.StreamUpstreamServer { - var upstreamServers []nginxClient.StreamUpstreamServer + upstreamServers := []nginxClient.StreamUpstreamServer{} for _, server := range servers { upstreamServers = append(upstreamServers, asNginxStreamUpstreamServer(server)) diff --git a/internal/application/nginx_stream_border_client_test.go b/internal/application/nginx_stream_border_client_test.go index ddcb3466..cf4d302d 100644 --- a/internal/application/nginx_stream_border_client_test.go +++ b/internal/application/nginx_stream_border_client_test.go @@ -2,21 +2,24 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_http_border_client_test.go +//nolint:dupl package application import ( + "context" "testing" ) func TestTcpBorderClient_Delete(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxStream) borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -27,13 +30,14 @@ func TestTcpBorderClient_Delete(t *testing.T) { } func TestTcpBorderClient_Update(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(createEventType, ClientTypeNginxStream) borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -44,6 +48,7 @@ func TestTcpBorderClient_Update(t *testing.T) { } func TestTcpBorderClient_BadNginxClient(t *testing.T) { + t.Parallel() var emptyInterface interface{} _, err := NewBorderClient(ClientTypeNginxStream, emptyInterface) if err == nil { @@ -52,13 +57,14 @@ func TestTcpBorderClient_BadNginxClient(t *testing.T) { } func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxStream) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxStream) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) @@ -66,13 +72,14 @@ func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { } func TestTcpBorderClient_UpdateReturnsError(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(createEventType, ClientTypeNginxStream) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxStream) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) diff --git a/internal/application/null_border_client.go b/internal/application/null_border_client.go index 8370fe02..dc4467d1 100644 --- a/internal/application/null_border_client.go +++ b/internal/application/null_border_client.go @@ -6,14 +6,16 @@ package application import ( + "context" + "log/slog" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) // NullBorderClient is a BorderClient that does nothing. -// / It serves only to prevent a panic if the BorderClient is not set correctly and errors from the factory methods are ignored. -type NullBorderClient struct { -} +// It serves only to prevent a panic if the BorderClient +// is not set correctly and errors from the factory methods are ignored. +type NullBorderClient struct{} // NewNullBorderClient is the Factory function for creating a NullBorderClient func NewNullBorderClient() (Interface, error) { @@ -21,13 +23,13 @@ func NewNullBorderClient() (Interface, error) { } // Update logs a Warning. It is, after all, a NullObject Pattern implementation. -func (nbc *NullBorderClient) Update(_ *core.ServerUpdateEvent) error { - logrus.Warn("NullBorderClient.Update called") +func (nbc *NullBorderClient) Update(_ context.Context, _ *core.ServerUpdateEvent) error { + slog.Warn("NullBorderClient.Update called") return nil } // Delete logs a Warning. It is, after all, a NullObject Pattern implementation. -func (nbc *NullBorderClient) Delete(_ *core.ServerUpdateEvent) error { - logrus.Warn("NullBorderClient.Delete called") +func (nbc *NullBorderClient) Delete(_ context.Context, _ *core.ServerUpdateEvent) error { + slog.Warn("NullBorderClient.Delete called") return nil } diff --git a/internal/application/null_border_client_test.go b/internal/application/null_border_client_test.go index 42e9dfb0..01d9fe23 100644 --- a/internal/application/null_border_client_test.go +++ b/internal/application/null_border_client_test.go @@ -5,19 +5,24 @@ package application -import "testing" +import ( + "context" + "testing" +) func TestNullBorderClient_Delete(t *testing.T) { + t.Parallel() client := NullBorderClient{} - err := client.Delete(nil) + err := client.Delete(context.Background(), nil) if err != nil { t.Errorf(`expected no error deleting border client, got: %v`, err) } } func TestNullBorderClient_Update(t *testing.T) { + t.Parallel() client := NullBorderClient{} - err := client.Update(nil) + err := client.Update(context.Background(), nil) if err != nil { t.Errorf(`expected no error updating border client, got: %v`, err) } diff --git a/internal/authentication/doc.go b/internal/authentication/doc.go deleted file mode 100644 index 109255ed..00000000 --- a/internal/authentication/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package authentication includes functionality to secure communications between NLK and NGINX Plus hosts. -*/ - -package authentication diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go deleted file mode 100644 index 8b8d06ec..00000000 --- a/internal/authentication/factory.go +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - * - * Factory for creating tls.Config objects based on the provided `tls-mode`. - */ - -package authentication - -import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" -) - -func NewTlsConfig(settings *configuration.Settings) (*tls.Config, error) { - logrus.Debugf("authentication::NewTlsConfig Creating TLS config for mode: '%s'", settings.TlsMode) - switch settings.TlsMode { - - case configuration.NoTLS: - return buildBasicTlsConfig(true), nil - - case configuration.SelfSignedTLS: // needs ca cert - return buildSelfSignedTlsConfig(settings.Certificates) - - case configuration.SelfSignedMutualTLS: // needs ca cert and client cert - return buildSelfSignedMtlsConfig(settings.Certificates) - - case configuration.CertificateAuthorityTLS: // needs nothing - return buildBasicTlsConfig(false), nil - - case configuration.CertificateAuthorityMutualTLS: // needs client cert - return buildCaTlsConfig(settings.Certificates) - - default: - return nil, fmt.Errorf("unknown TLS mode: %s", settings.TlsMode) - } -} - -func buildSelfSignedTlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") - certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) - if err != nil { - return nil, err - } - - return &tls.Config{ - InsecureSkipVerify: false, - RootCAs: certPool, - }, nil -} - -func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildSelfSignedMtlsConfig Building self-signed mTLS config") - certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) - if err != nil { - return nil, err - } - - certificate, err := buildCertificates(certificates.GetClientCertificate()) - if err != nil { - return nil, err - } - logrus.Debugf("buildSelfSignedMtlsConfig Certificate: %v", certificate) - - return &tls.Config{ - InsecureSkipVerify: false, - RootCAs: certPool, - ClientAuth: tls.RequireAndVerifyClientCert, - Certificates: []tls.Certificate{certificate}, - }, nil -} - -func buildBasicTlsConfig(skipVerify bool) *tls.Config { - logrus.Debugf("authentication::buildBasicTlsConfig skipVerify(%v)", skipVerify) - return &tls.Config{ - InsecureSkipVerify: skipVerify, - } -} - -func buildCaTlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildCaTlsConfig") - certificate, err := buildCertificates(certificates.GetClientCertificate()) - if err != nil { - return nil, err - } - - return &tls.Config{ - InsecureSkipVerify: false, - Certificates: []tls.Certificate{certificate}, - }, nil -} - -func buildCertificates(privateKeyPEM []byte, certificatePEM []byte) (tls.Certificate, error) { - logrus.Debug("authentication::buildCertificates") - return tls.X509KeyPair(certificatePEM, privateKeyPEM) -} - -func buildCaCertificatePool(caCert []byte) (*x509.CertPool, error) { - logrus.Debug("authentication::buildCaCertificatePool") - block, _ := pem.Decode(caCert) - if block == nil { - return nil, fmt.Errorf("failed to decode PEM block containing CA certificate") - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("error parsing certificate: %w", err) - } - - caCertPool := x509.NewCertPool() - caCertPool.AddCert(cert) - - return caCertPool, nil -} diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go deleted file mode 100644 index a5352007..00000000 --- a/internal/authentication/factory_test.go +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package authentication - -import ( - "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" - ClientCertificateSecretKey = "nlk-tls-client-secret" -) - -func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { - settings := configuration.Settings{} - - tlsConfig, err := NewTlsConfig(&settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != true { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be true`) - } -} - -func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - - settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, - } - - tlsConfig, err := NewTlsConfig(&settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) != 0 { - t.Fatalf(`tlsConfig.Certificates should be empty`) - } - - if tlsConfig.RootCAs == nil { - t.Fatalf(`tlsConfig.RootCAs should not be nil`) - } -} - -func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - - settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, - } - - _, err := NewTlsConfig(&settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "failed to decode PEM block containing CA certificate" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) - - settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, - } - - _, err := NewTlsConfig(&settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "error parsing certificate: x509: inner and outer signature algorithm identifiers don't match" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, - } - - tlsConfig, err := NewTlsConfig(&settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) == 0 { - t.Fatalf(`tlsConfig.Certificates should not be empty`) - } - - if tlsConfig.RootCAs == nil { - t.Fatalf(`tlsConfig.RootCAs should not be nil`) - } -} - -func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, - } - - _, err := NewTlsConfig(&settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "failed to decode PEM block containing CA certificate" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) - - settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, - } - - _, err := NewTlsConfig(&settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "tls: failed to find any PEM data in certificate input" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_CaTlsMode(t *testing.T) { - settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityTLS, - } - - tlsConfig, err := NewTlsConfig(&settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) != 0 { - t.Fatalf(`tlsConfig.Certificates should be empty`) - } - - if tlsConfig.RootCAs != nil { - t.Fatalf(`tlsConfig.RootCAs should be nil`) - } -} - -func TestTlsFactory_CaMtlsMode(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, - } - - tlsConfig, err := NewTlsConfig(&settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) == 0 { - t.Fatalf(`tlsConfig.Certificates should not be empty`) - } - - if tlsConfig.RootCAs != nil { - t.Fatalf(`tlsConfig.RootCAs should be nil`) - } -} - -func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) - - settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, - } - - _, err := NewTlsConfig(&settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "tls: failed to find any PEM data in certificate input" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -// caCertificatePEM returns a PEM-encoded CA certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func caCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIDTzCCAjcCFA4Zdj3E9TdjOP48eBRDGRLfkj7CMA0GCSqGSIb3DQEBCwUAMGQx -CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0 -dGxlMQ4wDAYDVQQKDAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFu -Y2VzMB4XDTIzMDkyOTE3MTY1MVoXDTIzMTAyOTE3MTY1MVowZDELMAkGA1UEBhMC -VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDjAMBgNV -BAoMBU5HSU5YMR4wHAYDVQQLDBVDb21tdW5pdHkgJiBBbGxpYW5jZXMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwlI4ZvJ/6hvqULFVL+1ZSRDTPQ48P -umehJhPz6xPhC9UkeTe2FZxm2Rsi1I5QXm/bTG2OcX775jgXzae9NQjctxwrz4Ks -LOWUvRkkfhQR67xk0Noux76/9GWGnB+Fapn54tlWql6uHQfOu1y7MCRkZ27zHbkk -lq4Oa2RmX8rIyECWgbTyL0kETBVJU8bYORQ5JjhRlz08inq3PggY8blrehIetrWN -dw+gzcqdvAI2uSCodHTHM/77KipnYmPiSiDjSDRlXdxTG8JnyIB78IoH/sw6RyBm -CvVa3ytvKziXAvbBoXq5On5WmMRF97p/MmBc53ExMuDZjA4fisnViS0PAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAJeoa2P59zopLjBInx/DnWn1N1CmFLb0ejKxG2jh -cOw15Sx40O0XrtrAto38iu4R/bkBeNCSUILlT+A3uYDila92Dayvls58WyIT3meD -G6+Sx/QDF69+4AXpVy9mQ+hxcofpFA32+GOMXwmk2OrAcdSkkGSBhZXgvTpQ64dl -xSiQ5EQW/K8LoBoEOXfjIZJNPORgKn5MI09AY7/47ycKDKTUU2yO8AtIHYKttw0x -kfIg7QOdo1F9IXVpGjJI7ynyrgsCEYxMoDyH42Dq84eKgrUFLEXemEz8hgdFgK41 -0eUYhAtzWHbRPBp+U/34CQoZ5ChNFp2YipvtXrzKE8KLkuM= ------END CERTIFICATE----- -` -} - -func invalidCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIClzCCAX+gAwIBAgIJAIfPhC0RG6CwMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNV -BAMMDm9pbCBhdXRob3JpdHkwHhcNMjAwNDA3MTUwOTU1WhcNMjEwNDA2MTUwOTU1 -WjBMMSAwHgYDVQQLDBd5b3VuZy1jaGFsbGVuZ2UgdGVzdCBjb25zdW1lczEfMB0G -A1UECwwWc28wMS5jb3Jwb3JhdGlvbnNvY2lhbDEhMB8GA1UEAwwYc29tMS5jb3Jw -b3JhdGlvbnNvY2lhbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB -AQDGRX31uzy+yLUOz7wOJHHm2dzrDgUbC6RZDjURvZxyt2Zi5wYWsEB5r5YhN7L0 -y1R9f+MGwNITIz9nYZuU/PLFOvzF5qX7A8TbdgjZEqvXe2NZ9J2z3iWvYQLN8Py3 -nv/Y6wadgXEBRCNNuIg/bQ9XuOr9tfB6j4Ut1GLU0eIlV/L3Rf9Y6SgrAl+58ITj -Wrg3Js/Wz3J2JU4qBD8U4I3XvUyfnX2SAG8Llm4KBuYz7g63Iu05s6RnmG+Xhu2T -5f2DWZUeATWbAlUW/M4NLO1+5H0gOr0TGulETQ6uElMchT7s/H6Rv1CV+CNCCgEI -adRjWJq9yQ+KrE+urSMCXu8XAgMBAAGjUzBRMB0GA1UdDgQWBBRb40pKGU4lNvqB -1f5Mz3t0N/K3hzAfBgNVHSMEGDAWgBRb40pKGU4lNvqB1f5Mz3t0N/K3hzAPBgNV -HREECDAGhwQAAAAAAAAwCgYIKoZIzj0EAwIDSAAwRQIhAP3ST/mXyRXsU2ciRoE -gE6trllODFY+9FgT6UbF2TwzAiAAuaUxtbk6uXLqtD5NtXqOQf0Ckg8GQxc5V1G2 -9PqTXQ== ------END CERTIFICATE----- -` -} - -// Yoinked from https://cs.opensource.google/go/go/+/refs/tags/go1.21.1:src/crypto/x509/x509_test.go, line 3385 -// This allows the `buildCaCertificatePool(...)` --> `x509.ParseCertificate(...)` call error branch to be covered. -func invalidCertificateDataPEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIBBzCBrqADAgECAgEAMAoGCCqGSM49BAMCMAAwIhgPMDAwMTAxMDEwMDAwMDBa -GA8wMDAxMDEwMTAwMDAwMFowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABOqV -EDuVXxwZgIU3+dOwv1SsMu0xuV48hf7xmK8n7sAMYgllB+96DnPqBeboJj4snYnx -0AcE0PDVQ1l4Z3YXsQWjFTATMBEGA1UdEQEB/wQHMAWCA2FzZDAKBggqhkjOPQQD -AwNIADBFAiBi1jz/T2HT5nAfrD7zsgR+68qh7Erc6Q4qlxYBOgKG4QIhAOtjIn+Q -tA+bq+55P3ntxTOVRq0nv1mwnkjwt9cQR9Fn ------END CERTIFICATE----- -` -} - -// clientCertificatePEM returns a PEM-encoded client certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// clientKeyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientKeyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} - -func buildClientCertificateEntry(keyPEM, certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - certification.CertificateKeyKey: core.SecretBytes([]byte(keyPEM)), - } -} - -func buildCaCertificateEntry(certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - } -} diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go deleted file mode 100644 index 53bd8435..00000000 --- a/internal/certification/certificates.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - * - * Establishes a Watcher for the Kubernetes Secrets that contain the various certificates and keys used to generate a tls.Config object; - * exposes the certificates and keys. - */ - -package certification - -import ( - "context" - "fmt" - - "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - // SecretsNamespace is the value used to filter the Secrets Resource in the Informer. - SecretsNamespace = "nlk" - - // CertificateKey is the key for the certificate in the Secret. - CertificateKey = "tls.crt" - - // CertificateKeyKey is the key for the certificate key in the Secret. - CertificateKeyKey = "tls.key" -) - -type Certificates struct { - Certificates map[string]map[string]core.SecretBytes - - // Context is the context used to control the application. - Context context.Context - - // CaCertificateSecretKey is the name of the Secret that contains the Certificate Authority certificate. - CaCertificateSecretKey string - - // ClientCertificateSecretKey is the name of the Secret that contains the Client certificate. - ClientCertificateSecretKey string - - // informer is the SharedInformer used to watch for changes to the Secrets . - informer cache.SharedInformer - - // K8sClient is the Kubernetes client used to communicate with the Kubernetes API. - k8sClient kubernetes.Interface - - // eventHandlerRegistration is the object used to track the event handlers with the SharedInformer. - eventHandlerRegistration cache.ResourceEventHandlerRegistration -} - -// NewCertificates factory method that returns a new Certificates object. -func NewCertificates(ctx context.Context, k8sClient kubernetes.Interface) *Certificates { - return &Certificates{ - k8sClient: k8sClient, - Context: ctx, - Certificates: nil, - } -} - -// GetCACertificate returns the Certificate Authority certificate. -func (c *Certificates) GetCACertificate() core.SecretBytes { - bytes := c.Certificates[c.CaCertificateSecretKey][CertificateKey] - - return bytes -} - -// GetClientCertificate returns the Client certificate and key. -func (c *Certificates) GetClientCertificate() (core.SecretBytes, core.SecretBytes) { - keyBytes := c.Certificates[c.ClientCertificateSecretKey][CertificateKeyKey] - certificateBytes := c.Certificates[c.ClientCertificateSecretKey][CertificateKey] - - return keyBytes, certificateBytes -} - -// Initialize initializes the Certificates object. Sets up a SharedInformer for the Secrets Resource. -func (c *Certificates) Initialize() error { - logrus.Info("Certificates::Initialize") - - var err error - - c.Certificates = make(map[string]map[string]core.SecretBytes) - - informer, err := c.buildInformer() - if err != nil { - return fmt.Errorf(`error occurred building an informer: %w`, err) - } - - c.informer = informer - - err = c.initializeEventHandlers() - if err != nil { - return fmt.Errorf(`error occurred initializing event handlers: %w`, err) - } - - return nil -} - -// Run starts the SharedInformer. -func (c *Certificates) Run() error { - logrus.Info("Certificates::Run") - - if c.informer == nil { - return fmt.Errorf(`initialize must be called before Run`) - } - - c.informer.Run(c.Context.Done()) - - <-c.Context.Done() - - return nil -} - -func (c *Certificates) buildInformer() (cache.SharedInformer, error) { - logrus.Debug("Certificates::buildInformer") - - options := informers.WithNamespace(SecretsNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(c.k8sClient, 0, options) - informer := factory.Core().V1().Secrets().Informer() - - return informer, nil -} - -func (c *Certificates) initializeEventHandlers() error { - logrus.Debug("Certificates::initializeEventHandlers") - - var err error - - handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: c.handleAddEvent, - DeleteFunc: c.handleDeleteEvent, - UpdateFunc: c.handleUpdateEvent, - } - - c.eventHandlerRegistration, err = c.informer.AddEventHandler(handlers) - if err != nil { - return fmt.Errorf(`error occurred registering event handlers: %w`, err) - } - - return nil -} - -func (c *Certificates) handleAddEvent(obj interface{}) { - logrus.Debug("Certificates::handleAddEvent") - - secret, ok := obj.(*corev1.Secret) - if !ok { - logrus.Errorf("Certificates::handleAddEvent: unable to cast object to Secret") - return - } - - c.Certificates[secret.Name] = map[string]core.SecretBytes{} - - // Input from the secret comes in the form - // tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVCVEN... - // tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0l... - // Where the keys are `tls.crt` and `tls.key` and the values are []byte - for k, v := range secret.Data { - c.Certificates[secret.Name][k] = core.SecretBytes(v) - } - - logrus.Debugf("Certificates::handleAddEvent: certificates (%d)", len(c.Certificates)) -} - -func (c *Certificates) handleDeleteEvent(obj interface{}) { - logrus.Debug("Certificates::handleDeleteEvent") - - secret, ok := obj.(*corev1.Secret) - if !ok { - logrus.Errorf("Certificates::handleDeleteEvent: unable to cast object to Secret") - return - } - - if c.Certificates[secret.Name] != nil { - delete(c.Certificates, secret.Name) - } - - logrus.Debugf("Certificates::handleDeleteEvent: certificates (%d)", len(c.Certificates)) -} - -func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { - logrus.Debug("Certificates::handleUpdateEvent") - - secret, ok := newValue.(*corev1.Secret) - if !ok { - logrus.Errorf("Certificates::handleUpdateEvent: unable to cast object to Secret") - return - } - - for k, v := range secret.Data { - c.Certificates[secret.Name][k] = v - } - - logrus.Debugf("Certificates::handleUpdateEvent: certificates (%d)", len(c.Certificates)) -} diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go deleted file mode 100644 index c8edf14e..00000000 --- a/internal/certification/certificates_test.go +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package certification - -import ( - "context" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - "testing" - "time" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" -) - -func TestNewCertificate(t *testing.T) { - ctx := context.Background() - - certificates := NewCertificates(ctx, nil) - - if certificates == nil { - t.Fatalf(`certificates should not be nil`) - } -} - -func TestCertificates_Initialize(t *testing.T) { - certificates := NewCertificates(context.Background(), nil) - - err := certificates.Initialize() - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } -} - -func TestCertificates_RunWithoutInitialize(t *testing.T) { - certificates := NewCertificates(context.Background(), nil) - - err := certificates.Run() - if err == nil { - t.Fatalf(`Expected error`) - } - - if err.Error() != `initialize must be called before Run` { - t.Fatalf(`Unexpected error: %v`, err) - } -} - -func TestCertificates_EmptyCertificates(t *testing.T) { - certificates := NewCertificates(context.Background(), nil) - - err := certificates.Initialize() - if err != nil { - t.Fatalf(`error Initializing Certificates: %v`, err) - } - - caBytes := certificates.GetCACertificate() - if caBytes != nil { - t.Fatalf(`Expected nil CA certificate`) - } - - clientKey, clientCert := certificates.GetClientCertificate() - if clientKey != nil { - t.Fatalf(`Expected nil client key`) - } - if clientCert != nil { - t.Fatalf(`Expected nil client certificate`) - } -} - -func TestCertificates_ExerciseHandlers(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - k8sClient := fake.NewSimpleClientset() - - certificates := NewCertificates(ctx, k8sClient) - - _ = certificates.Initialize() - - certificates.CaCertificateSecretKey = CaCertificateSecretKey - - go func() { - err := certificates.Run() - if err != nil { - t.Fatalf("error running Certificates: %v", err) - } - }() - - cache.WaitForCacheSync(ctx.Done(), certificates.informer.HasSynced) - - secret := buildSecret() - - /* -- Test Create -- */ - - created, err := k8sClient.CoreV1().Secrets(SecretsNamespace).Create(ctx, secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf(`error creating the Secret: %v`, err) - } - - if created.Name != secret.Name { - t.Fatalf(`Expected name %v, got %v`, secret.Name, created.Name) - } - - time.Sleep(2 * time.Second) - - caBytes := certificates.GetCACertificate() - if caBytes == nil { - t.Fatalf(`Expected non-nil CA certificate`) - } - - /* -- Test Update -- */ - - secret.Labels = map[string]string{"updated": "true"} - _, err = k8sClient.CoreV1().Secrets(SecretsNamespace).Update(ctx, secret, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf(`error updating the Secret: %v`, err) - } - - time.Sleep(2 * time.Second) - - caBytes = certificates.GetCACertificate() - if caBytes == nil { - t.Fatalf(`Expected non-nil CA certificate`) - } - - /* -- Test Delete -- */ - - err = k8sClient.CoreV1().Secrets(SecretsNamespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf(`error deleting the Secret: %v`, err) - } - - time.Sleep(2 * time.Second) - - caBytes = certificates.GetCACertificate() - if caBytes != nil { - t.Fatalf(`Expected nil CA certificate, got: %v`, caBytes) - } -} - -func buildSecret() *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: CaCertificateSecretKey, - Namespace: SecretsNamespace, - }, - Data: map[string][]byte{ - CertificateKey: []byte(certificatePEM()), - CertificateKeyKey: []byte(keyPEM()), - }, - Type: corev1.SecretTypeTLS, - } -} - -// certificatePEM returns a PEM-encoded client certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func certificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// keyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func keyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} diff --git a/internal/certification/doc.go b/internal/certification/doc.go deleted file mode 100644 index 3388ea0a..00000000 --- a/internal/certification/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package certification includes functionality to access the Secrets containing the TLS Certificates. -*/ - -package certification diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 9a3d4113..2084d5cc 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -6,21 +6,19 @@ package communication import ( - "crypto/tls" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" + "fmt" netHttp "net/http" "time" + + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" ) -// NewHttpClient is a factory method to create a new Http Client with a default configuration. -// RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, -// the Headers are configured for JSON. -func NewHttpClient(settings *configuration.Settings) (*netHttp.Client, error) { - headers := NewHeaders() - tlsConfig := NewTlsConfig(settings) - transport := NewTransport(tlsConfig) +// NewHTTPClient is a factory method to create a new Http Client configured for +// working with NGINXaaS or the N+ api. If skipVerify is set to true, the http +// transport will skip TLS certificate verification. +func NewHTTPClient(apiKey string, skipVerify bool) (*netHttp.Client, error) { + headers := NewHeaders(apiKey) + transport := NewTransport(skipVerify) roundTripper := NewRoundTripper(headers, transport) return &netHttp.Client{ @@ -32,29 +30,24 @@ func NewHttpClient(settings *configuration.Settings) (*netHttp.Client, error) { } // NewHeaders is a factory method to create a new basic Http Headers slice. -func NewHeaders() []string { - return []string{ +func NewHeaders(apiKey string) []string { + headers := []string{ "Content-Type: application/json", "Accept: application/json", + fmt.Sprintf("X-NLK-Version: %s", buildinfo.SemVer()), } -} -// NewTlsConfig is a factory method to create a new basic Tls Config. -// More attention should be given to the use of `InsecureSkipVerify: true`, as it is not recommended for production use. -func NewTlsConfig(settings *configuration.Settings) *tls.Config { - tlsConfig, err := authentication.NewTlsConfig(settings) - if err != nil { - logrus.Warnf("Failed to create TLS config: %v", err) - return &tls.Config{InsecureSkipVerify: true} + if apiKey != "" { + headers = append(headers, fmt.Sprintf("Authorization: ApiKey %s", apiKey)) } - return tlsConfig + return headers } // NewTransport is a factory method to create a new basic Http Transport. -func NewTransport(config *tls.Config) *netHttp.Transport { - transport := netHttp.DefaultTransport.(*netHttp.Transport) - transport.TLSClientConfig = config +func NewTransport(skipVerify bool) *netHttp.Transport { + transport := netHttp.DefaultTransport.(*netHttp.Transport).Clone() + transport.TLSClientConfig.InsecureSkipVerify = skipVerify return transport } diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index f25abefb..8c637a8a 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -6,17 +6,15 @@ package communication import ( - "context" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" "testing" + + "github.com/stretchr/testify/require" ) -func TestNewHttpClient(t *testing.T) { - k8sClient := fake.NewSimpleClientset() - settings, err := configuration.NewSettings(context.Background(), k8sClient) - client, err := NewHttpClient(settings) +func TestNewHTTPClient(t *testing.T) { + t.Parallel() + client, err := NewHTTPClient("fakeKey", true) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -26,14 +24,45 @@ func TestNewHttpClient(t *testing.T) { } } +//nolint:goconst func TestNewHeaders(t *testing.T) { - headers := NewHeaders() + t.Parallel() + headers := NewHeaders("fakeKey") if headers == nil { t.Fatalf(`headers should not be nil`) } - if len(headers) != 2 { + if len(headers) != 4 { + t.Fatalf(`headers should have 3 elements`) + } + + if headers[0] != "Content-Type: application/json" { + t.Fatalf(`headers[0] should be "Content-Type: application/json"`) + } + + if headers[1] != "Accept: application/json" { + t.Fatalf(`headers[1] should be "Accept: application/json"`) + } + + if headers[2] != "X-NLK-Version: " { + t.Fatalf(`headers[2] should be "X-NLK-Version: "`) + } + + if headers[3] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`headers[3] should be "Accept: Authorization: ApiKey fakeKey"`) + } +} + +func TestNewHeadersWithNoAPIKey(t *testing.T) { + t.Parallel() + headers := NewHeaders("") + + if headers == nil { + t.Fatalf(`headers should not be nil`) + } + + if len(headers) != 3 { t.Fatalf(`headers should have 2 elements`) } @@ -44,13 +73,16 @@ func TestNewHeaders(t *testing.T) { if headers[1] != "Accept: application/json" { t.Fatalf(`headers[1] should be "Accept: application/json"`) } + + if headers[2] != "X-NLK-Version: " { + t.Fatalf(`headers[2] should be "X-NLK-Version: "`) + } } func TestNewTransport(t *testing.T) { - k8sClient := fake.NewSimpleClientset() - settings, _ := configuration.NewSettings(context.Background(), k8sClient) - config := NewTlsConfig(settings) - transport := NewTransport(config) + t.Parallel() + + transport := NewTransport(false) if transport == nil { t.Fatalf(`transport should not be nil`) @@ -60,11 +92,5 @@ func TestNewTransport(t *testing.T) { t.Fatalf(`transport.TLSClientConfig should not be nil`) } - if transport.TLSClientConfig != config { - t.Fatalf(`transport.TLSClientConfig should be the same as config`) - } - - if !transport.TLSClientConfig.InsecureSkipVerify { - t.Fatalf(`transport.TLSClientConfig.InsecureSkipVerify should be true`) - } + require.False(t, transport.TLSClientConfig.InsecureSkipVerify) } diff --git a/internal/communication/roundtripper.go b/internal/communication/roundtripper.go index 3781c62d..1dbaf5b0 100644 --- a/internal/communication/roundtripper.go +++ b/internal/communication/roundtripper.go @@ -6,11 +6,13 @@ package communication import ( + "errors" "net/http" - netHttp "net/http" "strings" ) +const maxHeaders = 1000 + // RoundTripper is a simple type that wraps the default net/communication RoundTripper to add additional headers. type RoundTripper struct { Headers []string @@ -18,7 +20,7 @@ type RoundTripper struct { } // NewRoundTripper is a factory method to create a new RoundTripper. -func NewRoundTripper(headers []string, transport *netHttp.Transport) *RoundTripper { +func NewRoundTripper(headers []string, transport *http.Transport) *RoundTripper { return &RoundTripper{ Headers: headers, RoundTripper: transport, @@ -27,6 +29,10 @@ func NewRoundTripper(headers []string, transport *netHttp.Transport) *RoundTripp // RoundTrip This simply adds our default headers to the request before passing it on to the default RoundTripper. func (roundTripper *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if len(req.Header) > maxHeaders { + return nil, errors.New("request includes too many headers") + } + newRequest := new(http.Request) *newRequest = *req newRequest.Header = make(http.Header, len(req.Header)) diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index f46fb710..d00af173 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -7,18 +7,15 @@ package communication import ( "bytes" - "context" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" netHttp "net/http" "testing" ) func TestNewRoundTripper(t *testing.T) { - k8sClient := fake.NewSimpleClientset() - settings, _ := configuration.NewSettings(context.Background(), k8sClient) - headers := NewHeaders() - transport := NewTransport(NewTlsConfig(settings)) + t.Parallel() + + headers := NewHeaders("fakeKey") + transport := NewTransport(true) roundTripper := NewRoundTripper(headers, transport) if roundTripper == nil { @@ -29,8 +26,8 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers should not be nil`) } - if len(roundTripper.Headers) != 2 { - t.Fatalf(`roundTripper.Headers should have 2 elements`) + if len(roundTripper.Headers) != 4 { + t.Fatalf(`roundTripper.Headers should have 3 elements`) } if roundTripper.Headers[0] != "Content-Type: application/json" { @@ -41,16 +38,24 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers[1] should be "Accept: application/json"`) } + if roundTripper.Headers[2] != "X-NLK-Version: " { + t.Fatalf(`roundTripper.Headers[2] should be "X-NLK-Version: "`) + } + + if roundTripper.Headers[3] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`roundTripper.Headers[3] should be "Accept: Authorization: ApiKey fakeKey"`) + } + if roundTripper.RoundTripper == nil { t.Fatalf(`roundTripper.RoundTripper should not be nil`) } } func TestRoundTripperRoundTrip(t *testing.T) { - k8sClient := fake.NewSimpleClientset() - settings, err := configuration.NewSettings(context.Background(), k8sClient) - headers := NewHeaders() - transport := NewTransport(NewTlsConfig(settings)) + t.Parallel() + + headers := NewHeaders("fakeKey") + transport := NewTransport(true) roundTripper := NewRoundTripper(headers, transport) request, err := NewRequest("GET", "http://example.com", nil) @@ -69,10 +74,11 @@ func TestRoundTripperRoundTrip(t *testing.T) { if response == nil { t.Fatalf(`response should not be nil`) } + defer response.Body.Close() headerLen := len(response.Header) - if headerLen <= 2 { - t.Fatalf(`response.Header should have at least 2 elements, found %d`, headerLen) + if headerLen <= 3 { + t.Fatalf(`response.Header should have at least 3 elements, found %d`, headerLen) } } diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go new file mode 100644 index 00000000..45764d6e --- /dev/null +++ b/internal/configuration/configuration_test.go @@ -0,0 +1,138 @@ +package configuration_test + +import ( + "testing" + "time" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + + "github.com/stretchr/testify/require" +) + +func TestConfiguration_Read(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + testFile string + expectedSettings configuration.Settings + }{ + "one nginx plus host": { + testFile: "one-nginx-host", + expectedSettings: configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, + SkipVerifyTLS: false, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, + }, + }, + "multiple nginx plus hosts": { + testFile: "multiple-nginx-hosts", + expectedSettings: configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"}, + SkipVerifyTLS: true, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + settings, err := configuration.Read(tc.testFile, "./testdata") + require.NoError(t, err) + require.Equal(t, tc.expectedSettings, settings) + }) + } +} + +func TestConfiguration_TLS(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + tlsMode string + expectedSkipVerifyTLS bool + expectedErr bool + }{ + "no input": { + tlsMode: "", + expectedSkipVerifyTLS: false, + }, + "no tls": { + tlsMode: "no-tls", + expectedSkipVerifyTLS: true, + }, + "skip verify tls": { + tlsMode: "skip-verify-tls", + expectedSkipVerifyTLS: true, + }, + "ca tls": { + tlsMode: "ca-tls", + expectedSkipVerifyTLS: false, + }, + "unexpected input": { + tlsMode: "unexpected-tls-mode", + expectedErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + skipVerifyTLS, err := configuration.ValidateTLSMode(tc.tlsMode) + if tc.expectedErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, tc.expectedSkipVerifyTLS, skipVerifyTLS) + }) + } +} diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index d15ad845..75cec2e5 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -6,18 +6,12 @@ package configuration import ( - "context" + "encoding/base64" "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "strings" + "log/slog" "time" + + "github.com/spf13/viper" ) const ( @@ -39,13 +33,22 @@ const ( // The value of the annotation determines which BorderServer implementation will be used. // See the documentation in the `application/application_constants.go` file for details. PortAnnotationPrefix = "nginxinc.io" + + // ServiceAnnotationMatchKey is the key name of the annotation in the application's config map + // that identifies the ingress service whose events will be monitored. + ServiceAnnotationMatchKey = "service-annotation-match" + + // DefaultServiceAnnotation is the default name of the ingress service whose events will be + // monitored. + DefaultServiceAnnotation = "nginxaas" ) // WorkQueueSettings contains the configuration values needed by the Work Queues. // There are two work queues in the application: // 1. nlk-handler queue, used to move messages between the Watcher and the Handler. // 2. nlk-synchronizer queue, used to move message between the Handler and the Synchronizer. -// The queues are NamedDelayingQueue objects that use an ItemExponentialFailureRateLimiter as the underlying rate limiter. +// The queues are NamedDelayingQueue objects that use an ItemExponentialFailureRateLimiter +// as the underlying rate limiter. type WorkQueueSettings struct { // Name is the name of the queue. Name string @@ -60,7 +63,6 @@ type WorkQueueSettings struct { // HandlerSettings contains the configuration values needed by the Handler. type HandlerSettings struct { - // RetryCount is the number of times the Handler will attempt to process a message before giving up. RetryCount int @@ -73,9 +75,8 @@ type HandlerSettings struct { // WatcherSettings contains the configuration values needed by the Watcher. type WatcherSettings struct { - - // NginxIngressNamespace is the namespace used to filter Services in the Watcher. - NginxIngressNamespace string + // ServiceAnnotation is the annotation of the ingress service whose events the watcher should monitor. + ServiceAnnotation string // ResyncPeriod is the value used to set the resync period for the underlying SharedInformer. ResyncPeriod time.Duration @@ -83,7 +84,6 @@ type WatcherSettings struct { // SynchronizerSettings contains the configuration values needed by the Synchronizer. type SynchronizerSettings struct { - // MaxMillisecondsJitter is the maximum number of milliseconds that will be applied when adding an event to the queue. MaxMillisecondsJitter int @@ -102,27 +102,17 @@ type SynchronizerSettings struct { // Settings contains the configuration values needed by the application. type Settings struct { - - // Context is the context used to control the application. - Context context.Context + // LogLevel is the user-specified log level. Defaults to warn. + LogLevel string // NginxPlusHosts is a list of Nginx Plus hosts that will be used to update the Border Servers. NginxPlusHosts []string - // TlsMode is the value used to determine which of the five TLS modes will be used to communicate with the Border Servers (see: ../../docs/tls/README.md). - TlsMode TLSMode + // SkipVerifyTLS determines whether the http client will skip TLS verification or not. + SkipVerifyTLS bool - // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. - Certificates *certification.Certificates - - // K8sClient is the Kubernetes client used to communicate with the Kubernetes API. - K8sClient kubernetes.Interface - - // informer is the SharedInformer used to watch for changes to the ConfigMap . - informer cache.SharedInformer - - // eventHandlerRegistration is the object used to track the event handlers with the SharedInformer. - eventHandlerRegistration cache.ResourceEventHandlerRegistration + // APIKey is the api key used to authenticate with the dataplane API. + APIKey string // Handler contains the configuration values needed by the Handler. Handler HandlerSettings @@ -134,13 +124,39 @@ type Settings struct { Watcher WatcherSettings } -// NewSettings creates a new Settings object with default values. -func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings, error) { - settings := &Settings{ - Context: ctx, - K8sClient: k8sClient, - TlsMode: NoTLS, - Certificates: nil, +// Read parses all the config and returns the values +func Read(configName, configPath string) (s Settings, err error) { + v := viper.New() + v.SetConfigName(configName) + v.SetConfigType("yaml") + v.AddConfigPath(configPath) + if err = v.ReadInConfig(); err != nil { + return s, err + } + + if err = v.BindEnv("NGINXAAS_DATAPLANE_API_KEY"); err != nil { + return s, err + } + + skipVerifyTLS, err := ValidateTLSMode(v.GetString("tls-mode")) + if err != nil { + slog.Error("could not validate tls mode", "error", err) + } + + if skipVerifyTLS { + slog.Warn("skipping TLS verification for NGINX hosts") + } + + serviceAnnotation := DefaultServiceAnnotation + if sa := v.GetString(ServiceAnnotationMatchKey); sa != "" { + serviceAnnotation = sa + } + + return Settings{ + LogLevel: v.GetString("log-level"), + NginxPlusHosts: v.GetStringSlice("nginx-hosts"), + SkipVerifyTLS: skipVerifyTLS, + APIKey: base64.StdEncoding.EncodeToString([]byte(v.GetString("NGINXAAS_DATAPLANE_API_KEY"))), Handler: HandlerSettings{ RetryCount: 5, Threads: 1, @@ -162,208 +178,21 @@ func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings }, }, Watcher: WatcherSettings{ - NginxIngressNamespace: "nginx-ingress", - ResyncPeriod: 0, + ResyncPeriod: 0, + ServiceAnnotation: serviceAnnotation, }, - } - - return settings, nil -} - -// Initialize initializes the Settings object. Sets up a SharedInformer to watch for changes to the ConfigMap. -// This method must be called before the Run method. -func (s *Settings) Initialize() error { - logrus.Info("Settings::Initialize") - - var err error - - certificates := certification.NewCertificates(s.Context, s.K8sClient) - - err = certificates.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing certificates: %w`, err) - } - - s.Certificates = certificates - - go certificates.Run() - - logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieving nlk-config ConfigMap") - configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get(s.Context, "nlk-config", metav1.GetOptions{}) - if err != nil { - return err - } - - s.handleUpdateEvent(nil, configMap) - logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieved nlk-config ConfigMap") - - informer, err := s.buildInformer() - if err != nil { - return fmt.Errorf(`error occurred building ConfigMap informer: %w`, err) - } - - s.informer = informer - - err = s.initializeEventListeners() - if err != nil { - return fmt.Errorf(`error occurred initializing event listeners: %w`, err) - } - - return nil -} - -// Run starts the SharedInformer and waits for the Context to be cancelled. -func (s *Settings) Run() { - logrus.Debug("Settings::Run") - - defer utilruntime.HandleCrash() - - go s.informer.Run(s.Context.Done()) - - <-s.Context.Done() -} - -func (s *Settings) buildInformer() (cache.SharedInformer, error) { - options := informers.WithNamespace(ConfigMapsNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(s.K8sClient, ResyncPeriod, options) - informer := factory.Core().V1().ConfigMaps().Informer() - - return informer, nil -} - -func (s *Settings) initializeEventListeners() error { - logrus.Debug("Settings::initializeEventListeners") - - var err error - - handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: s.handleAddEvent, - UpdateFunc: s.handleUpdateEvent, - DeleteFunc: s.handleDeleteEvent, - } - - s.eventHandlerRegistration, err = s.informer.AddEventHandler(handlers) - if err != nil { - return fmt.Errorf(`error occurred registering event handlers: %w`, err) - } - - return nil -} - -func (s *Settings) handleAddEvent(obj interface{}) { - logrus.Debug("Settings::handleAddEvent") - - if _, yes := isOurConfig(obj); yes { - s.handleUpdateEvent(nil, obj) - } -} - -func (s *Settings) handleDeleteEvent(obj interface{}) { - logrus.Debug("Settings::handleDeleteEvent") - - if _, yes := isOurConfig(obj); yes { - s.updateHosts([]string{}) - } -} - -func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { - logrus.Debug("Settings::handleUpdateEvent") - - configMap, yes := isOurConfig(newValue) - if !yes { - return - } - - hosts, found := configMap.Data["nginx-hosts"] - if found { - newHosts := s.parseHosts(hosts) - s.updateHosts(newHosts) - } else { - logrus.Warnf("Settings::handleUpdateEvent: nginx-hosts key not found in ConfigMap") - } - - tlsMode, err := validateTlsMode(configMap) - if err != nil { - // NOTE: the TLSMode defaults to NoTLS on startup, or the last known good value if previously set. - logrus.Errorf("There was an error with the configured TLS Mode. TLS Mode has NOT been changed. The current mode is: '%v'. Error: %v. ", s.TlsMode, err) - } else { - s.TlsMode = tlsMode - } - - caCertificateSecretKey, found := configMap.Data["ca-certificate"] - if found { - s.Certificates.CaCertificateSecretKey = caCertificateSecretKey - logrus.Debugf("Settings::handleUpdateEvent: ca-certificate: %s", s.Certificates.CaCertificateSecretKey) - } else { - s.Certificates.CaCertificateSecretKey = "" - logrus.Warnf("Settings::handleUpdateEvent: ca-certificate key not found in ConfigMap") - } - - clientCertificateSecretKey, found := configMap.Data["client-certificate"] - if found { - s.Certificates.ClientCertificateSecretKey = clientCertificateSecretKey - logrus.Debugf("Settings::handleUpdateEvent: client-certificate: %s", s.Certificates.ClientCertificateSecretKey) - } else { - s.Certificates.ClientCertificateSecretKey = "" - logrus.Warnf("Settings::handleUpdateEvent: client-certificate key not found in ConfigMap") - } - - setLogLevel(configMap.Data["log-level"]) - - logrus.Debugf("Settings::handleUpdateEvent: \n\tHosts: %v,\n\tSettings: %v ", s.NginxPlusHosts, configMap) + }, nil } -func validateTlsMode(configMap *corev1.ConfigMap) (TLSMode, error) { - tlsConfigMode, tlsConfigModeFound := configMap.Data["tls-mode"] - if !tlsConfigModeFound { - return NoTLS, fmt.Errorf(`tls-mode key not found in ConfigMap`) +func ValidateTLSMode(tlsConfigMode string) (skipVerify bool, err error) { + if tlsConfigMode == "" { + return false, nil } - if tlsMode, tlsModeFound := TLSModeMap[tlsConfigMode]; tlsModeFound { - return tlsMode, nil + var tlsModeFound bool + if skipVerify, tlsModeFound = tlsModeMap[tlsConfigMode]; tlsModeFound { + return skipVerify, nil } - return NoTLS, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) -} - -func (s *Settings) parseHosts(hosts string) []string { - return strings.Split(hosts, ",") -} - -func (s *Settings) updateHosts(hosts []string) { - s.NginxPlusHosts = hosts -} - -func isOurConfig(obj interface{}) (*corev1.ConfigMap, bool) { - configMap, ok := obj.(*corev1.ConfigMap) - return configMap, ok && configMap.Name == ConfigMapName && configMap.Namespace == ConfigMapsNamespace -} - -func setLogLevel(logLevel string) { - logrus.Debugf("Settings::setLogLevel: %s", logLevel) - switch logLevel { - case "panic": - logrus.SetLevel(logrus.PanicLevel) - - case "fatal": - logrus.SetLevel(logrus.FatalLevel) - - case "error": - logrus.SetLevel(logrus.ErrorLevel) - - case "warn": - logrus.SetLevel(logrus.WarnLevel) - - case "info": - logrus.SetLevel(logrus.InfoLevel) - - case "debug": - logrus.SetLevel(logrus.DebugLevel) - - case "trace": - logrus.SetLevel(logrus.TraceLevel) - - default: - logrus.SetLevel(logrus.WarnLevel) - } + return false, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) } diff --git a/internal/configuration/testdata/multiple-nginx-hosts.yaml b/internal/configuration/testdata/multiple-nginx-hosts.yaml new file mode 100644 index 00000000..2235c1ae --- /dev/null +++ b/internal/configuration/testdata/multiple-nginx-hosts.yaml @@ -0,0 +1,11 @@ +ca-certificate: "fakeCAKey" +client-certificate: "fakeCertKey" +log-level: "warn" +nginx-hosts: ["https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"] +tls-mode: "no-tls" +service-annotation-match: "fakeServiceMatch" +creationTimestamp: "2024-09-04T17:59:20Z" +name: "nlk-config" +namespace: "nlk" +resourceVersion: "5909" +uid: "66d49974-49d6-4ad8-8135-dcebda7b5c9e" diff --git a/internal/configuration/testdata/one-nginx-host.yaml b/internal/configuration/testdata/one-nginx-host.yaml new file mode 100644 index 00000000..45e55eff --- /dev/null +++ b/internal/configuration/testdata/one-nginx-host.yaml @@ -0,0 +1,10 @@ +ca-certificate: "fakeCAKey" +client-certificate: "fakeCertKey" +log-level: "warn" +nginx-hosts: "https://10.0.0.1:9000/api" +service-annotation-match: "fakeServiceMatch" +creationTimestamp: "2024-09-04T17:59:20Z" +name: "nlk-config" +namespace: "nlk" +resourceVersion: "5909" +uid: "66d49974-49d6-4ad8-8135-dcebda7b5c9e" diff --git a/internal/configuration/tlsmodes.go b/internal/configuration/tlsmodes.go index 2f7271f2..e329047e 100644 --- a/internal/configuration/tlsmodes.go +++ b/internal/configuration/tlsmodes.go @@ -6,41 +6,19 @@ package configuration const ( - NoTLS TLSMode = iota - CertificateAuthorityTLS - CertificateAuthorityMutualTLS - SelfSignedTLS - SelfSignedMutualTLS + // NoTLS is deprecated as misleading. It is the same as SkipVerifyTLS. + NoTLS = "no-tls" + // SkipVerifyTLS causes the http client to skip verification of the NGINX + // host's certificate chain and host name. + SkipVerifyTLS = "skip-verify-tls" + // CertificateAuthorityTLS is deprecated as misleading. This is the same as + // the default behavior which is to verify the NGINX hosts's certificate + // chain and host name, if https is used. + CertificateAuthorityTLS = "ca-tls" ) -const ( - NoTLSString = "no-tls" - CertificateAuthorityTLSString = "ca-tls" - CertificateAuthorityMutualTLSString = "ca-mtls" - SelfSignedTLSString = "ss-tls" - SelfSignedMutualTLSString = "ss-mtls" -) - -type TLSMode int - -var TLSModeMap = map[string]TLSMode{ - NoTLSString: NoTLS, - CertificateAuthorityTLSString: CertificateAuthorityTLS, - CertificateAuthorityMutualTLSString: CertificateAuthorityMutualTLS, - SelfSignedTLSString: SelfSignedTLS, - SelfSignedMutualTLSString: SelfSignedMutualTLS, -} - -func (t TLSMode) String() string { - modes := []string{ - NoTLSString, - CertificateAuthorityTLSString, - CertificateAuthorityMutualTLSString, - SelfSignedTLSString, - SelfSignedMutualTLSString, - } - if t < NoTLS || t > SelfSignedMutualTLS { - return "" - } - return modes[t] +var tlsModeMap = map[string]bool{ + NoTLS: true, + SkipVerifyTLS: true, + CertificateAuthorityTLS: false, } diff --git a/internal/configuration/tlsmodes_test.go b/internal/configuration/tlsmodes_test.go deleted file mode 100644 index 62abf962..00000000 --- a/internal/configuration/tlsmodes_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package configuration - -import ( - "testing" -) - -func Test_String(t *testing.T) { - mode := NoTLS.String() - if mode != "no-tls" { - t.Errorf("Expected TLSModeNoTLS to be 'no-tls', got '%s'", mode) - } - - mode = CertificateAuthorityTLS.String() - if mode != "ca-tls" { - t.Errorf("Expected TLSModeCaTLS to be 'ca-tls', got '%s'", mode) - } - - mode = CertificateAuthorityMutualTLS.String() - if mode != "ca-mtls" { - t.Errorf("Expected TLSModeCaMTLS to be 'ca-mtls', got '%s'", mode) - } - - mode = SelfSignedTLS.String() - if mode != "ss-tls" { - t.Errorf("Expected TLSModeSsTLS to be 'ss-tls', got '%s'", mode) - } - - mode = SelfSignedMutualTLS.String() - if mode != "ss-mtls" { - t.Errorf("Expected TLSModeSsMTLS to be 'ss-mtls', got '%s',", mode) - } - - mode = TLSMode(5).String() - if mode != "" { - t.Errorf("Expected TLSMode(5) to be '', got '%s'", mode) - } -} - -func Test_TLSModeMap(t *testing.T) { - mode := TLSModeMap["no-tls"] - if mode != NoTLS { - t.Errorf("Expected TLSModeMap['no-tls'] to be TLSModeNoTLS, got '%d'", mode) - } - - mode = TLSModeMap["ca-tls"] - if mode != CertificateAuthorityTLS { - t.Errorf("Expected TLSModeMap['ca-tls'] to be TLSModeCaTLS, got '%d'", mode) - } - - mode = TLSModeMap["ca-mtls"] - if mode != CertificateAuthorityMutualTLS { - t.Errorf("Expected TLSModeMap['ca-mtls'] to be TLSModeCaMTLS, got '%d'", mode) - } - - mode = TLSModeMap["ss-tls"] - if mode != SelfSignedTLS { - t.Errorf("Expected TLSModeMap['ss-tls'] to be TLSModeSsTLS, got '%d'", mode) - } - - mode = TLSModeMap["ss-mtls"] - if mode != SelfSignedMutualTLS { - t.Errorf("Expected TLSModeMap['ss-mtls'] to be TLSModeSsMTLS, got '%d'", mode) - } - - mode = TLSModeMap["invalid"] - if mode != TLSMode(0) { - t.Errorf("Expected TLSModeMap['invalid'] to be TLSMode(0), got '%d'", mode) - } -} diff --git a/internal/core/event.go b/internal/core/event.go index 09776c94..c32511e1 100644 --- a/internal/core/event.go +++ b/internal/core/event.go @@ -24,27 +24,17 @@ const ( // Event represents a service event type Event struct { - // Type represents the event type, one of the constant values defined above. Type EventType // Service represents the service object in its current state Service *v1.Service - - // PreviousService represents the service object in its previous state - PreviousService *v1.Service - - // NodeIps represents the list of node IPs in the Cluster. This is populated by the Watcher when an event is created. - // The Node IPs are needed by the BorderClient. - NodeIps []string } // NewEvent factory method to create a new Event -func NewEvent(eventType EventType, service *v1.Service, previousService *v1.Service, nodeIps []string) Event { +func NewEvent(eventType EventType, service *v1.Service) Event { return Event{ - Type: eventType, - Service: service, - PreviousService: previousService, - NodeIps: nodeIps, + Type: eventType, + Service: service, } } diff --git a/internal/core/event_test.go b/internal/core/event_test.go index b3b89261..09724cfa 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -1,17 +1,17 @@ package core import ( - v1 "k8s.io/api/core/v1" "testing" + + v1 "k8s.io/api/core/v1" ) func TestNewEvent(t *testing.T) { + t.Parallel() expectedType := Created expectedService := &v1.Service{} - expectedPreviousService := &v1.Service{} - expectedNodeIps := []string{"127.0.0.1"} - event := NewEvent(expectedType, expectedService, expectedPreviousService, expectedNodeIps) + event := NewEvent(expectedType, expectedService) if event.Type != expectedType { t.Errorf("expected Created, got %v", event.Type) @@ -20,12 +20,4 @@ func TestNewEvent(t *testing.T) { if event.Service != expectedService { t.Errorf("expected service, got %#v", event.Service) } - - if event.PreviousService != expectedPreviousService { - t.Errorf("expected previous service, got %#v", event.PreviousService) - } - - if event.NodeIps[0] != expectedNodeIps[0] { - t.Errorf("expected node ips, got %#v", event.NodeIps) - } } diff --git a/internal/core/secret_bytes.go b/internal/core/secret_bytes.go deleted file mode 100644 index 0bbc3bf1..00000000 --- a/internal/core/secret_bytes.go +++ /dev/null @@ -1,21 +0,0 @@ -package core - -import ( - "encoding/json" -) - -// Wraps byte slices which potentially could contain -// sensitive data that should not be output to the logs. -// This will output [REDACTED] if attempts are made -// to print this type in logs, serialize to JSON, or -// otherwise convert it to a string. -// Usage: core.SecretBytes(myByteSlice) -type SecretBytes []byte - -func (sb SecretBytes) String() string { - return "[REDACTED]" -} - -func (sb SecretBytes) MarshalJSON() ([]byte, error) { - return json.Marshal("[REDACTED]") -} diff --git a/internal/core/secret_bytes_test.go b/internal/core/secret_bytes_test.go deleted file mode 100644 index 8dd80247..00000000 --- a/internal/core/secret_bytes_test.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package core - -import ( - "encoding/json" - "fmt" - "testing" -) - -func TestSecretBytesToString(t *testing.T) { - sensitive := SecretBytes([]byte("If you can see this we have a problem")) - - expected := "foo [REDACTED] bar" - result := fmt.Sprintf("foo %v bar", sensitive) - if result != expected { - t.Errorf("Expected %s, got %s", expected, result) - } -} - -func TestSecretBytesToJSON(t *testing.T) { - sensitive, _ := json.Marshal(SecretBytes([]byte("If you can see this we have a problem"))) - expected := `foo "[REDACTED]" bar` - result := fmt.Sprintf("foo %v bar", string(sensitive)) - if result != expected { - t.Errorf("Expected %s, got %s", expected, result) - } -} diff --git a/internal/core/server_update_event.go b/internal/core/server_update_event.go index f3961eae..04d2dc08 100644 --- a/internal/core/server_update_event.go +++ b/internal/core/server_update_event.go @@ -9,14 +9,10 @@ package core // from Events received from the Handler. These are then consumed by the Synchronizer and passed along to // the appropriate BorderClient. type ServerUpdateEvent struct { - // ClientType is the type of BorderClient that should handle this event. This is configured via Service Annotations. // See application_constants.go for the list of supported types. ClientType string - // Id is the unique identifier for this event. - Id string - // NginxHost is the host name of the NGINX Plus instance that should handle this event. NginxHost string @@ -34,7 +30,12 @@ type ServerUpdateEvent struct { type ServerUpdateEvents = []*ServerUpdateEvent // NewServerUpdateEvent creates a new ServerUpdateEvent. -func NewServerUpdateEvent(eventType EventType, upstreamName string, clientType string, upstreamServers UpstreamServers) *ServerUpdateEvent { +func NewServerUpdateEvent( + eventType EventType, + upstreamName string, + clientType string, + upstreamServers UpstreamServers, +) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: clientType, Type: eventType, @@ -43,11 +44,10 @@ func NewServerUpdateEvent(eventType EventType, upstreamName string, clientType s } } -// ServerUpdateEventWithIdAndHost creates a new ServerUpdateEvent with the specified Id and Host. -func ServerUpdateEventWithIdAndHost(event *ServerUpdateEvent, id string, nginxHost string) *ServerUpdateEvent { +// ServerUpdateEventWithHost creates a new ServerUpdateEvent with the specified Host. +func ServerUpdateEventWithHost(event *ServerUpdateEvent, nginxHost string) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: event.ClientType, - Id: id, NginxHost: nginxHost, Type: event.Type, UpstreamName: event.UpstreamName, diff --git a/internal/core/server_update_event_test.go b/internal/core/server_update_event_test.go index a891e237..3be4702c 100644 --- a/internal/core/server_update_event_test.go +++ b/internal/core/server_update_event_test.go @@ -14,32 +14,26 @@ const clientType = "clientType" var emptyUpstreamServers UpstreamServers func TestServerUpdateEventWithIdAndHost(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) - if event.Id != "" { - t.Errorf("expected empty Id, got %s", event.Id) - } - if event.NginxHost != "" { t.Errorf("expected empty NginxHost, got %s", event.NginxHost) } - eventWithIdAndHost := ServerUpdateEventWithIdAndHost(event, "id", "host") - - if eventWithIdAndHost.Id != "id" { - t.Errorf("expected Id to be 'id', got %s", eventWithIdAndHost.Id) - } + eventWithIDAndHost := ServerUpdateEventWithHost(event, "host") - if eventWithIdAndHost.NginxHost != "host" { - t.Errorf("expected NginxHost to be 'host', got %s", eventWithIdAndHost.NginxHost) + if eventWithIDAndHost.NginxHost != "host" { + t.Errorf("expected NginxHost to be 'host', got %s", eventWithIDAndHost.NginxHost) } - if eventWithIdAndHost.ClientType != clientType { - t.Errorf("expected ClientType to be '%s', got %s", clientType, eventWithIdAndHost.ClientType) + if eventWithIDAndHost.ClientType != clientType { + t.Errorf("expected ClientType to be '%s', got %s", clientType, eventWithIDAndHost.ClientType) } } func TestTypeNameCreated(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Created" { @@ -48,6 +42,7 @@ func TestTypeNameCreated(t *testing.T) { } func TestTypeNameUpdated(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Updated, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Updated" { @@ -56,6 +51,7 @@ func TestTypeNameUpdated(t *testing.T) { } func TestTypeNameDeleted(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Deleted, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Deleted" { @@ -64,6 +60,7 @@ func TestTypeNameDeleted(t *testing.T) { } func TestTypeNameUnknown(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(EventType(100), "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Unknown" { diff --git a/internal/core/upstream_server.go b/internal/core/upstream_server.go index 7c89b1e2..eeb72ac0 100644 --- a/internal/core/upstream_server.go +++ b/internal/core/upstream_server.go @@ -5,7 +5,8 @@ package core -// UpstreamServer represents a single upstream server. This is an internal representation used to abstract the definition +// UpstreamServer represents a single upstream server. +// This is an internal representation used to abstract the definition // of an upstream server from any specific client. type UpstreamServer struct { diff --git a/internal/core/upstream_server_test.go b/internal/core/upstream_server_test.go index 7b0eed52..91592cd3 100644 --- a/internal/core/upstream_server_test.go +++ b/internal/core/upstream_server_test.go @@ -8,6 +8,7 @@ package core import "testing" func TestNewUpstreamServer(t *testing.T) { + t.Parallel() host := "localhost" us := NewUpstreamServer(host) if us.Host != host { diff --git a/internal/observation/handler.go b/internal/observation/handler.go deleted file mode 100644 index 83601b0f..00000000 --- a/internal/observation/handler.go +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package observation - -import ( - "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/workqueue" -) - -// HandlerInterface is the interface for the event handler -type HandlerInterface interface { - - // AddRateLimitedEvent defines the interface for adding an event to the event queue - AddRateLimitedEvent(event *core.Event) - - // Run defines the interface used to start the event handler - Run(stopCh <-chan struct{}) - - // ShutDown defines the interface used to stop the event handler - ShutDown() -} - -// Handler is responsible for processing events in the "nlk-handler" queue. -// When processing a message the Translation module is used to translate the event into an internal representation. -// The translation process may result in multiple events being generated. This fan-out mainly supports the differences -// in NGINX Plus API calls for creating/updating Upstreams and deleting Upstreams. -type Handler struct { - - // eventQueue is the queue used to store events - eventQueue workqueue.RateLimitingInterface - - // settings is the configuration settings - settings *configuration.Settings - - // synchronizer is the synchronizer used to synchronize the internal representation with a Border Server - synchronizer synchronization.Interface -} - -// NewHandler creates a new event handler -func NewHandler(settings *configuration.Settings, synchronizer synchronization.Interface, eventQueue workqueue.RateLimitingInterface) *Handler { - return &Handler{ - eventQueue: eventQueue, - settings: settings, - synchronizer: synchronizer, - } -} - -// AddRateLimitedEvent adds an event to the event queue -func (h *Handler) AddRateLimitedEvent(event *core.Event) { - logrus.Debugf(`Handler::AddRateLimitedEvent: %#v`, event) - h.eventQueue.AddRateLimited(event) -} - -// Run starts the event handler, spins up Goroutines to process events, and waits for a stop signal -func (h *Handler) Run(stopCh <-chan struct{}) { - logrus.Debug("Handler::Run") - - for i := 0; i < h.settings.Handler.Threads; i++ { - go wait.Until(h.worker, 0, stopCh) - } - - <-stopCh -} - -// ShutDown stops the event handler and shuts down the event queue -func (h *Handler) ShutDown() { - logrus.Debug("Handler::ShutDown") - h.eventQueue.ShutDown() -} - -// handleEvent feeds translated events to the synchronizer -func (h *Handler) handleEvent(e *core.Event) error { - logrus.Debugf(`Handler::handleEvent: %#v`, e) - // TODO: Add Telemetry - - events, err := translation.Translate(e) - if err != nil { - return fmt.Errorf(`Handler::handleEvent error translating: %v`, err) - } - - h.synchronizer.AddEvents(events) - - return nil -} - -// handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (h *Handler) handleNextEvent() bool { - logrus.Debug("Handler::handleNextEvent") - evt, quit := h.eventQueue.Get() - logrus.Debugf(`Handler::handleNextEvent: %#v, quit: %v`, evt, quit) - if quit { - return false - } - - defer h.eventQueue.Done(evt) - - event := evt.(*core.Event) - h.withRetry(h.handleEvent(event), event) - - return true -} - -// worker is the main message loop -func (h *Handler) worker() { - for h.handleNextEvent() { - // TODO: Add Telemetry - } -} - -// withRetry handles errors from the event handler and requeues events that fail -func (h *Handler) withRetry(err error, event *core.Event) { - logrus.Debug("Handler::withRetry") - if err != nil { - // TODO: Add Telemetry - if h.eventQueue.NumRequeues(event) < h.settings.Handler.RetryCount { - h.eventQueue.AddRateLimited(event) - logrus.Infof(`Handler::withRetry: requeued event: %#v; error: %v`, event, err) - } else { - h.eventQueue.Forget(event) - logrus.Warnf(`Handler::withRetry: event %#v has been dropped due to too many retries`, event) - } - } // TODO: Add error logging -} diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go deleted file mode 100644 index f4a617f8..00000000 --- a/internal/observation/handler_test.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package observation - -import ( - "context" - "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - v1 "k8s.io/api/core/v1" - "k8s.io/client-go/util/workqueue" - "testing" -) - -func TestHandler_AddsEventToSynchronizer(t *testing.T) { - _, _, synchronizer, handler, err := buildHandler() - if err != nil { - t.Errorf(`should have been no error, %v`, err) - } - - event := &core.Event{ - Type: core.Created, - Service: &v1.Service{ - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "nlk-back", - }, - }, - }, - }, - } - - handler.AddRateLimitedEvent(event) - - handler.handleNextEvent() - - if len(synchronizer.Events) != 1 { - t.Errorf(`handler.AddRateLimitedEvent did not add the event to the queue`) - } -} - -func buildHandler() (*configuration.Settings, workqueue.RateLimitingInterface, *mocks.MockSynchronizer, *Handler, error) { - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - return nil, nil, nil, nil, fmt.Errorf(`should have been no error, %v`, err) - } - - eventQueue := &mocks.MockRateLimiter{} - synchronizer := &mocks.MockSynchronizer{} - - handler := NewHandler(settings, synchronizer, eventQueue) - - return settings, eventQueue, synchronizer, handler, nil -} diff --git a/internal/observation/register.go b/internal/observation/register.go new file mode 100644 index 00000000..bfe61f80 --- /dev/null +++ b/internal/observation/register.go @@ -0,0 +1,63 @@ +package observation + +import ( + "sync" + + v1 "k8s.io/api/core/v1" +) + +// register holds references to the services that the user has configured for use with NLK +type register struct { + mu sync.RWMutex // protects register + services map[registerKey]*v1.Service +} + +type registerKey struct { + serviceName string + namespace string +} + +func newRegister() *register { + return ®ister{ + services: make(map[registerKey]*v1.Service), + } +} + +// addOrUpdateService adds the service to the register if not found, else updates the existing service +func (r *register) addOrUpdateService(service *v1.Service) { + r.mu.Lock() + defer r.mu.Unlock() + + r.services[registerKey{namespace: service.Namespace, serviceName: service.Name}] = service +} + +// removeService removes the service from the register +func (r *register) removeService(service *v1.Service) { + r.mu.Lock() + defer r.mu.Unlock() + + delete(r.services, registerKey{namespace: service.Namespace, serviceName: service.Name}) +} + +// getService returns the service from the register if found +func (r *register) getService(namespace string, serviceName string) (*v1.Service, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + s, ok := r.services[registerKey{namespace: namespace, serviceName: serviceName}] + return s, ok +} + +// listServices returns all the services in the register +func (r *register) listServices() []*v1.Service { + r.mu.RLock() + defer r.mu.RUnlock() + + services := make([]*v1.Service, 0, len(r.services)) + + for _, service := range r.services { + services = append(services, service) + } + + return services +} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 3ee9d3fe..9711347c 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -6,199 +6,304 @@ package observation import ( - "errors" + "context" "fmt" + "log/slog" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + discovery "k8s.io/api/discovery/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" + discoveryinformers "k8s.io/client-go/informers/discovery/v1" "k8s.io/client-go/tools/cache" - "time" ) // Watcher is responsible for watching for changes to Kubernetes resources. // Particularly, Services in the namespace defined in the WatcherSettings::NginxIngressNamespace setting. // When a change is detected, an Event is generated and added to the Handler's queue. type Watcher struct { + synchronizer synchronization.Interface - // eventHandlerRegistration is used to track the event handlers - eventHandlerRegistration interface{} + // settings is the configuration settings + settings configuration.Settings - // handler is the event handler - handler HandlerInterface + // servicesInformer is the informer used to watch for changes to services + servicesInformer cache.SharedIndexInformer - // informer is the informer used to watch for changes to Kubernetes resources - informer cache.SharedIndexInformer + // endpointSliceInformer is the informer used to watch for changes to endpoint slices + endpointSliceInformer cache.SharedIndexInformer - // settings is the configuration settings - settings *configuration.Settings + // nodesInformer is the informer used to watch for changes to nodes + nodesInformer cache.SharedIndexInformer + + register *register } // NewWatcher creates a new Watcher -func NewWatcher(settings *configuration.Settings, handler HandlerInterface) (*Watcher, error) { - return &Watcher{ - handler: handler, - settings: settings, - }, nil -} +func NewWatcher( + settings configuration.Settings, + synchronizer synchronization.Interface, + serviceInformer coreinformers.ServiceInformer, + endpointSliceInformer discoveryinformers.EndpointSliceInformer, + nodeInformer coreinformers.NodeInformer, +) (*Watcher, error) { + if serviceInformer == nil { + return nil, fmt.Errorf("service informer cannot be nil") + } -// Initialize initializes the Watcher, must be called before Watch -func (w *Watcher) Initialize() error { - logrus.Debug("Watcher::Initialize") - var err error + if endpointSliceInformer == nil { + return nil, fmt.Errorf("endpoint slice informer cannot be nil") + } - w.informer, err = w.buildInformer() - if err != nil { - return fmt.Errorf(`initialization error: %w`, err) + if nodeInformer == nil { + return nil, fmt.Errorf("node informer cannot be nil") } - err = w.initializeEventListeners() - if err != nil { - return fmt.Errorf(`initialization error: %w`, err) + servicesInformer := serviceInformer.Informer() + endpointSlicesInformer := endpointSliceInformer.Informer() + nodesInformer := nodeInformer.Informer() + + w := &Watcher{ + synchronizer: synchronizer, + settings: settings, + servicesInformer: servicesInformer, + endpointSliceInformer: endpointSlicesInformer, + nodesInformer: nodesInformer, + register: newRegister(), } - return nil + if err := w.initializeEventListeners(servicesInformer); err != nil { + return nil, err + } + + return w, nil } -// Watch starts the process of watching for changes to Kubernetes resources. +// Run starts the process of watching for changes to Kubernetes resources. // Initialize must be called before Watch. -func (w *Watcher) Watch() error { - logrus.Debug("Watcher::Watch") - - if w.informer == nil { - return errors.New("error: Initialize must be called before Watch") +func (w *Watcher) Run(ctx context.Context) error { + if w.servicesInformer == nil { + return fmt.Errorf(`servicesInformer is nil`) } + slog.Debug("Watcher::Watch") + defer utilruntime.HandleCrash() - defer w.handler.ShutDown() + defer w.synchronizer.ShutDown() - go w.informer.Run(w.settings.Context.Done()) + <-ctx.Done() + return nil +} - if !cache.WaitForNamedCacheSync(w.settings.Handler.WorkQueueSettings.Name, w.settings.Context.Done(), w.informer.HasSynced) { - return fmt.Errorf(`error occurred waiting for the cache to sync`) +// isDesiredService returns whether the user has configured the given service for watching. +func (w *Watcher) isDesiredService(service *v1.Service) bool { + annotation, ok := service.Annotations["nginx.com/nginxaas"] + if !ok { + return false } - <-w.settings.Context.Done() - return nil + return annotation == w.settings.Watcher.ServiceAnnotation } -// buildEventHandlerForAdd creates a function that is used as an event handler for the informer when Add events are raised. -func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { - logrus.Info("Watcher::buildEventHandlerForAdd") +func (w *Watcher) buildNodesEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForAdd") return func(obj interface{}) { - nodeIps, err := w.retrieveNodeIps() - if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) - return + slog.Debug("received node add event") + for _, service := range w.register.listServices() { + e := core.NewEvent(core.Updated, service) + w.synchronizer.AddEvent(e) + } + } +} + +func (w *Watcher) buildNodesEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForUpdate") + return func(previous, updated interface{}) { + slog.Debug("received node update event") + for _, service := range w.register.listServices() { + e := core.NewEvent(core.Updated, service) + w.synchronizer.AddEvent(e) + } + } +} + +func (w *Watcher) buildNodesEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForDelete") + return func(obj interface{}) { + slog.Debug("received node delete event") + for _, service := range w.register.listServices() { + e := core.NewEvent(core.Updated, service) + w.synchronizer.AddEvent(e) } - service := obj.(*v1.Service) - var previousService *v1.Service - e := core.NewEvent(core.Created, service, previousService, nodeIps) - w.handler.AddRateLimitedEvent(&e) } } -// buildEventHandlerForDelete creates a function that is used as an event handler for the informer when Delete events are raised. -func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { - logrus.Info("Watcher::buildEventHandlerForDelete") +func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForAdd") return func(obj interface{}) { - nodeIps, err := w.retrieveNodeIps() - if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Debug("received endpoint slice add event") + endpointSlice, ok := obj.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", obj) return } - service := obj.(*v1.Service) - var previousService *v1.Service - e := core.NewEvent(core.Deleted, service, previousService, nodeIps) - w.handler.AddRateLimitedEvent(&e) + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + e := core.NewEvent(core.Updated, service) + w.synchronizer.AddEvent(e) } } -// buildEventHandlerForUpdate creates a function that is used as an event handler for the informer when Update events are raised. -func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { - logrus.Info("Watcher::buildEventHandlerForUpdate") +func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForUpdate") return func(previous, updated interface{}) { - nodeIps, err := w.retrieveNodeIps() - if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Debug("received endpoint slice update event") + endpointSlice, ok := updated.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", updated) return } - service := updated.(*v1.Service) - previousService := previous.(*v1.Service) - e := core.NewEvent(core.Updated, service, previousService, nodeIps) - w.handler.AddRateLimitedEvent(&e) + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + e := core.NewEvent(core.Updated, service) + w.synchronizer.AddEvent(e) } } -// buildInformer creates the informer used to watch for changes to Kubernetes resources. -func (w *Watcher) buildInformer() (cache.SharedIndexInformer, error) { - logrus.Debug("Watcher::buildInformer") +func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForDelete") + return func(obj interface{}) { + slog.Debug("received endpoint slice delete event") + endpointSlice, ok := obj.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", obj) + return + } - options := informers.WithNamespace(w.settings.Watcher.NginxIngressNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, options) - informer := factory.Core().V1().Services().Informer() + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } - return informer, nil + e := core.NewEvent(core.Deleted, service) + w.synchronizer.AddEvent(e) + } } -// initializeEventListeners initializes the event listeners for the informer. -func (w *Watcher) initializeEventListeners() error { - logrus.Debug("Watcher::initializeEventListeners") - var err error +// buildServiceEventHandlerForAdd creates a function that is used as an event handler +// for the informer when Add events are raised. +func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForAdd") + return func(obj interface{}) { + service := obj.(*v1.Service) + if !w.isDesiredService(service) { + return + } - handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: w.buildEventHandlerForAdd(), - DeleteFunc: w.buildEventHandlerForDelete(), - UpdateFunc: w.buildEventHandlerForUpdate(), - } + w.register.addOrUpdateService(service) - w.eventHandlerRegistration, err = w.informer.AddEventHandler(handlers) - if err != nil { - return fmt.Errorf(`error occurred adding event handlers: %w`, err) + e := core.NewEvent(core.Created, service) + w.synchronizer.AddEvent(e) } - - return nil } -// notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is -// because the master node may or may not be a worker node and thus may not be able to route traffic. -func (w *Watcher) retrieveNodeIps() ([]string, error) { - started := time.Now() - logrus.Debug("Watcher::retrieveNodeIps") +// buildServiceEventHandlerForDelete creates a function that is used as an event handler +// for the informer when Delete events are raised. +func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForDelete") + return func(obj interface{}) { + service := obj.(*v1.Service) + if !w.isDesiredService(service) { + return + } - var nodeIps []string + w.register.removeService(service) - nodes, err := w.settings.K8sClient.CoreV1().Nodes().List(w.settings.Context, metav1.ListOptions{}) - if err != nil { - logrus.Errorf(`error occurred retrieving the list of nodes: %v`, err) - return nil, err + e := core.NewEvent(core.Deleted, service) + w.synchronizer.AddEvent(e) } +} - for _, node := range nodes.Items { +// buildServiceEventHandlerForUpdate creates a function that is used as an event handler +// for the informer when Update events are raised. +func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForUpdate") + return func(previous, updated interface{}) { + previousService := previous.(*v1.Service) + service := updated.(*v1.Service) - // this is kind of a broad assumption, should probably make this a configurable option - if w.notMasterNode(node) { - for _, address := range node.Status.Addresses { - if address.Type == v1.NodeInternalIP { - nodeIps = append(nodeIps, address.Address) - } - } + if w.isDesiredService(previousService) && !w.isDesiredService(service) { + slog.Info("Watcher::service annotation removed", "serviceName", service.Name) + w.register.removeService(previousService) + e := core.NewEvent(core.Deleted, previousService) + w.synchronizer.AddEvent(e) + return + } + + if !w.isDesiredService(service) { + return } - } - logrus.Debugf("Watcher::retrieveNodeIps duration: %d", time.Since(started).Nanoseconds()) + w.register.addOrUpdateService(service) - return nodeIps, nil + e := core.NewEvent(core.Updated, service) + w.synchronizer.AddEvent(e) + } } -// notMasterNode determines if the node is a master node. -func (w *Watcher) notMasterNode(node v1.Node) bool { - logrus.Debug("Watcher::notMasterNode") +// initializeEventListeners initializes the event listeners for the informer. +func (w *Watcher) initializeEventListeners( + servicesInformer cache.SharedIndexInformer, +) error { + slog.Debug("Watcher::initializeEventListeners") + var err error + + serviceHandlers := cache.ResourceEventHandlerFuncs{ + AddFunc: w.buildServiceEventHandlerForAdd(), + DeleteFunc: w.buildServiceEventHandlerForDelete(), + UpdateFunc: w.buildServiceEventHandlerForUpdate(), + } + + endpointSliceHandlers := cache.ResourceEventHandlerFuncs{ + AddFunc: w.buildEndpointSlicesEventHandlerForAdd(), + DeleteFunc: w.buildEndpointSlicesEventHandlerForDelete(), + UpdateFunc: w.buildEndpointSlicesEventHandlerForUpdate(), + } + + nodeHandlers := cache.ResourceEventHandlerFuncs{ + AddFunc: w.buildNodesEventHandlerForAdd(), + DeleteFunc: w.buildNodesEventHandlerForDelete(), + UpdateFunc: w.buildNodesEventHandlerForUpdate(), + } - _, found := node.Labels["node-role.kubernetes.io/master"] + _, err = servicesInformer.AddEventHandler(serviceHandlers) + if err != nil { + return fmt.Errorf(`error occurred adding service event handlers: %w`, err) + } - return !found + _, err = w.endpointSliceInformer.AddEventHandler(endpointSliceHandlers) + if err != nil { + return fmt.Errorf(`error occurred adding endpoint slice event handlers: %w`, err) + } + + _, err = w.nodesInformer.AddEventHandler(nodeHandlers) + if err != nil { + return fmt.Errorf(`error occurred adding node event handlers: %w`, err) + } + + return nil } diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 36d64ee2..b8e8369a 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -6,24 +6,18 @@ package observation import ( - "context" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "k8s.io/client-go/kubernetes" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "github.com/stretchr/testify/require" ) -func TestWatcher_MustInitialize(t *testing.T) { - watcher, _ := buildWatcher() - if err := watcher.Watch(); err == nil { - t.Errorf("Expected error, got %s", err) - } +func TestWatcher_ErrWithNilInformers(t *testing.T) { + t.Parallel() + _, err := buildWatcherWithNilInformer() + require.Error(t, err, "expected construction of watcher with nil informer to fail") } -func buildWatcher() (*Watcher, error) { - k8sClient := &kubernetes.Clientset{} - settings, _ := configuration.NewSettings(context.Background(), k8sClient) - handler := &mocks.MockHandler{} - - return NewWatcher(settings, handler) +func buildWatcherWithNilInformer() (*Watcher, error) { + return NewWatcher(configuration.Settings{}, nil, nil, nil, nil) } diff --git a/internal/probation/check_test.go b/internal/probation/check_test.go index 208c9a41..95358e58 100644 --- a/internal/probation/check_test.go +++ b/internal/probation/check_test.go @@ -8,6 +8,7 @@ package probation import "testing" func TestCheck_LiveCheck(t *testing.T) { + t.Parallel() check := LiveCheck{} if !check.Check() { t.Errorf("LiveCheck should return true") @@ -15,6 +16,7 @@ func TestCheck_LiveCheck(t *testing.T) { } func TestCheck_ReadyCheck(t *testing.T) { + t.Parallel() check := ReadyCheck{} if !check.Check() { t.Errorf("ReadyCheck should return true") @@ -22,6 +24,7 @@ func TestCheck_ReadyCheck(t *testing.T) { } func TestCheck_StartupCheck(t *testing.T) { + t.Parallel() check := StartupCheck{} if !check.Check() { t.Errorf("StartupCheck should return true") diff --git a/internal/probation/server.go b/internal/probation/server.go index 12b16993..c3328c70 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -7,8 +7,10 @@ package probation import ( "fmt" - "github.com/sirupsen/logrus" + "log/slog" + "net" "net/http" + "time" ) const ( @@ -25,7 +27,6 @@ const ( // HealthServer is a server that spins up endpoints for the various k8s health checks. type HealthServer struct { - // The underlying HTTP server. httpServer *http.Server @@ -50,7 +51,7 @@ func NewHealthServer() *HealthServer { // Start spins up the health server. func (hs *HealthServer) Start() { - logrus.Debugf("Starting probe listener on port %d", ListenPort) + slog.Debug("Starting probe listener", "port", ListenPort) address := fmt.Sprintf(":%d", ListenPort) @@ -58,21 +59,28 @@ func (hs *HealthServer) Start() { mux.HandleFunc("/livez", hs.HandleLive) mux.HandleFunc("/readyz", hs.HandleReady) mux.HandleFunc("/startupz", hs.HandleStartup) - hs.httpServer = &http.Server{Addr: address, Handler: mux} + + listener, err := net.Listen("tcp", address) + if err != nil { + slog.Error("failed to listen", "error", err) + return + } + + hs.httpServer = &http.Server{Handler: mux, ReadTimeout: 2 * time.Second} go func() { - if err := hs.httpServer.ListenAndServe(); err != nil { - logrus.Errorf("unable to start probe listener on %s: %v", hs.httpServer.Addr, err) + if err := hs.httpServer.Serve(listener); err != nil { + slog.Error("unable to start probe listener", "address", hs.httpServer.Addr, "error", err) } }() - logrus.Info("Started probe listener on", hs.httpServer.Addr) + slog.Info("Started probe listener", "address", hs.httpServer.Addr) } // Stop shuts down the health server. func (hs *HealthServer) Stop() { if err := hs.httpServer.Close(); err != nil { - logrus.Errorf("unable to stop probe listener on %s: %v", hs.httpServer.Addr, err) + slog.Error("unable to stop probe listener", "address", hs.httpServer.Addr, "error", err) } } @@ -97,14 +105,14 @@ func (hs *HealthServer) handleProbe(writer http.ResponseWriter, _ *http.Request, writer.WriteHeader(http.StatusOK) if _, err := fmt.Fprint(writer, Ok); err != nil { - logrus.Error(err) + slog.Error(err.Error()) } } else { writer.WriteHeader(http.StatusServiceUnavailable) if _, err := fmt.Fprint(writer, ServiceNotAvailable); err != nil { - logrus.Error(err) + slog.Error(err.Error()) } } } diff --git a/internal/probation/server_test.go b/internal/probation/server_test.go index f594bffe..981aa3b7 100644 --- a/internal/probation/server_test.go +++ b/internal/probation/server_test.go @@ -6,13 +6,15 @@ package probation import ( - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "github.com/sirupsen/logrus" + "log/slog" "net/http" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" ) func TestHealthServer_HandleLive(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleLive(writer, nil) @@ -23,6 +25,7 @@ func TestHealthServer_HandleLive(t *testing.T) { } func TestHealthServer_HandleReady(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleReady(writer, nil) @@ -33,6 +36,7 @@ func TestHealthServer_HandleReady(t *testing.T) { } func TestHealthServer_HandleStartup(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleStartup(writer, nil) @@ -43,6 +47,7 @@ func TestHealthServer_HandleStartup(t *testing.T) { } func TestHealthServer_HandleFailCheck(t *testing.T) { + t.Parallel() failCheck := mocks.NewMockCheck(false) server := NewHealthServer() writer := mocks.NewMockResponseWriter() @@ -55,6 +60,7 @@ func TestHealthServer_HandleFailCheck(t *testing.T) { } func TestHealthServer_Start(t *testing.T) { + t.Parallel() server := NewHealthServer() server.Start() @@ -64,10 +70,11 @@ func TestHealthServer_Start(t *testing.T) { if err != nil { t.Error(err) } + defer response.Body.Close() if response.StatusCode != http.StatusOK { t.Errorf("Expected status code %v, got %v", http.StatusAccepted, response.StatusCode) } - logrus.Infof("received a response from the probe server: %v", response) + slog.Info("received a response from the probe server", "response", response) } diff --git a/internal/synchronization/cache.go b/internal/synchronization/cache.go new file mode 100644 index 00000000..14effb99 --- /dev/null +++ b/internal/synchronization/cache.go @@ -0,0 +1,49 @@ +package synchronization + +import ( + "sync" + "time" + + v1 "k8s.io/api/core/v1" +) + +// cache contains the most recent definitions for services monitored by NLK. +// We need these so that if a service is deleted from the shared informer cache, the +// caller can access the spec of the deleted service for cleanup. +type cache struct { + mu sync.RWMutex + store map[ServiceKey]service +} + +type service struct { + service *v1.Service + // removedAt indicates when the service was removed from NGINXaaS + // monitoring. A zero time indicates that the service is still actively + // being monitored by NGINXaaS. + removedAt time.Time +} + +func newCache() *cache { + return &cache{ + store: make(map[ServiceKey]service), + } +} + +func (s *cache) get(key ServiceKey) (service, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + svc, ok := s.store[key] + return svc, ok +} + +func (s *cache) add(key ServiceKey, service service) { + s.mu.Lock() + defer s.mu.Unlock() + s.store[key] = service +} + +func (s *cache) delete(key ServiceKey) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.store, key) +} diff --git a/internal/synchronization/rand.go b/internal/synchronization/rand.go deleted file mode 100644 index 425b99ad..00000000 --- a/internal/synchronization/rand.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package synchronization - -import ( - "math/rand" - "time" -) - -// charset contains all characters that can be used in random string generation -var charset = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -// number contains all numbers that can be used in random string generation -var number = []byte("0123456789") - -// alphaNumeric contains all characters and numbers that can be used in random string generation -var alphaNumeric = append(charset, number...) - -// RandomString where n is the length of random string we want to generate -func RandomString(n int) string { - b := make([]byte, n) - for i := range b { - // randomly select 1 character from given charset - b[i] = alphaNumeric[rand.Intn(len(alphaNumeric))] - } - return string(b) -} - -// RandomMilliseconds returns a random duration between min and max milliseconds -func RandomMilliseconds(min, max int) time.Duration { - randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) - random := randomizer.Intn(max-min) + min - - return time.Millisecond * time.Duration(random) -} diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 1061b014..8f0fff60 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -6,96 +6,123 @@ package synchronization import ( + "context" + "errors" "fmt" + "log/slog" + "net/http" + "time" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" - "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" + corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/util/workqueue" ) // Interface defines the interface needed to implement a synchronizer. type Interface interface { - - // AddEvents adds a list of events to the queue. - AddEvents(events core.ServerUpdateEvents) - // AddEvent adds an event to the queue. - AddEvent(event *core.ServerUpdateEvent) + AddEvent(event core.Event) // Run starts the synchronizer. - Run(stopCh <-chan struct{}) + Run(ctx context.Context) error // ShutDown shuts down the synchronizer. ShutDown() } +// StatusError is a wrapper for errors from the go plus client that contain http +// status codes. +type StatusError interface { + Status() int + Code() string +} + +type Translator interface { + Translate(*core.Event) (core.ServerUpdateEvents, error) +} + +type ServiceKey struct { + Name string + Namespace string +} + // Synchronizer is responsible for synchronizing the state of the Border Servers. -// Operating against the "nlk-synchronizer", it handles events by creating a Border Client as specified in the -// Service annotation for the Upstream. see application/border_client.go and application/application_constants.go for details. +// Operating against the "nlk-synchronizer", it handles events by creating +// a Border Client as specified in the Service annotation for the Upstream. +// See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { - eventQueue workqueue.RateLimitingInterface - settings *configuration.Settings + eventQueue workqueue.TypedRateLimitingInterface[ServiceKey] + settings configuration.Settings + translator Translator + cache *cache + serviceLister corelisters.ServiceLister } // NewSynchronizer creates a new Synchronizer. -func NewSynchronizer(settings *configuration.Settings, eventQueue workqueue.RateLimitingInterface) (*Synchronizer, error) { +func NewSynchronizer( + settings configuration.Settings, + eventQueue workqueue.TypedRateLimitingInterface[ServiceKey], + translator Translator, + serviceLister corelisters.ServiceLister, +) (*Synchronizer, error) { synchronizer := Synchronizer{ - eventQueue: eventQueue, - settings: settings, + eventQueue: eventQueue, + settings: settings, + cache: newCache(), + translator: translator, + serviceLister: serviceLister, } return &synchronizer, nil } -// AddEvents adds a list of events to the queue. If no hosts are specified this is a null operation. -// Events will fan out to the number of hosts specified before being added to the queue. -func (s *Synchronizer) AddEvents(events core.ServerUpdateEvents) { - logrus.Debugf(`Synchronizer::AddEvents adding %d events`, len(events)) +// AddEvent adds an event to the rate-limited queue. If no hosts are specified this is a null operation. +func (s *Synchronizer) AddEvent(event core.Event) { + slog.Debug(`Synchronizer::AddEvent`) if len(s.settings.NginxPlusHosts) == 0 { - logrus.Warnf(`No Nginx Plus hosts were specified. Skipping synchronization.`) + slog.Warn(`No Nginx Plus hosts were specified. Skipping synchronization.`) return } - updatedEvents := s.fanOutEventToHosts(events) - - for _, event := range updatedEvents { - s.AddEvent(event) + key := ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace} + var deletedAt time.Time + if event.Type == core.Deleted { + deletedAt = time.Now() } -} -// AddEvent adds an event to the queue. If no hosts are specified this is a null operation. -// Events will be added to the queue after a random delay between MinMillisecondsJitter and MaxMillisecondsJitter. -func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { - logrus.Debugf(`Synchronizer::AddEvent: %#v`, event) - - if event.NginxHost == `` { - logrus.Warnf(`Nginx host was not specified. Skipping synchronization.`) - return - } - - after := RandomMilliseconds(s.settings.Synchronizer.MinMillisecondsJitter, s.settings.Synchronizer.MaxMillisecondsJitter) - s.eventQueue.AddAfter(event, after) + s.cache.add(key, service{event.Service, deletedAt}) + s.eventQueue.AddRateLimited(key) } // Run starts the Synchronizer, spins up Goroutines to process events, and waits for a stop signal. -func (s *Synchronizer) Run(stopCh <-chan struct{}) { - logrus.Debug(`Synchronizer::Run`) +func (s *Synchronizer) Run(ctx context.Context) error { + slog.Debug(`Synchronizer::Run`) + + // worker is the main message loop + worker := func() { + slog.Debug(`Synchronizer::worker`) + for s.handleNextServiceEvent(ctx) { + } + } for i := 0; i < s.settings.Synchronizer.Threads; i++ { - go wait.Until(s.worker, 0, stopCh) + go wait.Until(worker, 0, ctx.Done()) } - <-stopCh + <-ctx.Done() + return nil } // ShutDown stops the Synchronizer and shuts down the event queue func (s *Synchronizer) ShutDown() { - logrus.Debugf(`Synchronizer::ShutDown`) + slog.Debug(`Synchronizer::ShutDown`) s.eventQueue.ShutDownWithDrain() } @@ -103,16 +130,15 @@ func (s *Synchronizer) ShutDown() { // NOTE: There is an open issue (https://github.com/nginxinc/nginx-loadbalancer-kubernetes/issues/36) to move creation // of the underlying Border Server client to the NewBorderClient function. func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (application.Interface, error) { - logrus.Debugf(`Synchronizer::buildBorderClient`) + slog.Debug(`Synchronizer::buildBorderClient`) var err error - - httpClient, err := communication.NewHttpClient(s.settings) + httpClient, err := communication.NewHTTPClient(s.settings.APIKey, s.settings.SkipVerifyTLS) if err != nil { return nil, fmt.Errorf(`error creating HTTP client: %v`, err) } - ngxClient, err := nginxClient.NewNginxClient(httpClient, event.NginxHost) + ngxClient, err := nginxClient.NewNginxClient(event.NginxHost, nginxClient.WithHTTPClient(httpClient)) if err != nil { return nil, fmt.Errorf(`error creating Nginx Plus client: %v`, err) } @@ -122,14 +148,13 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica // fanOutEventToHosts takes a list of events and returns a list of events, one for each Border Server. func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.ServerUpdateEvents { - logrus.Debugf(`Synchronizer::fanOutEventToHosts: %#v`, event) + slog.Debug(`Synchronizer::fanOutEventToHosts`) var events core.ServerUpdateEvents - for hidx, host := range s.settings.NginxPlusHosts { - for eidx, event := range event { - id := fmt.Sprintf(`[%d:%d]-[%s]-[%s]-[%s]`, hidx, eidx, RandomString(12), event.UpstreamName, host) - updatedEvent := core.ServerUpdateEventWithIdAndHost(event, id, host) + for _, host := range s.settings.NginxPlusHosts { + for _, event := range event { + updatedEvent := core.ServerUpdateEventWithHost(event, host) events = append(events, updatedEvent) } @@ -138,36 +163,86 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se return events } -// handleEvent dispatches an event to the proper handler function. -func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleEvent: Id: %s`, event.Id) - - var err error +// handleServiceEvent gets the latest state for the service from the shared +// informer cache, translates the service event into server update events and +// dispatches these events to the proper handler function. +func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) (err error) { + logger := slog.With("service", key) + logger.Debug(`Synchronizer::handleServiceEvent`) + + // if a service exists in the shared informer cache, we can assume that we need to update it + event := core.Event{Type: core.Updated} + + cachedService, exists := s.cache.get(key) + + namespaceLister := s.serviceLister.Services(key.Namespace) + k8sService, err := namespaceLister.Get(key.Name) + switch { + // the service has been deleted. We need to rely on the local cache to + // gather the last known state of the service so we can delete its + // upstream servers + case err != nil && apierrors.IsNotFound(err): + if !exists { + logger.Warn(`Synchronizer::handleServiceEvent: no information could be gained about service`) + return nil + } + // no matter what type the cached event has, the service no longer exists, so the type is Deleted + event.Type = core.Deleted + event.Service = cachedService.service + case err != nil: + return err + case exists && !cachedService.removedAt.IsZero(): + event.Type = core.Deleted + event.Service = cachedService.service + default: + event.Service = k8sService + } - switch event.Type { - case core.Created: - fallthrough + events, err := s.translator.Translate(&event) + if err != nil { + return err + } - case core.Updated: - err = s.handleCreatedUpdatedEvent(event) + if len(events) == 0 { + slog.Warn("Synchronizer::handleServiceEvent: no events to process") + return nil + } - case core.Deleted: - err = s.handleDeletedEvent(event) + events = s.fanOutEventToHosts(events) + + for _, evt := range events { + switch event.Type { + case core.Created, core.Updated: + if handleErr := s.handleCreatedUpdatedEvent(ctx, evt); handleErr != nil { + err = errors.Join(err, handleErr) + } + case core.Deleted: + if handleErr := s.handleDeletedEvent(ctx, evt); handleErr != nil { + err = errors.Join(err, handleErr) + } + default: + slog.Warn(`Synchronizer::handleServiceEvent: unknown event type`, "type", event.Type) + } + } - default: - logrus.Warnf(`Synchronizer::handleEvent: unknown event type: %d`, event.Type) + if err != nil { + return err } - if err == nil { - logrus.Infof(`Synchronizer::handleEvent: successfully %s the nginx+ host(s) for Upstream: %s: Id(%s)`, event.TypeName(), event.UpstreamName, event.Id) + if event.Type == core.Deleted { + s.cache.delete(ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace}) } - return err + slog.Debug( + "Synchronizer::handleServiceEvent: successfully handled the service change", "service", key, + ) + + return nil } // handleCreatedUpdatedEvent handles events of type Created or Updated. -func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleCreatedUpdatedEvent: Id: %s`, serverUpdateEvent.Id) +func (s *Synchronizer) handleCreatedUpdatedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { + slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`) var err error @@ -176,7 +251,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU return fmt.Errorf(`error occurred creating the border client: %w`, err) } - if err = borderClient.Update(serverUpdateEvent); err != nil { + if err = borderClient.Update(ctx, serverUpdateEvent); err != nil { return fmt.Errorf(`error occurred updating the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) } @@ -184,8 +259,8 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU } // handleDeletedEvent handles events of type Deleted. -func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleDeletedEvent: Id: %s`, serverUpdateEvent.Id) +func (s *Synchronizer) handleDeletedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { + slog.Debug(`Synchronizer::handleDeletedEvent`) var err error @@ -194,50 +269,46 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv return fmt.Errorf(`error occurred creating the border client: %w`, err) } - if err = borderClient.Delete(serverUpdateEvent); err != nil { + err = borderClient.Update(ctx, serverUpdateEvent) + + var se StatusError + switch { + case err == nil: + return nil + case errors.As(err, &se) && se.Status() == http.StatusNotFound: + // if the user has already removed the upstream from their NGINX + // configuration there is nothing left to do + return nil + default: return fmt.Errorf(`error occurred deleting the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) } - - return nil } -// handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (s *Synchronizer) handleNextEvent() bool { - logrus.Debug(`Synchronizer::handleNextEvent`) +// handleNextServiceEvent pulls a service from the event queue and feeds it to +// the service event handler with retry logic +func (s *Synchronizer) handleNextServiceEvent(ctx context.Context) bool { + slog.Debug(`Synchronizer::handleNextServiceEvent`) - evt, quit := s.eventQueue.Get() + svc, quit := s.eventQueue.Get() if quit { return false } - defer s.eventQueue.Done(evt) + defer s.eventQueue.Done(svc) - event := evt.(*core.ServerUpdateEvent) - s.withRetry(s.handleEvent(event), event) + s.withRetry(s.handleServiceEvent(ctx, svc), svc) return true } -// worker is the main message loop -func (s *Synchronizer) worker() { - logrus.Debug(`Synchronizer::worker`) - for s.handleNextEvent() { - } -} - // withRetry handles errors from the event handler and requeues events that fail -func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { - logrus.Debug("Synchronizer::withRetry") +func (s *Synchronizer) withRetry(err error, key ServiceKey) { + slog.Debug("Synchronizer::withRetry") if err != nil { // TODO: Add Telemetry - if s.eventQueue.NumRequeues(event) < s.settings.Synchronizer.RetryCount { // TODO: Make this configurable - s.eventQueue.AddRateLimited(event) - logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.Id, err) - } else { - s.eventQueue.Forget(event) - logrus.Warnf(`Synchronizer::withRetry: event %#v has been dropped due to too many retries`, event) - } + s.eventQueue.AddRateLimited(key) + slog.Info(`Synchronizer::withRetry: requeued service update`, "service", key, "error", err) } else { - s.eventQueue.Forget(event) + s.eventQueue.Forget(key) } // TODO: Add error logging } diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index 315def73..ba2253b2 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -6,20 +6,30 @@ package synchronization import ( - "context" "fmt" + "testing" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "testing" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" ) func TestSynchronizer_NewSynchronizer(t *testing.T) { - settings, err := configuration.NewSettings(context.Background(), nil) + t.Parallel() - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer( + configuration.Settings{}, + rateLimiter, + &fakeTranslator{}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -30,18 +40,17 @@ func TestSynchronizer_NewSynchronizer(t *testing.T) { } func TestSynchronizer_AddEventNoHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 - event := &core.ServerUpdateEvent{ - Id: "", - NginxHost: "", - Type: 0, - UpstreamName: "", - UpstreamServers: nil, - } - settings, err := configuration.NewSettings(context.Background(), nil) - rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} + + synchronizer, err := NewSynchronizer( + defaultSettings(), + rateLimiter, + &fakeTranslator{}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -52,7 +61,7 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { // NOTE: Ideally we have a custom logger that can be mocked to capture the log message // and assert a warning was logged that the NGINX Plus host was not specified. - synchronizer.AddEvent(event) + synchronizer.AddEvent(core.Event{}) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -60,13 +69,18 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { } func TestSynchronizer_AddEventOneHost(t *testing.T) { + t.Parallel() const expectedEventCount = 1 - events := buildEvents(1) - settings, err := configuration.NewSettings(context.Background(), nil) - settings.NginxPlusHosts = []string{"https://localhost:8080"} - rateLimiter := &mocks.MockRateLimiter{} + events := buildServerUpdateEvents(1) + + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings("https://localhost:8080"), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -75,7 +89,7 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvent(events[0]) + synchronizer.AddEvent(buildServiceUpdateEvent(1)) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -83,17 +97,23 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { } func TestSynchronizer_AddEventManyHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 1 - events := buildEvents(1) - settings, err := configuration.NewSettings(context.Background(), nil) - settings.NginxPlusHosts = []string{ + events := buildServerUpdateEvents(1) + hosts := []string{ "https://localhost:8080", "https://localhost:8081", "https://localhost:8082", } - rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} + + synchronizer, err := NewSynchronizer( + defaultSettings(hosts...), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -102,7 +122,7 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvent(events[0]) + synchronizer.AddEvent(buildServiceUpdateEvent(1)) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -110,12 +130,17 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { } func TestSynchronizer_AddEventsNoHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 - events := buildEvents(4) - settings, err := configuration.NewSettings(context.Background(), nil) - rateLimiter := &mocks.MockRateLimiter{} + events := buildServerUpdateEvents(4) + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -126,7 +151,10 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { // NOTE: Ideally we have a custom logger that can be mocked to capture the log message // and assert a warning was logged that the NGINX Plus host was not specified. - synchronizer.AddEvents(events) + for i := 0; i < 4; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -134,13 +162,17 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { } func TestSynchronizer_AddEventsOneHost(t *testing.T) { + t.Parallel() const expectedEventCount = 4 - events := buildEvents(4) - settings, err := configuration.NewSettings(context.Background(), nil) - settings.NginxPlusHosts = []string{"https://localhost:8080"} - rateLimiter := &mocks.MockRateLimiter{} + events := buildServerUpdateEvents(1) + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings("https://localhost:8080"), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -149,7 +181,10 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvents(events) + for i := 0; i < 4; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -157,18 +192,25 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { } func TestSynchronizer_AddEventsManyHosts(t *testing.T) { + t.Parallel() const eventCount = 4 - events := buildEvents(eventCount) - rateLimiter := &mocks.MockRateLimiter{} - settings, err := configuration.NewSettings(context.Background(), nil) - settings.NginxPlusHosts = []string{ + events := buildServerUpdateEvents(eventCount) + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} + + hosts := []string{ "https://localhost:8080", "https://localhost:8081", "https://localhost:8082", } - expectedEventCount := eventCount * len(settings.NginxPlusHosts) - synchronizer, err := NewSynchronizer(settings, rateLimiter) + expectedEventCount := 4 + + synchronizer, err := NewSynchronizer( + defaultSettings(hosts...), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -177,18 +219,31 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvents(events) + for i := 0; i < eventCount; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) } } -func buildEvents(count int) core.ServerUpdateEvents { +func buildServiceUpdateEvent(serviceID int) core.Event { + return core.Event{ + Service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-service%d", serviceID), + Namespace: "test-namespace", + }, + }, + } +} + +func buildServerUpdateEvents(count int) core.ServerUpdateEvents { events := make(core.ServerUpdateEvents, count) for i := 0; i < count; i++ { events[i] = &core.ServerUpdateEvent{ - Id: fmt.Sprintf("id-%v", i), NginxHost: "https://localhost:8080", Type: 0, UpstreamName: "", @@ -197,3 +252,70 @@ func buildEvents(count int) core.ServerUpdateEvents { } return events } + +func defaultSettings(nginxHosts ...string) configuration.Settings { + return configuration.Settings{ + NginxPlusHosts: nginxHosts, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + } +} + +type fakeTranslator struct { + events core.ServerUpdateEvents + err error +} + +func (t *fakeTranslator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { + return t.events, t.err +} + +func newFakeServicesLister(list ...*v1.Service) corelisters.ServiceLister { + return &servicesLister{ + list: list, + } +} + +type servicesLister struct { + list []*v1.Service + err error +} + +func (l *servicesLister) List(selector labels.Selector) (ret []*v1.Service, err error) { + return l.list, l.err +} + +func (l *servicesLister) Get(name string) (*v1.Service, error) { + for _, service := range l.list { + if service.Name == name { + return service, nil + } + } + + return nil, nil +} + +func (l *servicesLister) Services(name string) corelisters.ServiceNamespaceLister { + return l +} + +func defaultService() *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-service", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + }, + } +} diff --git a/internal/translation/translator.go b/internal/translation/translator.go index b2d0e87c..8491a668 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -7,64 +7,201 @@ package translation import ( "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "log/slog" + "strings" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" - "strings" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" ) +type Translator struct { + endpointSliceLister discoverylisters.EndpointSliceLister + nodeLister corelisters.NodeLister +} + +func NewTranslator( + endpointSliceLister discoverylisters.EndpointSliceLister, + nodeLister corelisters.NodeLister, +) *Translator { + return &Translator{ + endpointSliceLister: endpointSliceLister, + nodeLister: nodeLister, + } +} + // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations // and used to update the Border Servers. -func Translate(event *core.Event) (core.ServerUpdateEvents, error) { - logrus.Debug("Translate::Translate") +func (t *Translator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { + slog.Debug("Translate::Translate") + + return t.buildServerUpdateEvents(event.Service.Spec.Ports, event) +} - portsOfInterest := filterPorts(event.Service.Spec.Ports) +// buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type +// The NGINX+ Client uses a list of servers for Created and Updated events. +// The client performs reconciliation between the list of servers in the NGINX+ Client call +// and the list of servers in NGINX+. +// The NGINX+ Client uses a single server for Deleted events; +// so the list of servers is broken up into individual events. +func (t *Translator) buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event, +) (events core.ServerUpdateEvents, err error) { + slog.Debug("Translate::buildServerUpdateEvents", "ports", ports) - return buildServerUpdateEvents(portsOfInterest, event) + switch event.Service.Spec.Type { + case v1.ServiceTypeNodePort: + return t.buildNodeIPEvents(ports, event) + case v1.ServiceTypeClusterIP: + return t.buildClusterIPEvents(event) + case v1.ServiceTypeLoadBalancer: + return t.buildLoadBalancerEvents(event) + default: + return events, fmt.Errorf("unsupported service type: %s", event.Service.Spec.Type) + } } -// filterPorts returns a list of ports that have the NlkPrefix in the port name. -func filterPorts(ports []v1.ServicePort) []v1.ServicePort { - var portsOfInterest []v1.ServicePort +type upstream struct { + context string + name string +} - for _, port := range ports { - if strings.HasPrefix(port.Name, configuration.NlkPrefix) { - portsOfInterest = append(portsOfInterest, port) +func (t *Translator) buildLoadBalancerEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { + slog.Debug("Translator::buildLoadBalancerEvents", "ports", event.Service.Spec.Ports) + + addresses := make([]string, 0, len(event.Service.Status.LoadBalancer.Ingress)) + for _, ingress := range event.Service.Status.LoadBalancer.Ingress { + addresses = append(addresses, ingress.IP) + } + + for _, port := range event.Service.Spec.Ports { + context, upstreamName, err := getContextAndUpstreamName(port.Name) + if err != nil { + slog.Info("Translator::buildLoadBalancerEvents: ignoring port", "err", err, "name", port.Name) + continue + } + + upstreamServers := buildUpstreamServers(addresses, port.Port) + + switch event.Type { + case core.Created, core.Updated: + events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) + case core.Deleted: + events = append(events, core.NewServerUpdateEvent( + core.Updated, upstreamName, context, nil, + )) + default: + slog.Warn(`Translator::buildLoadBalancerEvents: unknown event type`, "type", event.Type) } } - return portsOfInterest + return events, nil } -// buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type -// The NGINX+ Client uses a list of servers for Created and Updated events; the client performs reconciliation between -// the list of servers in the NGINX+ Client call and the list of servers in NGINX+. -// The NGINX+ Client uses a single server for Deleted events; so the list of servers is broken up into individual events. -func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.ServerUpdateEvents, error) { - logrus.Debugf("Translate::buildServerUpdateEvents(ports=%#v)", ports) +func (t *Translator) buildClusterIPEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { + namespace := event.Service.GetObjectMeta().GetNamespace() + serviceName := event.Service.Name + + logger := slog.With("namespace", namespace, "serviceName", serviceName) + logger.Debug("Translate::buildClusterIPEvents") + + if event.Type == core.Deleted { + for _, port := range event.Service.Spec.Ports { + context, upstreamName, pErr := getContextAndUpstreamName(port.Name) + if pErr != nil { + logger.Info(pErr.Error()) + continue + } + events = append(events, core.NewServerUpdateEvent(core.Updated, upstreamName, context, nil)) + } + return events, nil + } + + lister := t.endpointSliceLister.EndpointSlices(namespace) + selector, err := labels.Parse(fmt.Sprintf("kubernetes.io/service-name=%s", serviceName)) + if err != nil { + logger.Error(`error occurred parsing the selector`, "error", err) + return events, err + } + + list, err := lister.List(selector) + if err != nil { + logger.Error(`error occurred retrieving the list of endpoint slices`, "error", err) + return events, err + } + + upstreams := make(map[upstream][]*core.UpstreamServer) + + for _, endpointSlice := range list { + for _, port := range endpointSlice.Ports { + if port.Name == nil || port.Port == nil { + continue + } + + context, upstreamName, err := getContextAndUpstreamName(*port.Name) + if err != nil { + logger.Info(err.Error()) + continue + } + + u := upstream{ + context: context, + name: upstreamName, + } + servers := upstreams[u] + + for _, endpoint := range endpointSlice.Endpoints { + for _, address := range endpoint.Addresses { + host := fmt.Sprintf("%s:%d", address, *port.Port) + servers = append(servers, core.NewUpstreamServer(host)) + } + } + + upstreams[u] = servers + } + } + + for u, servers := range upstreams { + events = append(events, core.NewServerUpdateEvent(core.Updated, u.name, u.context, servers)) + } + + return events, nil +} + +func (t *Translator) buildNodeIPEvents(ports []v1.ServicePort, event *core.Event, +) (core.ServerUpdateEvents, error) { + slog.Debug("Translate::buildNodeIPEvents", "ports", ports) events := core.ServerUpdateEvents{} for _, port := range ports { - ingressName := fixIngressName(port.Name) - upstreamServers, _ := buildUpstreamServers(event.NodeIps, port) - clientType := getClientType(port.Name, event.Service.Annotations) + context, upstreamName, err := getContextAndUpstreamName(port.Name) + if err != nil { + slog.Info(err.Error()) + continue + } + + addresses, err := t.retrieveNodeIps() + if err != nil { + return nil, err + } + + upstreamServers := buildUpstreamServers(addresses, port.NodePort) switch event.Type { case core.Created: fallthrough case core.Updated: - events = append(events, core.NewServerUpdateEvent(event.Type, ingressName, clientType, upstreamServers)) + events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) case core.Deleted: - for _, server := range upstreamServers { - events = append(events, core.NewServerUpdateEvent(event.Type, ingressName, clientType, core.UpstreamServers{server})) - } - + events = append(events, core.NewServerUpdateEvent( + core.Updated, upstreamName, context, nil, + )) default: - logrus.Warnf(`Translator::buildServerUpdateEvents: unknown event type: %d`, event.Type) + slog.Warn(`Translator::buildNodeIPEvents: unknown event type`, "type", event.Type) } } @@ -72,32 +209,73 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se return events, nil } -func buildUpstreamServers(nodeIps []string, port v1.ServicePort) (core.UpstreamServers, error) { +func buildUpstreamServers(ipAddresses []string, port int32) core.UpstreamServers { var servers core.UpstreamServers - for _, nodeIp := range nodeIps { - host := fmt.Sprintf("%s:%d", nodeIp, port.NodePort) + for _, ip := range ipAddresses { + host := fmt.Sprintf("%s:%d", ip, port) server := core.NewUpstreamServer(host) servers = append(servers, server) } - return servers, nil + return servers } -// fixIngressName removes the NlkPrefix from the port name -func fixIngressName(name string) string { - return name[4:] +// getContextAndUpstreamName returns the nginx context being supplied by the port (either "http" or "stream") +// and the upstream name. +func getContextAndUpstreamName(portName string) (clientType string, appName string, err error) { + context, upstreamName, found := strings.Cut(portName, "-") + switch { + case !found: + return clientType, appName, + fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", portName) + case context != "http" && context != "stream": + return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", portName) + default: + return context, upstreamName, nil + } } -// getClientType returns the client type for the port, defaults to ClientTypeNginxHttp if no Annotation is found. -func getClientType(portName string, annotations map[string]string) string { - key := fmt.Sprintf("%s/%s", configuration.PortAnnotationPrefix, portName) - logrus.Infof("getClientType: key=%s", key) - if annotations != nil { - if clientType, ok := annotations[key]; ok { - return clientType +// notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is +// because the master node may or may not be a worker node and thus may not be able to route traffic. +func (t *Translator) retrieveNodeIps() ([]string, error) { + started := time.Now() + slog.Debug("Translator::retrieveNodeIps") + + var nodeIps []string + + nodes, err := t.nodeLister.List(labels.Everything()) + if err != nil { + slog.Error("error occurred retrieving the list of nodes", "error", err) + return nil, err + } + + for _, node := range nodes { + if node == nil { + slog.Error("list contains nil node") + continue + } + + // this is kind of a broad assumption, should probably make this a configurable option + if notMasterNode(*node) { + for _, address := range node.Status.Addresses { + if address.Type == v1.NodeInternalIP { + nodeIps = append(nodeIps, address.Address) + } + } } } - return application.ClientTypeNginxHttp + slog.Debug("Translator::retrieveNodeIps duration", "duration", time.Since(started).Nanoseconds()) + + return nodeIps, nil +} + +// notMasterNode determines if the node is a master node. +func notMasterNode(node v1.Node) bool { + slog.Debug("Translator::notMasterNode") + + _, found := node.Labels["node-role.kubernetes.io/master"] + + return !found } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 2acfd34d..309a07cb 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -7,12 +7,18 @@ package translation import ( "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - v1 "k8s.io/api/core/v1" "math/rand" "testing" "time" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" + v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" ) const ( @@ -20,6 +26,9 @@ const ( ManyNodes = 7 NoNodes = 0 OneNode = 1 + ManyEndpointSlices = 7 + NoEndpointSlices = 0 + OneEndpointSlice = 1 TranslateErrorFormat = "Translate() error = %v" ) @@ -28,125 +37,299 @@ const ( */ func TestCreatedTranslateNoPorts(t *testing.T) { - const expectedEventCount = 0 + t.Parallel() + testcases := map[string]struct{ serviceType v1.ServiceType }{ + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + "loadBalancer": {v1.ServiceTypeLoadBalancer}, + } - service := defaultService() - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildCreatedEvent(service, 0) - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + translator := NewTranslator( + NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), + NewFakeNodeLister([]*v1.Node{}, nil), + ) + + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestCreatedTranslateNoInterestingPorts(t *testing.T) { - const expectedEventCount = 0 - const portCount = 1 + t.Parallel() + testcases := map[string]struct{ serviceType v1.ServiceType }{ + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + "loadBalancer": {v1.ServiceTypeLoadBalancer}, + } - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 0 + const portCount = 1 - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } -} + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service, 0) -func TestCreatedTranslateOneInterestingPort(t *testing.T) { - const expectedEventCount = 1 - const portCount = 1 + translator := NewTranslator( + NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), + NewFakeNodeLister([]*v1.Node{}, nil), + ) - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +//nolint:dupl +func TestCreatedTranslateOneInterestingPort(t *testing.T) { + t.Parallel() + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } - assertExpectedServerCount(t, OneNode, translatedEvents) -} + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() -func TestCreatedTranslateManyInterestingPorts(t *testing.T) { - const expectedEventCount = 4 - const portCount = 4 + const expectedEventCount = 1 + const portCount = 1 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service, tc.ingresses) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +//nolint:dupl +func TestCreatedTranslateManyInterestingPorts(t *testing.T) { + t.Parallel() + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } - assertExpectedServerCount(t, OneNode, translatedEvents) -} + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() -func TestCreatedTranslateManyMixedPorts(t *testing.T) { - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 + const expectedEventCount = 4 + const portCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +//nolint:dupl +func TestCreatedTranslateManyMixedPorts(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } - assertExpectedServerCount(t, OneNode, translatedEvents) -} + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() -func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, ManyNodes) + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service, tc.ingresses) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + expectedServerCount: ManyNodes, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + expectedServerCount: ManyEndpointSlices, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + expectedServerCount: ManyNodes, + }, } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, ManyNodes, translatedEvents) + }) + } } /* @@ -154,125 +337,326 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { */ func TestUpdatedTranslateNoPorts(t *testing.T) { - const expectedEventCount = 0 - - service := defaultService() - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildUpdatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { - const expectedEventCount = 0 - const portCount = 1 - - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestUpdatedTranslateOneInterestingPort(t *testing.T) { - const expectedEventCount = 1 - const portCount = 1 - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 1 + const portCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, OneNode, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { - const expectedEventCount = 4 - const portCount = 4 - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 4 + const portCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyMixedPorts(t *testing.T) { - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + expectedServerCount: OneEndpointSlice, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + ingresses int + endpoints []*discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + expectedServerCount: ManyNodes, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + expectedServerCount: ManyEndpointSlices, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: ManyNodes, + ingresses: ManyNodes, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } /* @@ -280,316 +664,749 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { */ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { - const expectedEventCount = 0 - - service := defaultService() - event := buildDeletedEvent(service, NoNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - assertExpectedServerCount(t, ManyNodes, translatedEvents) -} + const expectedEventCount = 0 -func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { - const expectedEventCount = 0 - const portCount = 1 + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service, tc.ingresses) - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { - const expectedEventCount = 0 - const portCount = 1 - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 1, 1), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - assertExpectedServerCount(t, ManyNodes, translatedEvents) -} + const expectedEventCount = 1 + const portCount = 1 -func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { - const expectedEventCount = 0 - const portCount = 4 + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + assertExpectedServerCount(t, 0, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +//nolint:dupl +func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 4, 4), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { - const expectedEventCount = 0 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 6, 2), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - assertExpectedServerCount(t, ManyNodes, translatedEvents) -} + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 -func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { - const expectedEventCount = 0 + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) - service := defaultService() - event := buildDeletedEvent(service, OneNode) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +//nolint:dupl +func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } - assertExpectedServerCount(t, ManyNodes, translatedEvents) -} + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() -func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { - const expectedEventCount = 0 - const portCount = 1 + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service, tc.ingresses) - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const expectedEventCount = 0 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { - const expectedEventCount = 1 - const portCount = 1 - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - assertExpectedServerCount(t, OneNode, translatedEvents) -} + const portCount = 1 + const expectedEventCount = 1 -func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { - const expectedEventCount = 4 - const portCount = 4 + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + assertExpectedServerCount(t, 0, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +//nolint:dupl +func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } - assertExpectedServerCount(t, OneNode, translatedEvents) -} + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() -func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 + const portCount = 4 + const expectedEventCount = 4 - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } - assertExpectedServerCount(t, OneNode, translatedEvents) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { - const expectedEventCount = 0 - - service := defaultService() - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 0, 0), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 - assertExpectedServerCount(t, ManyNodes, translatedEvents) -} + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service, tc.ingresses) -func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { - const portCount = 1 - const updatablePortCount = 0 - const expectedEventCount = updatablePortCount * ManyNodes + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + assertExpectedServerCount(t, 0, translatedEvents) + }) } +} - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) +func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 0), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const updatablePortCount = 0 + const expectedEventCount = updatablePortCount * ManyNodes + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { - const portCount = 1 - const expectedEventCount = portCount * ManyNodes - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 1), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const expectedEventCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { - const portCount = 4 - const expectedEventCount = portCount * ManyNodes - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 4, 4), + }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { - const portCount = 6 - const updatablePortCount = 2 - const expectedEventCount = updatablePortCount * ManyNodes - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + t.Parallel() + + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []*v1.Node + endpoints []*discovery.EndpointSlice + ingresses int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service, tc.ingresses) + + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } func assertExpectedServerCount(t *testing.T, expectedCount int, events core.ServerUpdateEvents) { @@ -601,46 +1418,102 @@ func assertExpectedServerCount(t *testing.T, expectedCount int, events core.Serv } } -func defaultService() *v1.Service { - return &v1.Service{} +func defaultService(serviceType v1.ServiceType) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-service", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Spec: v1.ServiceSpec{ + Type: serviceType, + }, + } } -func serviceWithPorts(ports []v1.ServicePort) *v1.Service { +func serviceWithPorts(serviceType v1.ServiceType, ports []v1.ServicePort) *v1.Service { return &v1.Service{ Spec: v1.ServiceSpec{ + Type: serviceType, Ports: ports, }, } } -func buildCreatedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Created, service, nodeCount) +func buildCreatedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Created, service, ingressCount) } -func buildDeletedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Deleted, service, nodeCount) +func buildDeletedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Deleted, service, ingressCount) } -func buildUpdatedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Updated, service, nodeCount) +func buildUpdatedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Updated, service, ingressCount) } -func buildEvent(eventType core.EventType, service *v1.Service, nodeCount int) core.Event { - previousService := defaultService() +func buildEvent(eventType core.EventType, service *v1.Service, ingressCount int) core.Event { + event := core.NewEvent(eventType, service) + event.Service.Name = "default-service" + ingresses := make([]v1.LoadBalancerIngress, 0, ingressCount) + for i := range ingressCount { + ingress := v1.LoadBalancerIngress{IP: fmt.Sprintf("ipAddress%d", i)} + ingresses = append(ingresses, ingress) + } + event.Service.Status.LoadBalancer.Ingress = ingresses + return event +} - nodeIps := generateNodeIps(nodeCount) +func generateNodes(count int) (nodes []*v1.Node) { + for i := 0; i < count; i++ { + nodes = append(nodes, &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("node%d", i), + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: fmt.Sprintf("10.0.0.%v", i), + }, + }, + }, + }) + } - return core.NewEvent(eventType, service, previousService, nodeIps) + return nodes } -func generateNodeIps(count int) []string { - var nodeIps []string +func generateEndpointSlices(endpointCount, portCount, updatablePortCount int, +) (endpointSlices []*discovery.EndpointSlice) { + servicePorts := generateUpdatablePorts(portCount, updatablePortCount) - for i := 0; i < count; i++ { - nodeIps = append(nodeIps, fmt.Sprintf("10.0.0.%v", i)) + ports := make([]discovery.EndpointPort, 0, len(servicePorts)) + for _, servicePort := range servicePorts { + ports = append(ports, discovery.EndpointPort{ + Name: pointer.To(servicePort.Name), + Port: pointer.To(int32(8080)), + }) + } + + var endpoints []discovery.Endpoint + for i := 0; i < endpointCount; i++ { + endpoints = append(endpoints, discovery.Endpoint{ + Addresses: []string{ + fmt.Sprintf("10.0.0.%v", i), + }, + }) } - return nodeIps + endpointSlices = append(endpointSlices, &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "endpointSlice", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Endpoints: endpoints, + Ports: ports, + }) + + return endpointSlices } func generatePorts(portCount int) []v1.ServicePort { @@ -649,20 +1522,24 @@ func generatePorts(portCount int) []v1.ServicePort { // This is probably A Little Bit of Too Muchâ„¢, but helps to ensure ordering is not a factor. func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort { - var ports []v1.ServicePort + ports := []v1.ServicePort{} updatable := make([]string, updatableCount) nonupdatable := make([]string, portCount-updatableCount) + contexts := []string{"http-", "stream-"} for i := range updatable { - updatable[i] = configuration.NlkPrefix + randomIndex := int(rand.Float32() * 2.0) + updatable[i] = contexts[randomIndex] } for j := range nonupdatable { nonupdatable[j] = "olm-" } - prefixes := append(updatable, nonupdatable...) + var prefixes []string + prefixes = append(prefixes, updatable...) + prefixes = append(prefixes, nonupdatable...) source := rand.NewSource(time.Now().UnixNano()) random := rand.New(source) @@ -670,9 +1547,54 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort for i, prefix := range prefixes { ports = append(ports, v1.ServicePort{ - Name: fmt.Sprintf("%sport-%d", prefix, i), + Name: fmt.Sprintf("%supstream%d", prefix, i), }) } return ports } + +func NewFakeEndpointSliceLister(list []*discovery.EndpointSlice, err error) discoverylisters.EndpointSliceLister { + return &endpointSliceLister{ + list: list, + err: err, + } +} + +func NewFakeNodeLister(list []*v1.Node, err error) corelisters.NodeLister { + return &nodeLister{ + list: list, + err: err, + } +} + +type nodeLister struct { + list []*v1.Node + err error +} + +func (l *nodeLister) List(selector labels.Selector) (ret []*v1.Node, err error) { + return l.list, l.err +} + +// currently unused +func (l *nodeLister) Get(name string) (*v1.Node, error) { + return nil, nil +} + +type endpointSliceLister struct { + list []*discovery.EndpointSlice + err error +} + +func (l *endpointSliceLister) List(selector labels.Selector) (ret []*discovery.EndpointSlice, err error) { + return l.list, l.err +} + +func (l *endpointSliceLister) Get(name string) (*discovery.EndpointSlice, error) { + return nil, nil +} + +func (l *endpointSliceLister) EndpointSlices(name string) discoverylisters.EndpointSliceNamespaceLister { + return l +} diff --git a/pkg/buildinfo/buildinfo.go b/pkg/buildinfo/buildinfo.go new file mode 100644 index 00000000..5d8839d1 --- /dev/null +++ b/pkg/buildinfo/buildinfo.go @@ -0,0 +1,15 @@ +package buildinfo + +var semVer string + +// SemVer is the version number of this build as provided by build pipeline +func SemVer() string { + return semVer +} + +var shortHash string + +// ShortHash is the 8 char git shorthash +func ShortHash() string { + return shortHash +} diff --git a/pkg/pointer/pointer.go b/pkg/pointer/pointer.go new file mode 100644 index 00000000..08ff6672 --- /dev/null +++ b/pkg/pointer/pointer.go @@ -0,0 +1,56 @@ +// Package pointer provides utilities that assist in working with pointers. +package pointer + +// To returns a pointer to the given value +func To[T any](v T) *T { return &v } + +// From dereferences the pointer if it is not nil or returns d +func From[T any](p *T, d T) T { + if p != nil { + return *p + } + return d +} + +// ToSlice returns a slice of pointers to the given values. +func ToSlice[T any](values []T) []*T { + if len(values) == 0 { + return nil + } + ret := make([]*T, 0, len(values)) + for _, v := range values { + v := v + ret = append(ret, &v) + } + return ret +} + +// FromSlice returns a slice of values to the given pointers, dropping any nils. +func FromSlice[T any](values []*T) []T { + if len(values) == 0 { + return nil + } + ret := make([]T, 0, len(values)) + for _, v := range values { + if v != nil { + ret = append(ret, *v) + } + } + return ret +} + +// Equal reports if p is a pointer to a value equal to v +func Equal[T comparable](p *T, v T) bool { + if p == nil { + return false + } + return *p == v +} + +// ValueEqual reports if value of pointer referenced by p is equal to value of pointer referenced by q +func ValueEqual[T comparable](p *T, q *T) bool { + if p == nil || q == nil { + return p == q + } + return *p == *q +} diff --git a/pkg/pointer/pointer_test.go b/pkg/pointer/pointer_test.go new file mode 100644 index 00000000..e929e58a --- /dev/null +++ b/pkg/pointer/pointer_test.go @@ -0,0 +1,62 @@ +package pointer_test + +import ( + "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" + "github.com/stretchr/testify/require" +) + +func TestTo(t *testing.T) { + t.Parallel() + + for _, v := range []string{"", "hello"} { + require.Equal(t, v, *pointer.To(v)) + } + for _, v := range []int{0, 123456, -123456} { + require.Equal(t, v, *pointer.To(v)) + } + for _, v := range []int64{0, 123456, -123456} { + require.Equal(t, v, *pointer.To(v)) + } +} + +func TestFrom(t *testing.T) { + t.Parallel() + + sv := "s" + sd := "default" + require.Equal(t, sd, pointer.From(nil, sd)) + require.Equal(t, sv, pointer.From(&sv, sd)) + + iv := 1 + id := 2 + require.Equal(t, id, pointer.From(nil, id)) + require.Equal(t, iv, pointer.From(&iv, id)) + + i64v := int64(1) + i64d := int64(2) + require.Equal(t, i64d, pointer.From(nil, i64d)) + require.Equal(t, i64v, pointer.From(&i64v, i64d)) +} + +func TestToSlice_FromSlice(t *testing.T) { + t.Parallel() + + v := []int{1, 2, 3} + require.Equal(t, v, pointer.FromSlice(pointer.ToSlice(v))) + require.Nil(t, pointer.ToSlice([]string{})) + require.Nil(t, pointer.FromSlice([]*string{})) + require.Equal(t, []string{"A", "B"}, pointer.FromSlice([]*string{pointer.To("A"), nil, pointer.To("B")})) +} + +func TestEqual(t *testing.T) { + t.Parallel() + + require.True(t, pointer.Equal(pointer.To(1), 1)) + require.False(t, pointer.Equal(nil, 1)) + require.False(t, pointer.Equal(pointer.To(1), 2)) + + s := new(struct{}) + require.False(t, pointer.Equal(&s, nil)) +} diff --git a/test/mocks/mock_handler.go b/test/mocks/mock_handler.go deleted file mode 100644 index b854db9e..00000000 --- a/test/mocks/mock_handler.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package mocks - -import "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - -type MockHandler struct { -} - -func (h *MockHandler) AddRateLimitedEvent(_ *core.Event) { - -} - -func (h *MockHandler) Initialize() { - -} - -func (h *MockHandler) Run(_ <-chan struct{}) { - -} - -func (h *MockHandler) ShutDown() { - -} diff --git a/test/mocks/mock_nginx_plus_client.go b/test/mocks/mock_nginx_plus_client.go index 2c16e12f..991ab083 100644 --- a/test/mocks/mock_nginx_plus_client.go +++ b/test/mocks/mock_nginx_plus_client.go @@ -5,7 +5,11 @@ package mocks -import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" +import ( + "context" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" +) type MockNginxClient struct { CalledFunctions map[string]bool @@ -26,7 +30,7 @@ func NewErroringMockClient(err error) *MockNginxClient { } } -func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { +func (m MockNginxClient) DeleteStreamServer(_ context.Context, _ string, _ string) error { m.CalledFunctions["DeleteStreamServer"] = true if m.Error != nil { @@ -36,7 +40,11 @@ func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { return nil } -func (m MockNginxClient) UpdateStreamServers(_ string, _ []nginxClient.StreamUpstreamServer) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { +func (m MockNginxClient) UpdateStreamServers( + _ context.Context, + _ string, + _ []nginxClient.StreamUpstreamServer, +) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { m.CalledFunctions["UpdateStreamServers"] = true if m.Error != nil { @@ -46,7 +54,7 @@ func (m MockNginxClient) UpdateStreamServers(_ string, _ []nginxClient.StreamUps return nil, nil, nil, nil } -func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { +func (m MockNginxClient) DeleteHTTPServer(_ context.Context, _ string, _ string) error { m.CalledFunctions["DeleteHTTPServer"] = true if m.Error != nil { @@ -56,7 +64,11 @@ func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { return nil } -func (m MockNginxClient) UpdateHTTPServers(_ string, _ []nginxClient.UpstreamServer) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { +func (m MockNginxClient) UpdateHTTPServers( + _ context.Context, + _ string, + _ []nginxClient.UpstreamServer, +) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { m.CalledFunctions["UpdateHTTPServers"] = true if m.Error != nil { diff --git a/test/mocks/mock_ratelimitinginterface.go b/test/mocks/mock_ratelimitinginterface.go index ee3ccd49..d5da3b71 100644 --- a/test/mocks/mock_ratelimitinginterface.go +++ b/test/mocks/mock_ratelimitinginterface.go @@ -7,51 +7,50 @@ package mocks import "time" -type MockRateLimiter struct { - items []interface{} +type MockRateLimiter[T any] struct { + items []T } -func (m *MockRateLimiter) Add(_ interface{}) { +func (m *MockRateLimiter[T]) Add(_ T) { } -func (m *MockRateLimiter) Len() int { +func (m *MockRateLimiter[T]) Len() int { return len(m.items) } -func (m *MockRateLimiter) Get() (item interface{}, shutdown bool) { +func (m *MockRateLimiter[T]) Get() (item T, shutdown bool) { if len(m.items) > 0 { item = m.items[0] m.items = m.items[1:] return item, false } - return nil, false + return item, false } -func (m *MockRateLimiter) Done(_ interface{}) { +func (m *MockRateLimiter[T]) Done(_ T) { } -func (m *MockRateLimiter) ShutDown() { +func (m *MockRateLimiter[T]) ShutDown() { } -func (m *MockRateLimiter) ShutDownWithDrain() { +func (m *MockRateLimiter[T]) ShutDownWithDrain() { } -func (m *MockRateLimiter) ShuttingDown() bool { +func (m *MockRateLimiter[T]) ShuttingDown() bool { return true } -func (m *MockRateLimiter) AddAfter(item interface{}, _ time.Duration) { +func (m *MockRateLimiter[T]) AddAfter(item T, _ time.Duration) { m.items = append(m.items, item) } -func (m *MockRateLimiter) AddRateLimited(item interface{}) { +func (m *MockRateLimiter[T]) AddRateLimited(item T) { m.items = append(m.items, item) } -func (m *MockRateLimiter) Forget(_ interface{}) { - +func (m *MockRateLimiter[T]) Forget(_ T) { } -func (m *MockRateLimiter) NumRequeues(_ interface{}) int { +func (m *MockRateLimiter[T]) NumRequeues(_ T) int { return 0 } diff --git a/version b/version new file mode 100644 index 00000000..6085e946 --- /dev/null +++ b/version @@ -0,0 +1 @@ +1.2.1