From 21585172c6e5979c09b3d4c4ae9d0944601ac5f1 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Fri, 8 Oct 2021 14:02:00 -0700 Subject: [PATCH 001/163] Revert kubetest2 integration (#17) Remove kubetest2 integration in favour of exploring another e2e framework. * Revert "Update test export API version (#12)" This reverts commit 50cf0a1f31af36aff033e3b158942befe5398e0b. * Revert "Create end to end testing deployment (#9)" This reverts commit b5fa92a1dd1b9a8c6b7a6bbccfe23f2c3e945e43. * Revert "Add kubetest2-kind for end to end testing (#8)" This reverts commit e0d0ea7dac6b6178d69493fbf8113c3cf96cbdc1. --- .github/workflows/build.yml | 5 +---- .gitignore | 5 +---- Makefile | 15 --------------- pkg/api/v1alpha1/zz_generated.deepcopy.go | 1 - testconfig/e2e-deployment.yaml | 22 ---------------------- testconfig/e2e-export.yaml | 7 ------- testconfig/e2e-service-one.yaml | 12 ------------ 7 files changed, 2 insertions(+), 65 deletions(-) delete mode 100644 testconfig/e2e-deployment.yaml delete mode 100644 testconfig/e2e-export.yaml delete mode 100644 testconfig/e2e-service-one.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aa238822..fca0fc95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,8 +16,5 @@ jobs: with: go-version: 1.17 - - name: Run unit tests + - name: make test run: make test - - - name: Run acceptance tests - run: make e2e-test diff --git a/.gitignore b/.gitignore index e828964d..56f77fe1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,5 @@ cover.out # OSX trash .DS_Store -# mocks generated by mockgen +#mocks generated by mockgen mocks/ - -# e2e test artifacts -_artifacts/ diff --git a/Makefile b/Makefile index dc4a79bf..15dbbdfd 100644 --- a/Makefile +++ b/Makefile @@ -58,18 +58,6 @@ test: manifests generate generate-mocks fmt vet ## Run tests. test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -KUBECTL=$(ENVTEST_ASSETS_DIR)/bin/kubectl -TEST_CONFIG=$(shell pwd)/testconfig -E2E_CLUSTER=aws-cloudmap-mcs-e2e -e2e-test: manifests kustomize kubetest2 fmt vet - $(KUBETEST2-KIND) --cluster-name $(E2E_CLUSTER) --up - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - $(KUBECTL) create namespace aws-cloudmap-mcs-e2e - $(KUBECTL) apply -f $(TEST_CONFIG)/e2e-deployment.yaml - $(KUBECTL) apply -f $(TEST_CONFIG)/e2e-service-one.yaml - $(KUBECTL) apply -f $(TEST_CONFIG)/e2e-export.yaml - $(KUBETEST2-KIND) --cluster-name $(E2E_CLUSTER) --down - ##@ Build build: manifests generate generate-mocks fmt vet ## Build manager binary. @@ -118,9 +106,6 @@ MOCKGEN = $(shell pwd)/bin/mockgen mockgen: ## Download mockgen $(call go-get-tool,$(MOCKGEN),github.com/golang/mock/mockgen@v1.6.0) -KUBETEST2-KIND = $(shell pwd)/bin/kubetest2-kind -kubetest2: ## Download kubetest2 - $(call go-get-tool,$(KUBETEST2-KIND),sigs.k8s.io/kubetest2/kubetest2-kind@latest) # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index f74564df..32140a61 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/testconfig/e2e-deployment.yaml b/testconfig/e2e-deployment.yaml deleted file mode 100644 index db54afa4..00000000 --- a/testconfig/e2e-deployment.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# e2e-deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment - labels: - app: nginx -spec: - replicas: 5 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx:1.14.2 - ports: - - containerPort: 80 diff --git a/testconfig/e2e-export.yaml b/testconfig/e2e-export.yaml deleted file mode 100644 index 07fb59b4..00000000 --- a/testconfig/e2e-export.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# e2e-export.yaml - -kind: ServiceExport -apiVersion: multicluster.x-k8s.io/v1alpha1 -metadata: - namespace: aws-cloudmap-mcs-e2e - name: e2e-service-one diff --git a/testconfig/e2e-service-one.yaml b/testconfig/e2e-service-one.yaml deleted file mode 100644 index 4c18fb88..00000000 --- a/testconfig/e2e-service-one.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# e2e-service-one.yaml -kind: Service -apiVersion: v1 -metadata: - namespace: aws-cloudmap-mcs-e2e - name: e2e-service-one -spec: - selector: - app: nginx - ports: - - port: 8080 - targetPort: 80 From 205c162cd7e677fdd97ad83503316581d6d35829 Mon Sep 17 00:00:00 2001 From: Ben Du Date: Sun, 10 Oct 2021 17:05:58 -0700 Subject: [PATCH 002/163] Add instructions to install from latest (#27) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 829847e9..7ef0f5bc 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ AWS Cloud Map MCS Controller for K8s adheres to the [SemVer](https://semver.org/ We also maintain a `latest` tag, which is updated to stay in line with the `main` branch. We **do not** recommend installing this on any production cluster, as any new major versions updated on the `main` branch will introduce breaking changes. +To install from `latest` tag run +```sh +kubectl apply -k github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest +``` + ## Contributing `aws-cloud-map-mcs-controller-for-k8s` is an open source project. See [CONTRIBUTING](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/CONTRIBUTING.md) for details. From 46c4d4f3aa5cf00f5a6127678c7211911e2c2a18 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Sun, 10 Oct 2021 19:33:07 -0700 Subject: [PATCH 003/163] Integrate Github's Codql Analysis (#28) --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..fad209fb --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '35 16 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 5e43764b6f59096ea35d5b122f602eb750e0182c Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Tue, 12 Oct 2021 15:58:12 -0700 Subject: [PATCH 004/163] Add first version to the changelog (#30) --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0cf709b..3894120e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1 +1,4 @@ # CHANGELOG + +## v0.1.0 +Initial release to cover basic end-to-end functionality (ServiceExport and ServiceImport support). From 9379044eea61ad4569d11ee87ae90b7323ca35e6 Mon Sep 17 00:00:00 2001 From: Ben Du Date: Wed, 13 Oct 2021 10:01:53 -0700 Subject: [PATCH 005/163] Add ability to install from release version + doc updates (#29) * Add instructions for installing from release * Development and instllation updates * more updates * add more quotes * Add usage section --- CONTRIBUTING.md | 4 +- README.md | 66 ++++++++++++++++++- .../kustomization.yaml | 7 ++ 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 config/controller_install_release/kustomization.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6d97c743..7236b8ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,7 +96,9 @@ kind delete cluster --name my-cluster ### Deploying to a cluster -You must first push a Docker image containing the changes to a Docker repository like ECR. +You must first push a Docker image containing the changes to a Docker repository like ECR, Github packages, or DockerHub. The repo is configured to use Github Actions to automatically publish the docker image upon push to `main` branch. The image URI will be `ghcr.io/[Your forked repo name here]` You can enable this for forked repos by enabling Github actions on your forked repo in the "Actions" tab of forked repo. + +If you are deploying to cluster using kustomize templates from the `config` directory, you will need to override the image URI away from `ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s` in order to use your own docker images. ### Build and push docker image to ECR diff --git a/README.md b/README.md index 7ef0f5bc..c4da90ee 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,79 @@ ## Introduction AWS Cloud Map multi-cluster service discovery for Kubernetes (K8s) is a controller that implements existing multi-cluster services API that allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. +## Usage +> ⚠ **There must exist network connectivity (i.e. VPC peering, security group rules, ACLs, etc.) between clusters**: Undefined behavior may occur if controller is set up without network connectivity between clusters. + +### Setup clusters + +First, install the controller with latest release on at least 2 AWS EKS clusters. Nodes must have sufficient IAM permissions to perform CloudMap operations. + +```sh +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release" +``` + +> 📌 See [Releases](#Releases) section for details on how to install other versions. + +### Export services + +Then assuming you already have a Service installed, apply a `ServiceExport` yaml to the cluster in which you want to export a service. This can be done for each service you want to export. + +```yaml +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: [Your service namespace here] + name: [Your service name] +``` + +**Example:** This will export a service with name *my-amazing-service* in namespace *hello* +```yaml +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: hello + name: my-amazing-service +``` + +*See the `samples` directory for a set of example yaml files to set up a service and export it. To apply the sample files run* +```sh +kubectl create namespace demo +kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/demo-deployment.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/demo-service.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/demo-export.yaml +``` + +### Import services + +In your other cluster, the controller will automatically sync services registered in AWS CloudMap by applying the appropriate `ServiceImport`. To list them all, run +```sh +kubectl get ServiceImport -A +``` + ## Releases AWS Cloud Map MCS Controller for K8s adheres to the [SemVer](https://semver.org/) specification. Each release updates the major version tag (eg. `vX`), a major/minor version tag (eg. `vX.Y`) and a major/minor/patch version tag (eg. `vX.Y.Z`). To see a full list of all releases, refer to our [Github releases page](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/releases). +To install from a release run +```sh +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release[?ref=*git version tag*]" +``` + +Example to install latest release +```sh +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release" +``` + +Example to install v0.1.0 +```sh +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release?ref=v0.1.0" +``` + We also maintain a `latest` tag, which is updated to stay in line with the `main` branch. We **do not** recommend installing this on any production cluster, as any new major versions updated on the `main` branch will introduce breaking changes. To install from `latest` tag run ```sh -kubectl apply -k github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" ``` ## Contributing diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml new file mode 100644 index 00000000..bc1bf2e1 --- /dev/null +++ b/config/controller_install_release/kustomization.yaml @@ -0,0 +1,7 @@ +bases: +- ../default + +images: +- name: controller + newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s + newTag: v0.1.0 From e7475961dfb96e8e99828ef66cc7f82fdde65e99 Mon Sep 17 00:00:00 2001 From: Jaromir Vanek Date: Wed, 13 Oct 2021 19:04:38 -0700 Subject: [PATCH 006/163] Update README.md badges (Godoc link) (#32) --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c4da90ee..e5dafe51 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # AWS Cloud Map MCS Controller for K8s +[![Documentation](https://img.shields.io/badge/godoc-reference-blue.svg)](https://godoc.org/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) +[![CodeQL](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/codeql-analysis.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/codeql-analysis.yml) [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/issues) -[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg?color=success)](http://www.apache.org/licenses/LICENSE-2.0) -![GitHub issues](https://img.shields.io/github/issues-raw/aws/aws-cloud-map-mcs-controller-for-k8s?style=flat) +[![Build status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/build.yml) +[![Deploy status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml) -[![Deploy status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml/badge.svg)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg?color=success)](http://www.apache.org/licenses/LICENSE-2.0) +[![GitHub issues](https://img.shields.io/github/issues-raw/aws/aws-cloud-map-mcs-controller-for-k8s?style=flat)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/issues) [![Go Report Card](https://goreportcard.com/badge/github.com/aws/aws-cloud-map-mcs-controller-for-k8s)](https://goreportcard.com/report/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) ## Introduction From 1727fe7079f383d967eec65be82dc44d320fc4ba Mon Sep 17 00:00:00 2001 From: Jaromir Vanek Date: Thu, 14 Oct 2021 13:22:26 -0700 Subject: [PATCH 007/163] Integrate Codecov to publish code coverage data (#33) --- .github/.codecov.yml | 26 +++++++++++++++++++++++ .github/workflows/build.yml | 9 +++++++- Makefile | 2 +- README.md | 3 ++- pkg/api/v1alpha1/zz_generated.deepcopy.go | 1 + 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .github/.codecov.yml diff --git a/.github/.codecov.yml b/.github/.codecov.yml new file mode 100644 index 00000000..e7ef3b00 --- /dev/null +++ b/.github/.codecov.yml @@ -0,0 +1,26 @@ +# validate +# cat .codecov.yml | curl --data-binary @- https://codecov.io/validate + +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + if_ci_failed: error #success, failure, error, ignore + informational: true + only_pulls: true + +comment: + layout: "reach,diff,flags,files,footer" + behavior: default + require_changes: no + +ignore: + - "config/**/*" + - "pkg/api/**/*" + - "mocks/**/*" \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fca0fc95..23fa678a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,8 @@ name: build on: + push: + branches: [ main ] pull_request: branches: [ main ] @@ -16,5 +18,10 @@ jobs: with: go-version: 1.17 - - name: make test + - name: Unit tests run: make test + + - name: Upload code coverage + uses: codecov/codecov-action@v2 + with: + files: cover.out diff --git a/Makefile b/Makefile index 15dbbdfd..8be8e80b 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin test: manifests generate generate-mocks fmt vet ## Run tests. mkdir -p ${ENVTEST_ASSETS_DIR} test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh - source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out + source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -covermode=atomic ##@ Build diff --git a/README.md b/README.md index e5dafe51..3af46301 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,12 @@ [![Documentation](https://img.shields.io/badge/godoc-reference-blue.svg)](https://godoc.org/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) [![CodeQL](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/codeql-analysis.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/codeql-analysis.yml) -[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/issues) [![Build status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/build.yml) [![Deploy status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml) +[![codecov](https://codecov.io/gh/aws/aws-cloud-map-mcs-controller-for-k8s/branch/main/graph/badge.svg)](https://codecov.io/gh/aws/aws-cloud-map-mcs-controller-for-k8s) [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg?color=success)](http://www.apache.org/licenses/LICENSE-2.0) +[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/issues) [![GitHub issues](https://img.shields.io/github/issues-raw/aws/aws-cloud-map-mcs-controller-for-k8s?style=flat)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/issues) [![Go Report Card](https://goreportcard.com/badge/github.com/aws/aws-cloud-map-mcs-controller-for-k8s)](https://goreportcard.com/report/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 32140a61..f74564df 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* From 1298c5de5869ef4649e1d7b0a14dec3495ee5b6d Mon Sep 17 00:00:00 2001 From: Jaromir Vanek Date: Thu, 14 Oct 2021 16:20:41 -0700 Subject: [PATCH 008/163] Fix instance port conversion to be CPU arch independent (#35) --- pkg/model/types.go | 18 ++++++------ pkg/model/types_test.go | 62 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 pkg/model/types_test.go diff --git a/pkg/model/types.go b/pkg/model/types.go index 0dcd4dbe..41d21698 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -23,8 +23,10 @@ type Endpoint struct { Attributes map[string]string } -const ipv4Attr = "AWS_INSTANCE_IPV4" -const portAttr = "AWS_INSTANCE_PORT" +const ( + Ipv4Attr = "AWS_INSTANCE_IPV4" + PortAttr = "AWS_INSTANCE_PORT" +) // NewEndpointFromInstance converts a Cloud Map InstanceSummary to an endpoint func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { @@ -33,14 +35,14 @@ func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { Attributes: make(map[string]string, 0), } - if ipv4, hasIp := inst.Attributes[ipv4Attr]; hasIp { + if ipv4, hasIp := inst.Attributes[Ipv4Attr]; hasIp { endpoint.IP = ipv4 } else { return nil, errors.New(fmt.Sprintf("cannot convert service instance %s to endpoint without IP address", *inst.Id)) } - if portStr, hasPort := inst.Attributes[portAttr]; hasPort { - port, parseError := strconv.Atoi(portStr) + if portStr, hasPort := inst.Attributes[PortAttr]; hasPort { + port, parseError := strconv.ParseUint(portStr, 10, 16) if parseError != nil { return nil, parseError @@ -52,7 +54,7 @@ func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { } for key, val := range inst.Attributes { - if key != ipv4Attr && key != portAttr { + if key != Ipv4Attr && key != PortAttr { endpoint.Attributes[key] = val } } @@ -64,10 +66,10 @@ func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { func (e *Endpoint) GetAttributes() map[string]string { attrs := make(map[string]string, 0) - attrs[ipv4Attr] = e.IP + attrs[Ipv4Attr] = e.IP port := strconv.FormatInt(int64(e.Port), 10) - attrs[portAttr] = port + attrs[PortAttr] = port for key, val := range e.Attributes { attrs[key] = val diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go new file mode 100644 index 00000000..c25e3b8a --- /dev/null +++ b/pkg/model/types_test.go @@ -0,0 +1,62 @@ +package model + +import ( + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" + "reflect" + "testing" +) + +var instId = "my-instance" + +func TestNewEndpointFromInstance(t *testing.T) { + tests := []struct { + name string + inst *types.InstanceSummary + want *Endpoint + wantErr bool + }{ + { + name: "happy case", + inst: &types.InstanceSummary{ + Id: &instId, + Attributes: map[string]string{ + Ipv4Attr: "192.168.0.1", + PortAttr: "65535", + "custom-attr": "custom-val", + }, + }, + want: &Endpoint{ + Id: instId, + IP: "192.168.0.1", + Port: 65535, + Attributes: map[string]string{ + "custom-attr": "custom-val", + }, + }, + }, + { + name: "invalid port", + inst: &types.InstanceSummary{ + Id: &instId, + Attributes: map[string]string{ + Ipv4Attr: "192.168.0.1", + PortAttr: "99999", + "custom-attr": "custom-val", + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewEndpointFromInstance(tt.inst) + if (err != nil) != tt.wantErr { + t.Errorf("NewEndpointFromInstance() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewEndpointFromInstance() got = %v, want %v", got, tt.want) + } + }) + } +} From ed85cdf6976c881d5a5031dd98b8e86a9ff80602 Mon Sep 17 00:00:00 2001 From: thalleslmF Date: Fri, 15 Oct 2021 02:15:34 -0300 Subject: [PATCH 009/163] Create namespace when it does not exist (#25) Create http namespace in Cloud Map if necessary when creating a service. Co-authored-by: thalles --- pkg/cloudmap/client.go | 55 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 81052d4e..3efaaa72 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -2,7 +2,6 @@ package cloudmap import ( "context" - "errors" "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" @@ -10,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/cache" + "k8s.io/apimachinery/pkg/util/wait" ctrl "sigs.k8s.io/controller-runtime" "time" ) @@ -86,11 +86,28 @@ func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, service *m sdc.log.Info("creating a new service", "namespace", service.Namespace, "name", service.Name) nsId, nsErr := sdc.getNamespaceId(ctx, service.Namespace) - if nsErr != nil { return nsErr } + if nsId == "" { + nsOutput, nsErr := sdc.sdApi.CreateHttpNamespace(ctx, &sd.CreateHttpNamespaceInput{ + Name: &service.Namespace, + }) + + if nsErr != nil { + return nsErr + } + opResult, opErr := sdc.WaitUntilSuccessOperation(ctx, nsOutput.OperationId) + if opErr != nil { + return opErr + } + nsId = opResult.Operation.Targets["NAMESPACE"] + sdc.namespaceIdCache.Add( + service.Namespace, + nsId, defaultNamespaceIdCacheTTL) + } + //TODO: Handle non-http namespaces sdSrv, srvErr := sdc.sdApi.CreateService(ctx, &sd.CreateServiceInput{ Name: &service.Name, @@ -106,6 +123,32 @@ func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, service *m return sdc.RegisterEndpoints(ctx, service) } +func (sdc *serviceDiscoveryClient) WaitUntilSuccessOperation(ctx context.Context, operationId *string) (*sd.GetOperationOutput, error) { + opResult := &sd.GetOperationOutput{} + var opErr error + err := wait.PollUntil(defaultOperationPollInterval, func() (bool, error) { + opResult, opErr = sdc.sdApi.GetOperation(ctx, &sd.GetOperationInput{ + OperationId: operationId, + }) + if opErr != nil { + return true, opErr + } + + if opResult.Operation.Status == types.OperationStatusFail { + return true, fmt.Errorf("failed to create namespace.Reason: %s", *opResult.Operation.ErrorMessage) + } + + if opResult.Operation.Status == types.OperationStatusSuccess { + return true, nil + } + + return false, nil + }, ctx.Done()) + if err != nil { + return nil, err + } + return opResult, nil +} func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, namespaceName string, serviceName string) (*model.Service, error) { sdc.log.Info("fetching a service", "namespaceName", namespaceName, "serviceName", serviceName) @@ -179,7 +222,9 @@ func (sdc *serviceDiscoveryClient) getNamespaceId(ctx context.Context, nsName st return "", err } - sdc.namespaceIdCache.Add(nsName, nsId, defaultNamespaceIdCacheTTL) + if nsId != "" { + sdc.namespaceIdCache.Add(nsName, nsId, defaultNamespaceIdCacheTTL) + } return nsId, err } @@ -201,7 +246,7 @@ func (sdc *serviceDiscoveryClient) getNamespaceIdFromCloudMap(ctx context.Contex } } - return "", errors.New(fmt.Sprintf("namespace %s not found", nsName)) + return "", nil } func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName string, svcName string) (string, error) { @@ -244,7 +289,7 @@ func (sdc *serviceDiscoveryClient) listServicesFromCloudMap(ctx context.Context, svcs := make([]*types.ServiceSummary, 0) nsId, nsErr := sdc.getNamespaceId(ctx, nsName) - if nsErr != nil { + if nsErr != nil || nsId == "" { return svcs, nil } From 7e2c176090483324121819f6fbc6de874ad84ad0 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Fri, 15 Oct 2021 13:38:44 -0700 Subject: [PATCH 010/163] Modularize Cloud Map client (#31) Encapsulate Cloud map client logic into modular and maintainable interfaces. AWS facade provides minimal SDK API surface to facilitate mocking. Service discovery API acts as a thin wrapper to handle conversion of SD request/response data structures. Operation poller simplifies logic to monitor multiple inflight operation status. Operation collector asynchronously collects operations to poll. Client and Endpoint Manager have been unified due to simplified design gains following above. --- Makefile | 5 + pkg/cloudmap/api.go | 250 ++++++++++++++ pkg/cloudmap/aws_facade.go | 47 +++ pkg/cloudmap/client.go | 323 +++++++++--------- pkg/cloudmap/endpoint_manager.go | 291 ---------------- pkg/cloudmap/operation_collector.go | 66 ++++ pkg/cloudmap/operation_poller.go | 150 ++++++++ .../serviceexport_controller_test.go | 2 - 8 files changed, 688 insertions(+), 446 deletions(-) create mode 100644 pkg/cloudmap/api.go create mode 100644 pkg/cloudmap/aws_facade.go delete mode 100644 pkg/cloudmap/endpoint_manager.go create mode 100644 pkg/cloudmap/operation_collector.go create mode 100644 pkg/cloudmap/operation_poller.go diff --git a/Makefile b/Makefile index 8be8e80b..76f36b10 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,11 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi MOCKS_DESTINATION=mocks generate-mocks: mockgen $(MOCKGEN) --source pkg/cloudmap/client.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/client_mock.go --package cloudmap + $(MOCKGEN) --source pkg/cloudmap/operation_poller.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_poller_mock.go --package cloudmap + $(MOCKGEN) --source pkg/cloudmap/operation_collector.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_collector_mock.go --package cloudmap + $(MOCKGEN) --source pkg/cloudmap/api.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/api_mock.go --package cloudmap + $(MOCKGEN) --source pkg/cloudmap/aws_facade.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/aws_facade_mock.go --package cloudmap + CONTROLLER_GEN = $(shell pwd)/bin/controller-gen controller-gen: ## Download controller-gen locally if necessary. diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go new file mode 100644 index 00000000..eacd146a --- /dev/null +++ b/pkg/cloudmap/api.go @@ -0,0 +1,250 @@ +package cloudmap + +import ( + "context" + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-sdk-go-v2/aws" + sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/wait" + ctrl "sigs.k8s.io/controller-runtime" +) + +// ServiceDiscoveryApi handles the AWS Cloud Map API request and response processing logic, and converts results to +// internal data structures. It manages all interactions with the AWS SDK. +type ServiceDiscoveryApi interface { + // ListNamespaces returns a list of all namespaces. + ListNamespaces(ctx context.Context) (namespaces []*Resource, err error) + + // ListServices returns a list of services for a given namespace. + ListServices(ctx context.Context, namespaceId string) (services []*Resource, err error) + + // ListInstances returns a list of service instances registered to a given service. + ListInstances(ctx context.Context, serviceId string) ([]*model.Endpoint, error) + + // ListOperations returns a map of operations to their status matching a list of filters. + ListOperations(ctx context.Context, opFilters []types.OperationFilter) (operationStatusMap map[string]types.OperationStatus, err error) + + // GetOperation returns an operation. + GetOperation(ctx context.Context, operationId string) (operation *types.Operation, err error) + + // CreateHttpNamespace creates a HTTP namespace in AWS Cloud Map for a given name. + CreateHttpNamespace(ctx context.Context, namespaceName string) (operationId string, err error) + + // CreateService creates a named service in AWS Cloud Map under the given namespace. + CreateService(ctx context.Context, namespaceId string, serviceName string) (serviceId string, err error) + + // RegisterInstance registers a service instance in AWS Cloud Map. + RegisterInstance(ctx context.Context, serviceId string, instanceId string, instanceAttrs map[string]string) (operationId string, err error) + + // DeregisterInstance de-registers a service instance in Cloud Map. + DeregisterInstance(ctx context.Context, serviceId string, instanceId string) (operationId string, err error) + + // PollCreateNamespace polls a create namespace operation, and returns the namespace ID. + PollCreateNamespace(ctx context.Context, operationId string) (namespaceId string, err error) +} + +type serviceDiscoveryApi struct { + log logr.Logger + awsFacade AwsFacade +} + +// Resource encapsulates a ID/name pair +type Resource struct { + Id string + Name string +} + +// NewServiceDiscoveryApiFromConfig creates a new AWS Cloud Map API connection manager from an AWS client config. +func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { + return &serviceDiscoveryApi{ + log: ctrl.Log.WithName("cloudmap"), + awsFacade: NewAwsFacadeFromConfig(cfg), + } +} + +func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*Resource, error) { + namespaces := make([]*Resource, 0) + pages := sd.NewListNamespacesPaginator(sdApi.awsFacade, &sd.ListNamespacesInput{}) + + for pages.HasMorePages() { + output, err := pages.NextPage(ctx) + if err != nil { + return namespaces, err + } + + for _, ns := range output.Namespaces { + namespaces = append(namespaces, &Resource{ + Id: aws.ToString(ns.Id), + Name: aws.ToString(ns.Name), + }) + } + } + + return namespaces, nil +} + +func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) ([]*Resource, error) { + svcs := make([]*Resource, 0) + + filter := types.ServiceFilter{ + Name: types.ServiceFilterNameNamespaceId, + Values: []string{nsId}, + } + sdApi.log.Info("paginating", "nsId", nsId) + + pages := sd.NewListServicesPaginator(sdApi.awsFacade, &sd.ListServicesInput{Filters: []types.ServiceFilter{filter}}) + + for pages.HasMorePages() { + output, err := pages.NextPage(ctx) + if err != nil { + return svcs, err + } + + for _, svc := range output.Services { + svcs = append(svcs, &Resource{ + Id: aws.ToString(svc.Id), + Name: aws.ToString(svc.Name), + }) + } + } + + return svcs, nil +} + +func (sdApi *serviceDiscoveryApi) ListInstances(ctx context.Context, svcId string) ([]*model.Endpoint, error) { + endpts := make([]*model.Endpoint, 0) + + pages := sd.NewListInstancesPaginator(sdApi.awsFacade, &sd.ListInstancesInput{ServiceId: &svcId}) + + for pages.HasMorePages() { + output, err := pages.NextPage(ctx) + if err != nil { + return endpts, err + } + + for _, inst := range output.Instances { + endpt, endptErr := model.NewEndpointFromInstance(&inst) + + if endptErr != nil { + sdApi.log.Info(fmt.Sprintf("skipping instance %s to endpoint conversion: %s", *inst.Id, endptErr.Error())) + continue + } + + endpts = append(endpts, endpt) + } + } + + return endpts, nil +} + +func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters []types.OperationFilter) (opStatusMap map[string]types.OperationStatus, err error) { + opStatusMap = make(map[string]types.OperationStatus, 0) + + pages := sd.NewListOperationsPaginator(sdApi.awsFacade, &sd.ListOperationsInput{ + Filters: opFilters, + }) + + for pages.HasMorePages() { + output, err := pages.NextPage(ctx) + + if err != nil { + return opStatusMap, err + } + + for _, sdOp := range output.Operations { + opStatusMap[aws.ToString(sdOp.Id)] = sdOp.Status + } + } + + return opStatusMap, nil +} + +func (sdApi *serviceDiscoveryApi) GetOperation(ctx context.Context, opId string) (operation *types.Operation, err error) { + opResp, err := sdApi.awsFacade.GetOperation(ctx, &sd.GetOperationInput{OperationId: &opId}) + + if err != nil { + return nil, err + } + + return opResp.Operation, nil +} + +func (sdApi *serviceDiscoveryApi) CreateHttpNamespace(ctx context.Context, nsName string) (opId string, err error) { + output, err := sdApi.awsFacade.CreateHttpNamespace(ctx, &sd.CreateHttpNamespaceInput{ + Name: &nsName, + }) + + if err != nil { + return "", err + } + + return aws.ToString(output.OperationId), nil +} + +func (sdApi *serviceDiscoveryApi) CreateService(ctx context.Context, nsId string, svcName string) (svcId string, err error) { + output, err := sdApi.awsFacade.CreateService(ctx, &sd.CreateServiceInput{ + NamespaceId: &nsId, + Name: &svcName}) + + if err != nil { + return "", err + } + + svcId = aws.ToString(output.Service.Id) + sdApi.log.Info("service created", "svcId", svcId) + return svcId, nil +} + +func (sdApi *serviceDiscoveryApi) RegisterInstance(ctx context.Context, svcId string, instId string, instAttrs map[string]string) (opId string, err error) { + regResp, err := sdApi.awsFacade.RegisterInstance(ctx, &sd.RegisterInstanceInput{ + Attributes: instAttrs, + InstanceId: &instId, + ServiceId: &svcId, + }) + + if err != nil { + return "", err + } + + return aws.ToString(regResp.OperationId), nil +} + +func (sdApi *serviceDiscoveryApi) DeregisterInstance(ctx context.Context, svcId string, instId string) (opId string, err error) { + deregResp, err := sdApi.awsFacade.DeregisterInstance(ctx, &sd.DeregisterInstanceInput{ + InstanceId: &instId, + ServiceId: &svcId, + }) + + if err != nil { + return "", err + } + + return aws.ToString(deregResp.OperationId), err + +} + +func (sdApi *serviceDiscoveryApi) PollCreateNamespace(ctx context.Context, opId string) (nsId string, err error) { + return nsId, wait.Poll(defaultOperationPollInterval, defaultOperationPollTimeout, func() (done bool, pollErr error) { + sdApi.log.Info("polling operation", "opId", opId) + op, opErr := sdApi.GetOperation(ctx, opId) + + if opErr != nil { + return true, opErr + } + + if op.Status == types.OperationStatusFail { + return true, fmt.Errorf("failed to create namespace: %s", aws.ToString(op.ErrorMessage)) + } + + if op.Status == types.OperationStatusSuccess { + nsId = op.Targets[string(types.OperationTargetTypeNamespace)] + sdApi.log.Info("namespace created", "nsId", nsId) + return true, nil + } + + return false, nil + }) +} diff --git a/pkg/cloudmap/aws_facade.go b/pkg/cloudmap/aws_facade.go new file mode 100644 index 00000000..199ce1b3 --- /dev/null +++ b/pkg/cloudmap/aws_facade.go @@ -0,0 +1,47 @@ +package cloudmap + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" +) + +// AwsFacade wraps the minimal surface area of ServiceDiscovery API calls for the AWS SDK +// required by the AWS Cloud Map client. This enables mock generation for unit testing. +type AwsFacade interface { + // ListNamespaces provides ServiceDiscovery ListNamespaces wrapper interface for paginator. + ListNamespaces(context.Context, *sd.ListNamespacesInput, ...func(*sd.Options)) (*sd.ListNamespacesOutput, error) + + // ListServices provides ServiceDiscovery ListServices wrapper interface for paginator. + ListServices(context.Context, *sd.ListServicesInput, ...func(options *sd.Options)) (*sd.ListServicesOutput, error) + + // ListInstances provides ServiceDiscovery ListInstances wrapper interface for paginator. + ListInstances(context.Context, *sd.ListInstancesInput, ...func(*sd.Options)) (*sd.ListInstancesOutput, error) + + // ListOperations provides ServiceDiscovery ListOperations wrapper interface for paginator. + ListOperations(context.Context, *sd.ListOperationsInput, ...func(*sd.Options)) (*sd.ListOperationsOutput, error) + + // GetOperation provides ServiceDiscovery GetOperation wrapper interface. + GetOperation(context.Context, *sd.GetOperationInput, ...func(*sd.Options)) (*sd.GetOperationOutput, error) + + // CreateHttpNamespace provides ServiceDiscovery CreateHttpNamespace wrapper interface. + CreateHttpNamespace(context.Context, *sd.CreateHttpNamespaceInput, ...func(*sd.Options)) (*sd.CreateHttpNamespaceOutput, error) + + // CreateService provides ServiceDiscovery CreateService wrapper interface. + CreateService(context.Context, *sd.CreateServiceInput, ...func(*sd.Options)) (*sd.CreateServiceOutput, error) + + // RegisterInstance provides ServiceDiscovery RegisterInstance wrapper interface. + RegisterInstance(context.Context, *sd.RegisterInstanceInput, ...func(*sd.Options)) (*sd.RegisterInstanceOutput, error) + + // DeregisterInstance provides ServiceDiscovery DeregisterInstance wrapper interface. + DeregisterInstance(context.Context, *sd.DeregisterInstanceInput, ...func(*sd.Options)) (*sd.DeregisterInstanceOutput, error) +} + +type awsFacade struct { + *sd.Client +} + +// NewAwsFacadeFromConfig creates a new AWS facade from an AWS client config. +func NewAwsFacadeFromConfig(cfg *aws.Config) AwsFacade { + return &awsFacade{sd.NewFromConfig(*cfg)} +} diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 3efaaa72..7de78271 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -5,11 +5,8 @@ import ( "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" - sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" - "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/cache" - "k8s.io/apimachinery/pkg/util/wait" ctrl "sigs.k8s.io/controller-runtime" "time" ) @@ -19,62 +16,74 @@ const ( defaultNamespaceIdCacheSize = 100 defaultServiceIdCacheTTL = 2 * time.Minute defaultServiceIdCacheSize = 1024 + defaultEndpointsCacheTTL = 5 * time.Second + defaultEndpointsCacheSize = 1024 ) +// ServiceDiscoveryClient provides the service endpoint management functionality required by the AWS Cloud Map +// multi-cluster service discovery for Kubernetes controller. It maintains local caches for all AWS Cloud Map resources. type ServiceDiscoveryClient interface { - // ListServices returns all services and their endpoints for a given namespace + // ListServices returns all services and their endpoints for a given namespace. ListServices(ctx context.Context, namespaceName string) ([]*model.Service, error) - // CreateService creates a Cloud Map service resource and return created service struct + // CreateService creates a Cloud Map service resource and returns the created service struct. CreateService(ctx context.Context, service *model.Service) error - // GetService returns a service resource fetched from the Cloud Map API or nil if not found - GetService(ctx context.Context, namespace string, name string) (*model.Service, error) + // GetService returns a service resource fetched from AWS Cloud Map or nil if not found. + GetService(ctx context.Context, namespaceName string, serviceName string) (*model.Service, error) - // RegisterEndpoints registers all endpoints for given service + // RegisterEndpoints registers all endpoints for given service. RegisterEndpoints(ctx context.Context, service *model.Service) error - // DeleteEndpoints de-registers all endpoints for given service + // DeleteEndpoints de-registers all endpoints for given service. DeleteEndpoints(ctx context.Context, service *model.Service) error } type serviceDiscoveryClient struct { log logr.Logger - sdApi *sd.Client + sdApi ServiceDiscoveryApi namespaceIdCache *cache.LRUExpireCache serviceIdCache *cache.LRUExpireCache - EndpointManager + endpointCache *cache.LRUExpireCache } +// NewServiceDiscoveryClient creates a new service discovery client for AWS Cloud Map from a given AWS client config. func NewServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { return &serviceDiscoveryClient{ log: ctrl.Log.WithName("cloudmap"), - sdApi: sd.NewFromConfig(*cfg), + sdApi: NewServiceDiscoveryApiFromConfig(cfg), namespaceIdCache: cache.NewLRUExpireCache(defaultNamespaceIdCacheSize), serviceIdCache: cache.NewLRUExpireCache(defaultServiceIdCacheSize), - EndpointManager: NewEndpointManager(cfg), + endpointCache: cache.NewLRUExpireCache(defaultEndpointsCacheSize), } } -func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, namespaceName string) ([]*model.Service, error) { - svcs := make([]*model.Service, 0) +func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName string) (svcs []*model.Service, err error) { + svcs = make([]*model.Service, 0) - svcSums, svcErr := sdc.listServicesFromCloudMap(ctx, namespaceName) + nsId, err := sdc.getNamespaceId(ctx, nsName) + if err != nil || nsId == "" { + return svcs, err + } + + svcSums, err := sdc.sdApi.ListServices(ctx, nsId) - if svcErr != nil { - return svcs, svcErr + if err != nil { + return svcs, err } for _, svcSum := range svcSums { - endpts, endptsErr := sdc.EndpointManager.ListEndpoints(ctx, aws.ToString(svcSum.Id)) + endpts, endptsErr := sdc.ListEndpoints(ctx, svcSum.Id) - if endptsErr != nil { + if err != nil { return svcs, endptsErr } + sdc.cacheServiceId(nsName, svcSum.Name, svcSum.Id) + svcs = append(svcs, &model.Service{ - Namespace: namespaceName, - Name: aws.ToString(svcSum.Name), + Namespace: nsName, + Name: svcSum.Name, Endpoints: endpts, }) } @@ -82,242 +91,250 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, namespaceNa return svcs, nil } -func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, service *model.Service) error { +func (sdc *serviceDiscoveryClient) ListEndpoints(ctx context.Context, serviceId string) (endpts []*model.Endpoint, err error) { + + if cachedValue, exists := sdc.endpointCache.Get(serviceId); exists { + return cachedValue.([]*model.Endpoint), nil + } + + endpts, err = sdc.sdApi.ListInstances(ctx, serviceId) + + if err != nil { + return nil, err + } + + sdc.cacheEndpoints(serviceId, endpts) + + return endpts, nil +} + +func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, service *model.Service) (err error) { sdc.log.Info("creating a new service", "namespace", service.Namespace, "name", service.Name) - nsId, nsErr := sdc.getNamespaceId(ctx, service.Namespace) - if nsErr != nil { - return nsErr + nsId, err := sdc.getNamespaceId(ctx, service.Namespace) + if err != nil { + return err } if nsId == "" { - nsOutput, nsErr := sdc.sdApi.CreateHttpNamespace(ctx, &sd.CreateHttpNamespaceInput{ - Name: &service.Namespace, - }) - - if nsErr != nil { - return nsErr - } - opResult, opErr := sdc.WaitUntilSuccessOperation(ctx, nsOutput.OperationId) - if opErr != nil { - return opErr - } - nsId = opResult.Operation.Targets["NAMESPACE"] - sdc.namespaceIdCache.Add( - service.Namespace, - nsId, defaultNamespaceIdCacheTTL) + nsId, err = sdc.createNamespace(ctx, service.Namespace) } //TODO: Handle non-http namespaces - sdSrv, srvErr := sdc.sdApi.CreateService(ctx, &sd.CreateServiceInput{ - Name: &service.Name, - NamespaceId: &nsId}) + svcId, err := sdc.sdApi.CreateService(ctx, nsId, service.Name) - if srvErr != nil { - return srvErr + if err != nil { + return err } - sdc.serviceIdCache.Add( - sdc.buildServiceIdCacheKey(nsId, service.Name), - *sdSrv.Service.Id, defaultServiceIdCacheTTL) + sdc.cacheServiceId(service.Namespace, service.Name, svcId) return sdc.RegisterEndpoints(ctx, service) } -func (sdc *serviceDiscoveryClient) WaitUntilSuccessOperation(ctx context.Context, operationId *string) (*sd.GetOperationOutput, error) { - opResult := &sd.GetOperationOutput{} - var opErr error - err := wait.PollUntil(defaultOperationPollInterval, func() (bool, error) { - opResult, opErr = sdc.sdApi.GetOperation(ctx, &sd.GetOperationInput{ - OperationId: operationId, - }) - if opErr != nil { - return true, opErr - } - if opResult.Operation.Status == types.OperationStatusFail { - return true, fmt.Errorf("failed to create namespace.Reason: %s", *opResult.Operation.ErrorMessage) - } +func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string, svcName string) (svc *model.Service, err error) { + sdc.log.Info("fetching a service", "nsName", nsName, "svcName", svcName) - if opResult.Operation.Status == types.OperationStatusSuccess { - return true, nil - } + svcId, err := sdc.getServiceId(ctx, nsName, svcName) - return false, nil - }, ctx.Done()) if err != nil { return nil, err } - return opResult, nil -} - -func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, namespaceName string, serviceName string) (*model.Service, error) { - sdc.log.Info("fetching a service", "namespaceName", namespaceName, "serviceName", serviceName) - - svcId, svcIdErr := sdc.getServiceId(ctx, namespaceName, serviceName) - - if svcIdErr != nil { - return nil, svcIdErr - } if svcId == "" { return nil, nil } - endpts, endptsErr := sdc.EndpointManager.ListEndpoints(ctx, svcId) + endpts, err := sdc.ListEndpoints(ctx, svcId) - if endptsErr != nil { - return nil, endptsErr + if err != nil { + return nil, err } - svc := &model.Service{ - Namespace: namespaceName, - Name: serviceName, + svc = &model.Service{ + Namespace: nsName, + Name: svcName, Endpoints: endpts, } return svc, nil } -func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, service *model.Service) error { +func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, service *model.Service) (err error) { if len(service.Endpoints) == 0 { sdc.log.Info("skipping endpoint registration for empty endpoint list", "serviceName", service.Name) return nil } + sdc.log.Info("registering endpoints", "namespaceName", service.Namespace, "serviceName", service.Name, "endpoints", service.Endpoints) - svcId, svcErr := sdc.getServiceId(ctx, service.Namespace, service.Name) - if svcErr != nil { - return svcErr + svcId, err := sdc.getServiceId(ctx, service.Namespace, service.Name) + if err != nil { + return err + } + + startTime := Now() + opCollector := NewOperationCollector(len(service.Endpoints)) + + for _, endpt := range service.Endpoints { + go func(endpt *model.Endpoint) { + opId, endptErr := sdc.sdApi.RegisterInstance(ctx, svcId, endpt.Id, endpt.GetAttributes()) + opCollector.Add(endpt.Id, opId, endptErr) + }(endpt) + } + + err = NewRegisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), startTime).Poll(ctx) + + // Evict cache entry so next list call reflects changes + sdc.evictEndpoints(svcId) + + if err != nil { + return err + } + + if !opCollector.IsAllOperationsCreated() { + return fmt.Errorf("failure while registering endpoints") } - return sdc.EndpointManager.RegisterEndpoints(ctx, service, svcId) + return nil } -func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, service *model.Service) error { +func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, service *model.Service) (err error) { if len(service.Endpoints) == 0 { sdc.log.Info("skipping endpoint deletion for empty endpoint list", "serviceName", service.Name) return nil } + sdc.log.Info("deleting endpoints", "namespaceName", service.Namespace, "serviceName", service.Name, "endpoints", service.Endpoints) - svcId, svcErr := sdc.getServiceId(ctx, service.Namespace, service.Name) - if svcErr != nil { - return svcErr + svcId, err := sdc.getServiceId(ctx, service.Namespace, service.Name) + if err != nil { + return err } - return sdc.EndpointManager.DeregisterEndpoints(ctx, service, svcId) -} + startTime := Now() + opCollector := NewOperationCollector(len(service.Endpoints)) -func (sdc *serviceDiscoveryClient) getNamespaceId(ctx context.Context, nsName string) (string, error) { - // We are assuming a unique namespace name per account - if cachedValue, exists := sdc.namespaceIdCache.Get(nsName); exists { - return cachedValue.(string), nil + for _, endpt := range service.Endpoints { + go func(endpt *model.Endpoint) { + opId, endptErr := sdc.sdApi.DeregisterInstance(ctx, svcId, endpt.Id) + opCollector.Add(endpt.Id, opId, endptErr) + }(endpt) } - nsId, err := sdc.getNamespaceIdFromCloudMap(ctx, nsName) + err = NewDeregisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), startTime).Poll(ctx) + + // Evict cache entry so next list call reflects changes + sdc.evictEndpoints(svcId) if err != nil { - return "", err + return err } - if nsId != "" { - sdc.namespaceIdCache.Add(nsName, nsId, defaultNamespaceIdCacheTTL) + if !opCollector.IsAllOperationsCreated() { + return fmt.Errorf("failure while de-registering endpoints") } - return nsId, err + return nil } -func (sdc *serviceDiscoveryClient) getNamespaceIdFromCloudMap(ctx context.Context, nsName string) (string, error) { +func (sdc *serviceDiscoveryClient) getNamespaceId(ctx context.Context, nsName string) (nsId string, err error) { + // We are assuming a unique namespace name per account + if cachedValue, exists := sdc.namespaceIdCache.Get(nsName); exists { + return cachedValue.(string), nil + } - pages := sd.NewListNamespacesPaginator(sdc.sdApi, &sd.ListNamespacesInput{}) + namespaces, err := sdc.sdApi.ListNamespaces(ctx) - for pages.HasMorePages() { - output, err := pages.NextPage(ctx) - if err != nil { - return "", err - } + if err != nil { + return "", err + } - for _, ns := range output.Namespaces { - if nsName == aws.ToString(ns.Name) { - return aws.ToString(ns.Id), nil - } + for _, ns := range namespaces { + sdc.cacheNamespaceId(nsName, nsId) + if nsName == ns.Name { + nsId = ns.Id } } - return "", nil + return nsId, nil } -func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName string, svcName string) (string, error) { +func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName string, svcName string) (svcId string, err error) { cacheKey := sdc.buildServiceIdCacheKey(nsName, svcName) if cachedValue, exists := sdc.serviceIdCache.Get(cacheKey); exists { return cachedValue.(string), nil } - svcId, svcErr := sdc.getServiceIdFromCloudMap(ctx, nsName, svcName) + nsId, err := sdc.getNamespaceId(ctx, nsName) - if svcErr != nil { - return "", svcErr + if err != nil { + return "", err } - if svcId != "" { - sdc.serviceIdCache.Add(cacheKey, svcId, defaultServiceIdCacheTTL) + if nsId == "" { + return "", nil } - return svcId, nil -} - -func (sdc *serviceDiscoveryClient) getServiceIdFromCloudMap(ctx context.Context, nsName string, svcName string) (string, error) { - svcs, err := sdc.listServicesFromCloudMap(ctx, nsName) + svcs, err := sdc.sdApi.ListServices(ctx, nsId) if err != nil { return "", err } for _, svc := range svcs { - if svcName == aws.ToString(svc.Name) { - return aws.ToString(svc.Id), nil + sdc.cacheServiceId(nsName, svcName, svc.Id) + if svc.Name == svcName { + svcId = svc.Id } } - return "", nil + return svcId, nil } -func (sdc *serviceDiscoveryClient) listServicesFromCloudMap(ctx context.Context, nsName string) ([]*types.ServiceSummary, error) { - svcs := make([]*types.ServiceSummary, 0) +func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName string) (nsId string, err error) { + sdc.log.Info("creating a new namespace", "namespace", nsName) + opId, err := sdc.sdApi.CreateHttpNamespace(ctx, nsName) - nsId, nsErr := sdc.getNamespaceId(ctx, nsName) - if nsErr != nil || nsId == "" { - return svcs, nil + if err != nil { + return "", err } - filter := types.ServiceFilter{ - Name: types.ServiceFilterNameNamespaceId, - Values: []string{nsId}, + nsId, err = sdc.sdApi.PollCreateNamespace(ctx, opId) + + if err != nil { + return "", err } - pages := sd.NewListServicesPaginator(sdc.sdApi, &sd.ListServicesInput{Filters: []types.ServiceFilter{filter}}) + if nsId == "" { + return "", fmt.Errorf("failed to create namespace") + } - for pages.HasMorePages() { - output, err := pages.NextPage(ctx) - if err != nil { - return svcs, err - } + sdc.cacheNamespaceId(nsName, nsId) - for _, svc := range output.Services { - svcs = append(svcs, &svc) + return nsId, nil +} - cacheKey := sdc.buildServiceIdCacheKey(nsName, aws.ToString(svc.Name)) - svcId := aws.ToString(svc.Id) - sdc.serviceIdCache.Add(cacheKey, svcId, defaultServiceIdCacheTTL) - } - } +func (sdc *serviceDiscoveryClient) cacheNamespaceId(nsName string, nsId string) { + sdc.namespaceIdCache.Add(nsName, nsId, defaultNamespaceIdCacheTTL) +} - return svcs, nil +func (sdc *serviceDiscoveryClient) cacheServiceId(nsName string, svcName string, svcId string) { + cacheKey := sdc.buildServiceIdCacheKey(nsName, svcName) + sdc.serviceIdCache.Add(cacheKey, svcId, defaultServiceIdCacheTTL) +} + +func (sdc *serviceDiscoveryClient) cacheEndpoints(svcId string, endpts []*model.Endpoint) { + sdc.endpointCache.Add(svcId, endpts, defaultEndpointsCacheTTL) +} + +func (sdc *serviceDiscoveryClient) evictEndpoints(svcId string) { + sdc.endpointCache.Remove(svcId) } -func (sdc *serviceDiscoveryClient) buildServiceIdCacheKey(nsName string, svcName string) string { +func (sdc *serviceDiscoveryClient) buildServiceIdCacheKey(nsName string, svcName string) (cacheKey string) { return fmt.Sprintf("%s/%s", nsName, svcName) } diff --git a/pkg/cloudmap/endpoint_manager.go b/pkg/cloudmap/endpoint_manager.go deleted file mode 100644 index 27398403..00000000 --- a/pkg/cloudmap/endpoint_manager.go +++ /dev/null @@ -1,291 +0,0 @@ -package cloudmap - -import ( - "context" - "errors" - "fmt" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" - "github.com/aws/aws-sdk-go-v2/aws" - sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" - "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/util/cache" - "k8s.io/apimachinery/pkg/util/wait" - ctrl "sigs.k8s.io/controller-runtime" - "strconv" - "time" -) - -const ( - defaultEndpointsCacheTTL = 5 * time.Second - defaultEndpointsCacheSize = 1024 - - defaultOperationPollInterval = 3 * time.Second -) - -type EndpointManager interface { - ListEndpoints(ctx context.Context, serviceId string) ([]*model.Endpoint, error) - - RegisterEndpoints(ctx context.Context, service *model.Service, serviceId string) error - - DeregisterEndpoints(ctx context.Context, service *model.Service, serviceId string) error -} - -type defaultEndpointManager struct { - log logr.Logger - sdApi *sd.Client - endpointCache *cache.LRUExpireCache - endpointCacheTTL time.Duration - - // interval between each getOperation call - operationPollInterval time.Duration - // maximum retries per getOperation call - operationPollMaxRetries int -} - -func NewEndpointManager(cfg *aws.Config) EndpointManager { - return &defaultEndpointManager{ - log: ctrl.Log.WithName("cloudmap"), - sdApi: sd.NewFromConfig(*cfg), - endpointCache: cache.NewLRUExpireCache(defaultEndpointsCacheSize), - endpointCacheTTL: defaultEndpointsCacheTTL, - operationPollInterval: defaultOperationPollInterval, - } -} - -func (mgr *defaultEndpointManager) ListEndpoints(ctx context.Context, serviceId string) ([]*model.Endpoint, error) { - - if cachedValue, exists := mgr.endpointCache.Get(serviceId); exists { - return cachedValue.([]*model.Endpoint), nil - } - - endpts, endptsErr := mgr.listEndpointsFromCloudMap(ctx, serviceId) - - if endptsErr != nil { - return nil, endptsErr - } - - mgr.endpointCache.Add(serviceId, endpts, defaultEndpointsCacheTTL) - - return endpts, nil -} - -func (mgr *defaultEndpointManager) listEndpointsFromCloudMap(ctx context.Context, svcId string) ([]*model.Endpoint, error) { - endpts := make([]*model.Endpoint, 0) - - pages := sd.NewListInstancesPaginator(mgr.sdApi, &sd.ListInstancesInput{ServiceId: &svcId}) - - for pages.HasMorePages() { - output, err := pages.NextPage(ctx) - if err != nil { - return endpts, err - } - - for _, inst := range output.Instances { - endpt, endptErr := model.NewEndpointFromInstance(&inst) - - if endptErr != nil { - mgr.log.Info(fmt.Sprintf("skipping instance %s to endpoint conversion: %s", *inst.Id, endptErr.Error())) - continue - } - - endpts = append(endpts, endpt) - } - } - - return endpts, nil -} - -type opResult struct { - instId string - opId string - err error -} - -func (mgr *defaultEndpointManager) RegisterEndpoints(ctx context.Context, service *model.Service, serviceId string) error { - opChan := make(chan opResult) - startTime := mgr.now() - - for _, endpt := range service.Endpoints { - go mgr.registerInstanceInCloudMap(ctx, serviceId, endpt.Id, endpt.GetAttributes(), opChan) - } - - ops, regSuccess := mgr.getOpsList(len(service.Endpoints), opChan) - opsErr := mgr.pollOperations(ctx, types.OperationTypeRegisterInstance, serviceId, startTime, ops) - - // Evict cache entry so next list call reflects changes - mgr.endpointCache.Remove(serviceId) - - if opsErr != nil { - return opsErr - } - - if !regSuccess { - return errors.New("failure registering endpoints") - } - - return nil -} - -func (mgr *defaultEndpointManager) registerInstanceInCloudMap(ctx context.Context, svcId string, instId string, instAttrs map[string]string, opChan chan opResult) { - - regResp, err := mgr.sdApi.RegisterInstance(ctx, &sd.RegisterInstanceInput{ - Attributes: instAttrs, - InstanceId: &instId, - ServiceId: &svcId, - }) - - opChan <- opResult{instId, aws.ToString(regResp.OperationId), err} -} - -func (mgr *defaultEndpointManager) DeregisterEndpoints(ctx context.Context, service *model.Service, serviceId string) error { - opChan := make(chan opResult) - startTime := mgr.now() - - for _, endpt := range service.Endpoints { - go mgr.deregisterInstanceInCloudMap(ctx, serviceId, endpt.Id, opChan) - } - - ops, deregSuccess := mgr.getOpsList(len(service.Endpoints), opChan) - - opsErr := mgr.pollOperations(ctx, types.OperationTypeDeregisterInstance, serviceId, startTime, ops) - - // Evict cache entry so next list call reflects changes - mgr.endpointCache.Remove(serviceId) - - if opsErr != nil { - return opsErr - } - - if !deregSuccess { - return errors.New("failure de-registering endpoints") - } - - return nil -} - -func (mgr *defaultEndpointManager) deregisterInstanceInCloudMap(ctx context.Context, svcId string, instId string, opChan chan opResult) { - deregResp, err := mgr.sdApi.DeregisterInstance(ctx, &sd.DeregisterInstanceInput{ - InstanceId: &instId, - ServiceId: &svcId, - }) - - if err != nil { - opChan <- opResult{instId, "", err} - } - - opChan <- opResult{instId, *deregResp.OperationId, nil} -} - -func (mgr *defaultEndpointManager) getOpsList(opCount int, opChan chan opResult) ([]string, bool) { - success := true - - ops := make([]string, 0) - - for i := 0; i < opCount; i++ { - op := <-opChan - - if op.err != nil { - mgr.log.Info("could not create operation", "error", op.err) - success = false - continue - } - - ops = append(ops, op.opId) - } - - return ops, success -} - -func (mgr *defaultEndpointManager) pollOperations(ctx context.Context, opType types.OperationType, svcId string, startTime int64, ops []string) error { - - if len(ops) == 0 { - mgr.log.Info("no operations to poll") - } - - svcFilter := types.OperationFilter{ - Name: types.OperationFilterNameServiceId, - Values: []string{svcId}, - } - statusFilter := types.OperationFilter{ - Name: types.OperationFilterNameStatus, - Condition: types.FilterConditionIn, - - Values: []string{ - string(types.OperationStatusFail), - string(types.OperationStatusSuccess)}, - } - typeFilter := types.OperationFilter{ - Name: types.OperationFilterNameType, - Values: []string{string(opType)}, - } - - timeFilter := types.OperationFilter{ - Name: types.OperationFilterNameUpdateDate, - Condition: types.FilterConditionBetween, - Values: []string{ - strconv.Itoa(int(startTime)), - // Add one minute to end range in case op updates while list request is in flight - strconv.Itoa(int(mgr.now() + 60000)), - }, - } - - return wait.PollUntil(mgr.operationPollInterval, func() (bool, error) { - mgr.log.Info("polling operations", "operations", ops) - completed := 0 - failedOps := make([]string, 0) - - pages := sd.NewListOperationsPaginator(mgr.sdApi, &sd.ListOperationsInput{ - Filters: []types.OperationFilter{svcFilter, statusFilter, typeFilter, timeFilter}, - }) - - for pages.HasMorePages() { - output, err := pages.NextPage(ctx) - - if err != nil { - return true, err - } - - for _, pollOp := range ops { - for _, sdOp := range output.Operations { - if pollOp == aws.ToString(sdOp.Id) { - completed++ - if sdOp.Status == types.OperationStatusFail { - failedOps = append(failedOps, pollOp) - } - } - } - } - } - - if completed != len(ops) { - return false, nil - } - - if len(failedOps) != 0 { - for _, failedOp := range failedOps { - mgr.log.Info("Operation failed", "failedOp", failedOp, "reason", mgr.getFailedOpReason(ctx, failedOp)) - } - return true, fmt.Errorf("operation failure") - } - - mgr.log.Info("operations completed successfully") - return true, nil - }, ctx.Done()) -} - -// getFailedOpReason returns operation error message, which is not available in ListOperations response -func (mgr *defaultEndpointManager) getFailedOpReason(ctx context.Context, op string) string { - opResp, err := mgr.sdApi.GetOperation(ctx, &sd.GetOperationInput{OperationId: &op}) - - if err != nil { - return "failed to retrieve operation failure reason" - } - - return aws.ToString(opResp.Operation.ErrorMessage) -} - -// now returns current time with milliseconds, as used by operation UPDATE_DATE field -func (mgr *defaultEndpointManager) now() int64 { - return time.Now().UnixNano() / 1000000 -} diff --git a/pkg/cloudmap/operation_collector.go b/pkg/cloudmap/operation_collector.go new file mode 100644 index 00000000..e33a6833 --- /dev/null +++ b/pkg/cloudmap/operation_collector.go @@ -0,0 +1,66 @@ +package cloudmap + +import ( + "github.com/go-logr/logr" + ctrl "sigs.k8s.io/controller-runtime" +) + +// OperationCollector collects a list operations with thread safety. +type OperationCollector interface { + // Add an operation to poll with thread safety, to be called from within a go routine. + Add(endpointId string, operationId string, operationError error) + + // Collect waits for all operations to be added and returns a list of successfully created operation IDs. + Collect() []string + + // IsAllOperationsCreated returns true if all operations were created successfully. + IsAllOperationsCreated() bool +} + +type opCollector struct { + log logr.Logger + opChan chan opResult + opCount int + createOpsSuccess bool +} + +type opResult struct { + instId string + opId string + err error +} + +func NewOperationCollector(opCount int) OperationCollector { + return &opCollector{ + log: ctrl.Log.WithName("cloudmap"), + opChan: make(chan opResult), + opCount: opCount, + createOpsSuccess: true, + } +} + +func (opColl *opCollector) Add(endptId string, opId string, opErr error) { + opColl.opChan <- opResult{endptId, opId, opErr} +} + +func (opColl *opCollector) Collect() []string { + opIds := make([]string, 0) + + for i := 0; i < opColl.opCount; i++ { + op := <-opColl.opChan + + if op.err != nil { + opColl.log.Info("could not create operation", "error", op.err) + opColl.createOpsSuccess = false + continue + } + + opIds = append(opIds, op.opId) + } + + return opIds +} + +func (opColl *opCollector) IsAllOperationsCreated() bool { + return opColl.createOpsSuccess +} diff --git a/pkg/cloudmap/operation_poller.go b/pkg/cloudmap/operation_poller.go new file mode 100644 index 00000000..488f7136 --- /dev/null +++ b/pkg/cloudmap/operation_poller.go @@ -0,0 +1,150 @@ +package cloudmap + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/wait" + ctrl "sigs.k8s.io/controller-runtime" + "strconv" + "time" +) + +const ( + // Interval between each getOperation call. + defaultOperationPollInterval = 3 * time.Second + + // Time until we stop polling the operation + defaultOperationPollTimeout = 5 * time.Minute +) + +// OperationPoller polls a list operations for a terminal status. +type OperationPoller interface { + // Poll monitors operations until they reach terminal state. + Poll(ctx context.Context) error +} + +type operationPoller struct { + log logr.Logger + sdApi ServiceDiscoveryApi + opIds []string + + svcId string + opType types.OperationType + start int +} + +func newOperationPoller(sdApi ServiceDiscoveryApi, svcId string, opIds []string, startTime int) operationPoller { + return operationPoller{ + log: ctrl.Log.WithName("cloudmap"), + sdApi: sdApi, + + opIds: opIds, + svcId: svcId, + start: startTime, + } +} + +// NewRegisterInstancePoller creates a new operation poller for register instance operations. +func NewRegisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int) OperationPoller { + poller := newOperationPoller(sdApi, serviceId, opIds, startTime) + poller.opType = types.OperationTypeRegisterInstance + return &poller +} + +// NewDeregisterInstancePoller creates a new operation poller for de-register instance operations. +func NewDeregisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int) OperationPoller { + poller := newOperationPoller(sdApi, serviceId, opIds, startTime) + poller.opType = types.OperationTypeDeregisterInstance + return &poller +} + +func (opPoller *operationPoller) Poll(ctx context.Context) (err error) { + if len(opPoller.opIds) == 0 { + opPoller.log.Info("no operations to poll") + return nil + } + + return wait.Poll(defaultOperationPollInterval, defaultOperationPollTimeout, func() (done bool, err error) { + opPoller.log.Info("polling operations", "operations", opPoller.opIds) + + sdOps, err := opPoller.sdApi.ListOperations(ctx, opPoller.buildFilters()) + + if err != nil { + return true, err + } + + failedOps := make([]string, 0) + + for _, pollOp := range opPoller.opIds { + status, hasVal := sdOps[pollOp] + if !hasVal { + // polled operation not terminal + return false, nil + } + + if status == types.OperationStatusFail { + failedOps = append(failedOps, pollOp) + } + } + + if len(failedOps) != 0 { + for _, failedOp := range failedOps { + opPoller.log.Info("Operation failed", "failedOp", failedOp, "reason", opPoller.getFailedOpReason(ctx, failedOp)) + } + return true, fmt.Errorf("operation failure") + } + + opPoller.log.Info("operations completed successfully") + return true, nil + }) +} + +func (opPoller *operationPoller) buildFilters() []types.OperationFilter { + svcFilter := types.OperationFilter{ + Name: types.OperationFilterNameServiceId, + Values: []string{opPoller.svcId}, + } + statusFilter := types.OperationFilter{ + Name: types.OperationFilterNameStatus, + Condition: types.FilterConditionIn, + + Values: []string{ + string(types.OperationStatusFail), + string(types.OperationStatusSuccess)}, + } + typeFilter := types.OperationFilter{ + Name: types.OperationFilterNameType, + Values: []string{string(opPoller.opType)}, + } + + timeFilter := types.OperationFilter{ + Name: types.OperationFilterNameUpdateDate, + Condition: types.FilterConditionBetween, + Values: []string{ + strconv.Itoa(opPoller.start), + // Add one minute to end range in case op updates while list request is in flight + strconv.Itoa(Now() + 60000), + }, + } + + return []types.OperationFilter{svcFilter, statusFilter, typeFilter, timeFilter} +} + +// getFailedOpReason returns operation error message, which is not available in ListOperations response +func (opPoller *operationPoller) getFailedOpReason(ctx context.Context, opId string) string { + op, err := opPoller.sdApi.GetOperation(ctx, opId) + + if err != nil { + return "failed to retrieve operation failure reason" + } + + return aws.ToString(op.ErrorMessage) +} + +// Now returns current time with milliseconds, as used by operation UPDATE_DATE field +func Now() int { + return int(time.Now().UnixNano() / 1000000) +} diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index 177050f5..89358de5 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -2,7 +2,6 @@ package controllers import ( "context" - "fmt" cloudmapmock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" @@ -37,7 +36,6 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { } cloudmapMock := cloudmapmock.NewMockServiceDiscoveryClient(mockController) - fmt.Printf("test output") // expected interactions with the Cloud Map client cloudmapMock.EXPECT().GetService(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) cloudmapMock.EXPECT().CreateService(gomock.Any(), gomock.Any()).Return(nil).Times(1) From 5c7b0335a1567d1e24f3d03d688e13732419c191 Mon Sep 17 00:00:00 2001 From: Jaromir Vanek Date: Fri, 15 Oct 2021 20:40:32 -0700 Subject: [PATCH 011/163] Add unit tests for model package (#38) --- pkg/cloudmap/client.go | 2 +- pkg/model/types.go | 5 +- pkg/model/types_test.go | 140 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 6 deletions(-) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 7de78271..d74dfa55 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -179,7 +179,7 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, servic for _, endpt := range service.Endpoints { go func(endpt *model.Endpoint) { - opId, endptErr := sdc.sdApi.RegisterInstance(ctx, svcId, endpt.Id, endpt.GetAttributes()) + opId, endptErr := sdc.sdApi.RegisterInstance(ctx, svcId, endpt.Id, endpt.GetCloudMapAttributes()) opCollector.Add(endpt.Id, opId, endptErr) }(endpt) } diff --git a/pkg/model/types.go b/pkg/model/types.go index 41d21698..558c0400 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -62,8 +62,8 @@ func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { return &endpoint, nil } -// GetAttributes extracts endpoint attributes for Cloud Map service instance registration -func (e *Endpoint) GetAttributes() map[string]string { +// GetCloudMapAttributes extracts endpoint attributes for Cloud Map service instance registration +func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs := make(map[string]string, 0) attrs[Ipv4Attr] = e.IP @@ -78,6 +78,7 @@ func (e *Endpoint) GetAttributes() map[string]string { return attrs } +// Equals evaluates if two Endpoints are "deeply equal" (including all fields) func (e *Endpoint) Equals(other *Endpoint) bool { return reflect.DeepEqual(e, other) } diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index c25e3b8a..1fa6a35d 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -7,6 +7,7 @@ import ( ) var instId = "my-instance" +var ip = "192.168.0.1" func TestNewEndpointFromInstance(t *testing.T) { tests := []struct { @@ -20,14 +21,14 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.InstanceSummary{ Id: &instId, Attributes: map[string]string{ - Ipv4Attr: "192.168.0.1", + Ipv4Attr: ip, PortAttr: "65535", "custom-attr": "custom-val", }, }, want: &Endpoint{ Id: instId, - IP: "192.168.0.1", + IP: ip, Port: 65535, Attributes: map[string]string{ "custom-attr": "custom-val", @@ -39,13 +40,35 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.InstanceSummary{ Id: &instId, Attributes: map[string]string{ - Ipv4Attr: "192.168.0.1", + Ipv4Attr: ip, PortAttr: "99999", "custom-attr": "custom-val", }, }, wantErr: true, }, + { + name: "missing IP", + inst: &types.InstanceSummary{ + Id: &instId, + Attributes: map[string]string{ + PortAttr: "80", + "custom-attr": "custom-val", + }, + }, + wantErr: true, + }, + { + name: "missing port", + inst: &types.InstanceSummary{ + Id: &instId, + Attributes: map[string]string{ + Ipv4Attr: ip, + "custom-attr": "custom-val", + }, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -60,3 +83,114 @@ func TestNewEndpointFromInstance(t *testing.T) { }) } } + +func TestEndpoint_GetAttributes(t *testing.T) { + type fields struct { + Id string + IP string + Port int32 + Attributes map[string]string + } + tests := []struct { + name string + fields fields + want map[string]string + }{ + { + name: "happy case", + fields: fields{ + IP: ip, + Port: 30, + Attributes: map[string]string{ + "custom-attr": "custom-val", + }, + }, + want: map[string]string{ + Ipv4Attr: ip, + PortAttr: "30", + "custom-attr": "custom-val", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &Endpoint{ + Id: tt.fields.Id, + IP: tt.fields.IP, + Port: tt.fields.Port, + Attributes: tt.fields.Attributes, + } + if got := e.GetCloudMapAttributes(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAttributes() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEndpointIdFromIPAddress(t *testing.T) { + tests := []struct { + name string + address string + want string + }{ + { + name: "happy case", + address: "192.168.0.1", + want: "192_168_0_1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EndpointIdFromIPAddress(tt.address); got != tt.want { + t.Errorf("EndpointIdFromIPAddress() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEndpoint_Equals(t *testing.T) { + firstEndpoint := Endpoint{ + Id: instId, + IP: ip, + Port: 80, + Attributes: map[string]string{ + "custom-key": "custom-val", + }, + } + + secondEndpoint := Endpoint{ + Id: instId, + IP: ip, + Port: 80, + Attributes: map[string]string{ + "custom-key": "different-val", + }, + } + + tests := []struct { + name string + x Endpoint + y Endpoint + want bool + }{ + { + name: "identical", + x: firstEndpoint, + y: firstEndpoint, + want: true, + }, + { + name: "different", + x: firstEndpoint, + y: secondEndpoint, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.x.Equals(&tt.y); got != tt.want { + t.Errorf("Equals() = %v, want %v", got, tt.want) + } + }) + } +} From bdcf3d027bf13b5453fdffad2ff96a5093ab8da6 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Mon, 18 Oct 2021 11:59:02 -0700 Subject: [PATCH 012/163] Improve operation collector logic (#39) Encapsulate go routine management during operation collection. Leverage wait groups and channel lifecycles to simplify collector interface. Track start time in collector to simplify state management for operation poller. Add unit tests for operation collector. --- pkg/cloudmap/api.go | 1 - pkg/cloudmap/client.go | 24 +++++------ pkg/cloudmap/operation_collector.go | 46 ++++++++++++++------- pkg/cloudmap/operation_collector_test.go | 51 ++++++++++++++++++++++++ pkg/cloudmap/operation_poller.go | 21 +++++----- 5 files changed, 105 insertions(+), 38 deletions(-) create mode 100644 pkg/cloudmap/operation_collector_test.go diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index eacd146a..d056d4c4 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -93,7 +93,6 @@ func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) Name: types.ServiceFilterNameNamespaceId, Values: []string{nsId}, } - sdApi.log.Info("paginating", "nsId", nsId) pages := sd.NewListServicesPaginator(sdApi.awsFacade, &sd.ListServicesInput{Filters: []types.ServiceFilter{filter}}) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index d74dfa55..49a3e68f 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -174,17 +174,15 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, servic return err } - startTime := Now() - opCollector := NewOperationCollector(len(service.Endpoints)) + opCollector := NewOperationCollector() for _, endpt := range service.Endpoints { - go func(endpt *model.Endpoint) { - opId, endptErr := sdc.sdApi.RegisterInstance(ctx, svcId, endpt.Id, endpt.GetCloudMapAttributes()) - opCollector.Add(endpt.Id, opId, endptErr) - }(endpt) + opCollector.Add(func() (opId string, err error) { + return sdc.sdApi.RegisterInstance(ctx, svcId, endpt.Id, endpt.GetCloudMapAttributes()) + }) } - err = NewRegisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), startTime).Poll(ctx) + err = NewRegisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) // Evict cache entry so next list call reflects changes sdc.evictEndpoints(svcId) @@ -214,17 +212,15 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, service return err } - startTime := Now() - opCollector := NewOperationCollector(len(service.Endpoints)) + opCollector := NewOperationCollector() for _, endpt := range service.Endpoints { - go func(endpt *model.Endpoint) { - opId, endptErr := sdc.sdApi.DeregisterInstance(ctx, svcId, endpt.Id) - opCollector.Add(endpt.Id, opId, endptErr) - }(endpt) + opCollector.Add(func() (opId string, err error) { + return sdc.sdApi.DeregisterInstance(ctx, svcId, endpt.Id) + }) } - err = NewDeregisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), startTime).Poll(ctx) + err = NewDeregisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) // Evict cache entry so next list call reflects changes sdc.evictEndpoints(svcId) diff --git a/pkg/cloudmap/operation_collector.go b/pkg/cloudmap/operation_collector.go index e33a6833..e38c5257 100644 --- a/pkg/cloudmap/operation_collector.go +++ b/pkg/cloudmap/operation_collector.go @@ -3,16 +3,20 @@ package cloudmap import ( "github.com/go-logr/logr" ctrl "sigs.k8s.io/controller-runtime" + "sync" ) -// OperationCollector collects a list operations with thread safety. +// OperationCollector collects a list of operation IDs asynchronously with thread safety. type OperationCollector interface { - // Add an operation to poll with thread safety, to be called from within a go routine. - Add(endpointId string, operationId string, operationError error) + // Add calls an operation provider function to asynchronously collect operations to poll. + Add(operationProvider func() (operationId string, err error)) - // Collect waits for all operations to be added and returns a list of successfully created operation IDs. + // Collect waits for all create operation results to be provided and returns a list of the successfully created operation IDs. Collect() []string + // GetStartTime returns the start time range to poll the collected operations. + GetStartTime() int64 + // IsAllOperationsCreated returns true if all operations were created successfully. IsAllOperationsCreated() bool } @@ -20,35 +24,45 @@ type OperationCollector interface { type opCollector struct { log logr.Logger opChan chan opResult - opCount int + wg sync.WaitGroup + startTime int64 createOpsSuccess bool } type opResult struct { - instId string - opId string - err error + opId string + err error } -func NewOperationCollector(opCount int) OperationCollector { +func NewOperationCollector() OperationCollector { return &opCollector{ log: ctrl.Log.WithName("cloudmap"), opChan: make(chan opResult), - opCount: opCount, + startTime: Now(), createOpsSuccess: true, } } -func (opColl *opCollector) Add(endptId string, opId string, opErr error) { - opColl.opChan <- opResult{endptId, opId, opErr} +func (opColl *opCollector) Add(opProvider func() (opId string, err error)) { + opColl.wg.Add(1) + go func() { + defer opColl.wg.Done() + + opId, opErr := opProvider() + opColl.opChan <- opResult{opId, opErr} + }() } func (opColl *opCollector) Collect() []string { opIds := make([]string, 0) - for i := 0; i < opColl.opCount; i++ { - op := <-opColl.opChan + // Run wait in separate go routine to unblock reading from the channel. + go func() { + opColl.wg.Wait() + close(opColl.opChan) + }() + for op := range opColl.opChan { if op.err != nil { opColl.log.Info("could not create operation", "error", op.err) opColl.createOpsSuccess = false @@ -61,6 +75,10 @@ func (opColl *opCollector) Collect() []string { return opIds } +func (opColl *opCollector) GetStartTime() int64 { + return opColl.startTime +} + func (opColl *opCollector) IsAllOperationsCreated() bool { return opColl.createOpsSuccess } diff --git a/pkg/cloudmap/operation_collector_test.go b/pkg/cloudmap/operation_collector_test.go new file mode 100644 index 00000000..1b8d6bab --- /dev/null +++ b/pkg/cloudmap/operation_collector_test.go @@ -0,0 +1,51 @@ +package cloudmap + +import ( + "errors" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestOpCollector_HappyCase(t *testing.T) { + oc := NewOperationCollector() + oc.Add(func() (opId string, err error) { return "one", nil }) + oc.Add(func() (opId string, err error) { return "two", nil }) + + result := oc.Collect() + assert.True(t, oc.IsAllOperationsCreated()) + assert.Equal(t, 2, len(result)) + assert.Contains(t, result, "one") + assert.Contains(t, result, "two") +} + +func TestOpCollector_AllFail(t *testing.T) { + oc := NewOperationCollector() + oc.Add(func() (opId string, err error) { return "one", errors.New("fail one") }) + oc.Add(func() (opId string, err error) { return "two", errors.New("fail two") }) + + result := oc.Collect() + assert.False(t, oc.IsAllOperationsCreated()) + assert.Equal(t, 0, len(result)) +} + +func TestOpCollector_MixedSuccess(t *testing.T) { + oc := NewOperationCollector() + oc.Add(func() (opId string, err error) { return "one", errors.New("fail one") }) + oc.Add(func() (opId string, err error) { return "two", nil }) + + result := oc.Collect() + assert.False(t, oc.IsAllOperationsCreated()) + assert.Equal(t, []string{"two"}, result) +} + +func TestOpCollector_GetStartTime(t *testing.T) { + oc1 := NewOperationCollector() + time.Sleep(time.Second) + oc2 := NewOperationCollector() + + assert.Equal(t, oc1.GetStartTime(), oc1.GetStartTime(), "Start time should not change") + assert.NotEqual(t, oc1.GetStartTime(), oc2.GetStartTime(), "Start time should reflect instantiation") + assert.Less(t, oc1.GetStartTime(), oc2.GetStartTime(), + "Start time should increase for later instantiations") +} diff --git a/pkg/cloudmap/operation_poller.go b/pkg/cloudmap/operation_poller.go index 488f7136..73ce3db1 100644 --- a/pkg/cloudmap/operation_poller.go +++ b/pkg/cloudmap/operation_poller.go @@ -33,10 +33,10 @@ type operationPoller struct { svcId string opType types.OperationType - start int + start int64 } -func newOperationPoller(sdApi ServiceDiscoveryApi, svcId string, opIds []string, startTime int) operationPoller { +func newOperationPoller(sdApi ServiceDiscoveryApi, svcId string, opIds []string, startTime int64) operationPoller { return operationPoller{ log: ctrl.Log.WithName("cloudmap"), sdApi: sdApi, @@ -48,14 +48,14 @@ func newOperationPoller(sdApi ServiceDiscoveryApi, svcId string, opIds []string, } // NewRegisterInstancePoller creates a new operation poller for register instance operations. -func NewRegisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int) OperationPoller { +func NewRegisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int64) OperationPoller { poller := newOperationPoller(sdApi, serviceId, opIds, startTime) poller.opType = types.OperationTypeRegisterInstance return &poller } // NewDeregisterInstancePoller creates a new operation poller for de-register instance operations. -func NewDeregisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int) OperationPoller { +func NewDeregisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int64) OperationPoller { poller := newOperationPoller(sdApi, serviceId, opIds, startTime) poller.opType = types.OperationTypeDeregisterInstance return &poller @@ -124,9 +124,9 @@ func (opPoller *operationPoller) buildFilters() []types.OperationFilter { Name: types.OperationFilterNameUpdateDate, Condition: types.FilterConditionBetween, Values: []string{ - strconv.Itoa(opPoller.start), + Itoa(opPoller.start), // Add one minute to end range in case op updates while list request is in flight - strconv.Itoa(Now() + 60000), + Itoa(Now() + 60000), }, } @@ -143,8 +143,11 @@ func (opPoller *operationPoller) getFailedOpReason(ctx context.Context, opId str return aws.ToString(op.ErrorMessage) } +func Itoa(i int64) string { + return strconv.FormatInt(i, 10) +} -// Now returns current time with milliseconds, as used by operation UPDATE_DATE field -func Now() int { - return int(time.Now().UnixNano() / 1000000) +// Now returns current time with milliseconds, as used by operation filter UPDATE_DATE field +func Now() int64 { + return time.Now().UnixNano() / 1000000 } From 0ad6491c424e259bcea0c43c7150788fe6bcb24e Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Mon, 18 Oct 2021 15:11:16 -0700 Subject: [PATCH 013/163] Add operation poller unit tests (#40) --- pkg/cloudmap/api.go | 37 +++-- pkg/cloudmap/operation_poller.go | 28 +++- pkg/cloudmap/operation_poller_test.go | 229 ++++++++++++++++++++++++++ pkg/model/types.go | 6 + 4 files changed, 273 insertions(+), 27 deletions(-) create mode 100644 pkg/cloudmap/operation_poller_test.go diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index d056d4c4..f529c99b 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -2,6 +2,7 @@ package cloudmap import ( "context" + "errors" "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" @@ -16,10 +17,10 @@ import ( // internal data structures. It manages all interactions with the AWS SDK. type ServiceDiscoveryApi interface { // ListNamespaces returns a list of all namespaces. - ListNamespaces(ctx context.Context) (namespaces []*Resource, err error) + ListNamespaces(ctx context.Context) (namespaces []*model.Resource, err error) // ListServices returns a list of services for a given namespace. - ListServices(ctx context.Context, namespaceId string) (services []*Resource, err error) + ListServices(ctx context.Context, namespaceId string) (services []*model.Resource, err error) // ListInstances returns a list of service instances registered to a given service. ListInstances(ctx context.Context, serviceId string) ([]*model.Endpoint, error) @@ -51,12 +52,6 @@ type serviceDiscoveryApi struct { awsFacade AwsFacade } -// Resource encapsulates a ID/name pair -type Resource struct { - Id string - Name string -} - // NewServiceDiscoveryApiFromConfig creates a new AWS Cloud Map API connection manager from an AWS client config. func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { return &serviceDiscoveryApi{ @@ -65,8 +60,8 @@ func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { } } -func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*Resource, error) { - namespaces := make([]*Resource, 0) +func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*model.Resource, error) { + namespaces := make([]*model.Resource, 0) pages := sd.NewListNamespacesPaginator(sdApi.awsFacade, &sd.ListNamespacesInput{}) for pages.HasMorePages() { @@ -76,7 +71,7 @@ func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*Resour } for _, ns := range output.Namespaces { - namespaces = append(namespaces, &Resource{ + namespaces = append(namespaces, &model.Resource{ Id: aws.ToString(ns.Id), Name: aws.ToString(ns.Name), }) @@ -86,8 +81,8 @@ func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*Resour return namespaces, nil } -func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) ([]*Resource, error) { - svcs := make([]*Resource, 0) +func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) ([]*model.Resource, error) { + svcs := make([]*model.Resource, 0) filter := types.ServiceFilter{ Name: types.ServiceFilterNameNamespaceId, @@ -103,7 +98,7 @@ func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) } for _, svc := range output.Services { - svcs = append(svcs, &Resource{ + svcs = append(svcs, &model.Resource{ Id: aws.ToString(svc.Id), Name: aws.ToString(svc.Name), }) @@ -226,12 +221,12 @@ func (sdApi *serviceDiscoveryApi) DeregisterInstance(ctx context.Context, svcId } func (sdApi *serviceDiscoveryApi) PollCreateNamespace(ctx context.Context, opId string) (nsId string, err error) { - return nsId, wait.Poll(defaultOperationPollInterval, defaultOperationPollTimeout, func() (done bool, pollErr error) { + err = wait.Poll(defaultOperationPollInterval, defaultOperationPollTimeout, func() (done bool, err error) { sdApi.log.Info("polling operation", "opId", opId) - op, opErr := sdApi.GetOperation(ctx, opId) + op, err := sdApi.GetOperation(ctx, opId) - if opErr != nil { - return true, opErr + if err != nil { + return true, err } if op.Status == types.OperationStatusFail { @@ -246,4 +241,10 @@ func (sdApi *serviceDiscoveryApi) PollCreateNamespace(ctx context.Context, opId return false, nil }) + + if err == wait.ErrWaitTimeout { + err = errors.New(operationPollTimoutErrorMessage) + } + + return nsId, err } diff --git a/pkg/cloudmap/operation_poller.go b/pkg/cloudmap/operation_poller.go index 73ce3db1..61467821 100644 --- a/pkg/cloudmap/operation_poller.go +++ b/pkg/cloudmap/operation_poller.go @@ -2,7 +2,7 @@ package cloudmap import ( "context" - "fmt" + "errors" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "github.com/go-logr/logr" @@ -18,6 +18,8 @@ const ( // Time until we stop polling the operation defaultOperationPollTimeout = 5 * time.Minute + + operationPollTimoutErrorMessage = "timed out while polling operations" ) // OperationPoller polls a list operations for a terminal status. @@ -27,10 +29,11 @@ type OperationPoller interface { } type operationPoller struct { - log logr.Logger - sdApi ServiceDiscoveryApi - opIds []string + log logr.Logger + sdApi ServiceDiscoveryApi + timeout time.Duration + opIds []string svcId string opType types.OperationType start int64 @@ -38,8 +41,9 @@ type operationPoller struct { func newOperationPoller(sdApi ServiceDiscoveryApi, svcId string, opIds []string, startTime int64) operationPoller { return operationPoller{ - log: ctrl.Log.WithName("cloudmap"), - sdApi: sdApi, + log: ctrl.Log.WithName("cloudmap"), + sdApi: sdApi, + timeout: defaultOperationPollTimeout, opIds: opIds, svcId: svcId, @@ -67,7 +71,7 @@ func (opPoller *operationPoller) Poll(ctx context.Context) (err error) { return nil } - return wait.Poll(defaultOperationPollInterval, defaultOperationPollTimeout, func() (done bool, err error) { + err = wait.Poll(defaultOperationPollInterval, opPoller.timeout, func() (done bool, err error) { opPoller.log.Info("polling operations", "operations", opPoller.opIds) sdOps, err := opPoller.sdApi.ListOperations(ctx, opPoller.buildFilters()) @@ -92,14 +96,20 @@ func (opPoller *operationPoller) Poll(ctx context.Context) (err error) { if len(failedOps) != 0 { for _, failedOp := range failedOps { - opPoller.log.Info("Operation failed", "failedOp", failedOp, "reason", opPoller.getFailedOpReason(ctx, failedOp)) + opPoller.log.Info("operation failed", "failedOp", failedOp, "reason", opPoller.getFailedOpReason(ctx, failedOp)) } - return true, fmt.Errorf("operation failure") + return true, errors.New("operation failure") } opPoller.log.Info("operations completed successfully") return true, nil }) + + if err == wait.ErrWaitTimeout { + return errors.New(operationPollTimoutErrorMessage) + } + + return err } func (opPoller *operationPoller) buildFilters() []types.OperationFilter { diff --git a/pkg/cloudmap/operation_poller_test.go b/pkg/cloudmap/operation_poller_test.go new file mode 100644 index 00000000..45cf2926 --- /dev/null +++ b/pkg/cloudmap/operation_poller_test.go @@ -0,0 +1,229 @@ +package cloudmap + +import ( + "context" + "errors" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" + testing2 "github.com/go-logr/logr/testing" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "strconv" + "testing" + "time" +) + +const ( + svcId = "test-svc-id" + opId1 = "operation-id-1" + opId2 = "operation-id-2" + startTime = 1 +) + +func TestOperationPoller_HappyCases(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + pollerTypes := []struct { + constructor func() OperationPoller + expectedOpType types.OperationType + }{ + { + constructor: func() OperationPoller { + return NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + }, + expectedOpType: types.OperationTypeRegisterInstance, + }, + { + constructor: func() OperationPoller { + return NewDeregisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + }, + expectedOpType: types.OperationTypeDeregisterInstance, + }, + } + + for _, pollerType := range pollerTypes { + p := pollerType.constructor() + + var firstEnd int + + sdApi.EXPECT(). + ListOperations(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, filters []types.OperationFilter) (map[string]types.OperationStatus, error) { + assert.Contains(t, filters, + types.OperationFilter{ + Name: types.OperationFilterNameServiceId, + Values: []string{svcId}, + }) + assert.Contains(t, filters, + types.OperationFilter{ + Name: types.OperationFilterNameStatus, + Condition: types.FilterConditionIn, + + Values: []string{ + string(types.OperationStatusFail), + string(types.OperationStatusSuccess)}, + }) + assert.Contains(t, filters, + types.OperationFilter{ + Name: types.OperationFilterNameType, + Values: []string{string(pollerType.expectedOpType)}, + }) + + timeFilter := findUpdateDateFilter(t, filters) + assert.NotNil(t, timeFilter) + assert.Equal(t, types.FilterConditionBetween, timeFilter.Condition) + assert.Equal(t, 2, len(timeFilter.Values)) + + filterStart, _ := strconv.Atoi(timeFilter.Values[0]) + assert.Equal(t, startTime, filterStart) + + firstEnd, _ = strconv.Atoi(timeFilter.Values[1]) + + return map[string]types.OperationStatus{}, nil + }) + + sdApi.EXPECT(). + ListOperations(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, filters []types.OperationFilter) (map[string]types.OperationStatus, error) { + timeFilter := findUpdateDateFilter(t, filters) + secondEnd, _ := strconv.Atoi(timeFilter.Values[1]) + assert.Greater(t, secondEnd, firstEnd, + "Filter time frame for operations must increase between invocations of ListOperations") + + return map[string]types.OperationStatus{ + opId1: types.OperationStatusSuccess, + opId2: types.OperationStatusSuccess, + }, nil + }) + + err := p.Poll(context.TODO()) + + assert.Nil(t, err) + } +} + +func TestOperationPoller_PollEmpty(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + p := NewRegisterInstancePoller(sdApi, svcId, []string{}, startTime) + err := p.Poll(context.TODO()) + assert.Nil(t, err) +} + +func TestOperationPoller_PollFailure(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + p := NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + + pollErr := errors.New("error polling operations") + + sdApi.EXPECT(). + ListOperations(gomock.Any(), gomock.Any()). + Return(map[string]types.OperationStatus{}, pollErr) + + err := p.Poll(context.TODO()) + assert.Equal(t, pollErr, err) +} + +func TestOperationPoller_PollOpFailure(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + p := NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + + sdApi.EXPECT(). + ListOperations(gomock.Any(), gomock.Any()). + Return( + map[string]types.OperationStatus{ + opId1: types.OperationStatusSuccess, + opId2: types.OperationStatusFail, + }, nil) + + opErr := "operation failure message" + + sdApi.EXPECT(). + GetOperation(gomock.Any(), opId2). + Return(&types.Operation{ErrorMessage: &opErr}, nil) + + err := p.Poll(context.TODO()) + assert.Equal(t, "operation failure", err.Error()) +} + +func TestOperationPoller_PollOpFailureAndMessageFailure(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + p := NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + + sdApi.EXPECT(). + ListOperations(gomock.Any(), gomock.Any()). + Return( + map[string]types.OperationStatus{ + opId1: types.OperationStatusFail, + opId2: types.OperationStatusSuccess, + }, nil) + + sdApi.EXPECT(). + GetOperation(gomock.Any(), opId1). + Return(nil, errors.New("failed to retrieve operation failure reason")) + + err := p.Poll(context.TODO()) + assert.Equal(t, "operation failure", err.Error()) +} + +func TestOperationPoller_PollTimeout(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + p := operationPoller{ + log: testing2.TestLogger{T: t}, + sdApi: sdApi, + timeout: 2 * time.Millisecond, + opIds: []string{opId1, opId2}, + } + + sdApi.EXPECT(). + ListOperations(gomock.Any(), gomock.Any()). + Return( + map[string]types.OperationStatus{}, nil) + + err := p.Poll(context.TODO()) + assert.Equal(t, operationPollTimoutErrorMessage, err.Error()) +} + +func TestItoa(t *testing.T) { + assert.Equal(t, "7", Itoa(7)) +} + +func TestNow(t *testing.T) { + now1 := Now() + time.Sleep(time.Millisecond * 5) + now2 := Now() + assert.Greater(t, now2, now1) +} + +func findUpdateDateFilter(t *testing.T, filters []types.OperationFilter) *types.OperationFilter { + for _, filter := range filters { + if filter.Name == types.OperationFilterNameUpdateDate { + return &filter + } + } + + t.Errorf("Missing update date filter") + return nil +} diff --git a/pkg/model/types.go b/pkg/model/types.go index 558c0400..78ba392c 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -10,6 +10,12 @@ import ( "strings" ) +// Resource encapsulates a ID/name pair +type Resource struct { + Id string + Name string +} + type Service struct { Namespace string Name string From f29486f1b1c105d04e8b2459fd66ec17c5ed003b Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Tue, 19 Oct 2021 20:31:58 -0700 Subject: [PATCH 014/163] Update controller and k8s client versions (#42) --- go.mod | 11 +- go.sum | 332 ++++++++++++++++++++++++++++++++++----------------------- 2 files changed, 205 insertions(+), 138 deletions(-) diff --git a/go.mod b/go.mod index 91c55fce..be498bf5 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,12 @@ require ( github.com/golang/mock v1.6.0 github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 + github.com/stretchr/testify v1.6.1 gotest.tools v2.2.0+incompatible - k8s.io/api v0.19.2 - k8s.io/apimachinery v0.19.2 - k8s.io/client-go v0.19.2 - sigs.k8s.io/controller-runtime v0.7.2 + k8s.io/api v0.20.2 + k8s.io/apimachinery v0.20.2 + k8s.io/client-go v0.20.2 + sigs.k8s.io/controller-runtime v0.8.3 ) + +replace github.com/spf13/viper v1.4.0 => github.com/spf13/viper v1.8.0 diff --git a/go.sum b/go.sum index bcc8ffa2..0e7d1fae 100644 --- a/go.sum +++ b/go.sum @@ -5,49 +5,55 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.6 h1:5YWtOnckcudzIw8lPPBcWOnmIFWMtHci1ZWAZulMSx0= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1 h1:eVvIXUKiTgv++6YnWb42DUA1YL7qDugnKP0HljexdnQ= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag= github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= @@ -74,7 +80,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -86,7 +93,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -99,11 +106,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -117,14 +121,16 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -135,49 +141,13 @@ github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -187,31 +157,37 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= @@ -223,28 +199,49 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -260,6 +257,7 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -274,20 +272,25 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -316,7 +319,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -325,6 +328,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -344,31 +348,35 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -376,28 +384,24 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= @@ -414,21 +418,25 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -438,13 +446,16 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= @@ -452,34 +463,39 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -487,14 +503,15 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -502,29 +519,38 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -533,20 +559,18 @@ golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjTo golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -556,9 +580,23 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -573,7 +611,12 @@ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -589,15 +632,25 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -606,8 +659,9 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -618,6 +672,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -631,6 +686,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -640,36 +696,44 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= -k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= -k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= -k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= -k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc= -k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= -k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= -k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= -k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/component-base v0.19.2 h1:jW5Y9RcZTb79liEhW3XDVTW7MuvEGP0tQZnfSX6/+gs= -k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= +k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8txgVQ= +k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= +k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= +k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= -sigs.k8s.io/controller-runtime v0.7.2 h1:gD2JZp0bBLLuvSRYVNvox+bRCz1UUUxKDjPUCb56Ukk= -sigs.k8s.io/controller-runtime v0.7.2/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= +sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 634ffa69352d634689706ca8dcfcb09f4d184991 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Wed, 20 Oct 2021 14:33:36 -0700 Subject: [PATCH 015/163] Fix ns id cache (#43) Stop cacheing empty namespace ID if found in Cloud Map, and cache all namespaces in account. Continue to cache empty ID if not found to prevent throttling for non-imported services. --- pkg/cloudmap/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 49a3e68f..14c9a547 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -249,12 +249,14 @@ func (sdc *serviceDiscoveryClient) getNamespaceId(ctx context.Context, nsName st } for _, ns := range namespaces { - sdc.cacheNamespaceId(nsName, nsId) + sdc.cacheNamespaceId(ns.Name, ns.Id) if nsName == ns.Name { nsId = ns.Id } } + // This will cache empty namespace IDs for namespaces not in Cloud Map + sdc.cacheNamespaceId(nsName, nsId) return nsId, nil } From bfd82eb391a3c125f37f1aa5cefb2e871165d7ce Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 21 Oct 2021 12:12:17 -0700 Subject: [PATCH 016/163] Refactor client interface for service structs (#46) --- pkg/cloudmap/client.go | 102 +++---- pkg/cloudmap/client_test.go | 278 ++++++++++++++++++ pkg/cloudmap/operation_poller_test.go | 42 ++- pkg/controllers/serviceexport_controller.go | 18 +- .../serviceexport_controller_test.go | 10 +- pkg/model/types.go | 13 +- test/test-constants.go | 34 +++ 7 files changed, 398 insertions(+), 99 deletions(-) create mode 100644 pkg/cloudmap/client_test.go create mode 100644 test/test-constants.go diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 14c9a547..815a4857 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -26,17 +26,17 @@ type ServiceDiscoveryClient interface { // ListServices returns all services and their endpoints for a given namespace. ListServices(ctx context.Context, namespaceName string) ([]*model.Service, error) - // CreateService creates a Cloud Map service resource and returns the created service struct. - CreateService(ctx context.Context, service *model.Service) error + // CreateService creates a Cloud Map service resource, and namespace if necessary. + CreateService(ctx context.Context, namespaceName string, serviceName string) error // GetService returns a service resource fetched from AWS Cloud Map or nil if not found. GetService(ctx context.Context, namespaceName string, serviceName string) (*model.Service, error) // RegisterEndpoints registers all endpoints for given service. - RegisterEndpoints(ctx context.Context, service *model.Service) error + RegisterEndpoints(ctx context.Context, namespaceName string, serviceName string, endpoints []*model.Endpoint) error // DeleteEndpoints de-registers all endpoints for given service. - DeleteEndpoints(ctx context.Context, service *model.Service) error + DeleteEndpoints(ctx context.Context, namespaceName string, serviceName string, endpoints []*model.Endpoint) error } type serviceDiscoveryClient struct { @@ -59,8 +59,6 @@ func NewServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { } func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName string) (svcs []*model.Service, err error) { - svcs = make([]*model.Service, 0) - nsId, err := sdc.getNamespaceId(ctx, nsName) if err != nil || nsId == "" { return svcs, err @@ -73,14 +71,14 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri } for _, svcSum := range svcSums { - endpts, endptsErr := sdc.ListEndpoints(ctx, svcSum.Id) + sdc.cacheServiceId(nsName, svcSum.Name, svcSum.Id) + + endpts, endptsErr := sdc.listEndpoints(ctx, svcSum.Id) - if err != nil { + if endptsErr != nil { return svcs, endptsErr } - sdc.cacheServiceId(nsName, svcSum.Name, svcSum.Id) - svcs = append(svcs, &model.Service{ Namespace: nsName, Name: svcSum.Name, @@ -91,45 +89,31 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri return svcs, nil } -func (sdc *serviceDiscoveryClient) ListEndpoints(ctx context.Context, serviceId string) (endpts []*model.Endpoint, err error) { - - if cachedValue, exists := sdc.endpointCache.Get(serviceId); exists { - return cachedValue.([]*model.Endpoint), nil - } - - endpts, err = sdc.sdApi.ListInstances(ctx, serviceId) - - if err != nil { - return nil, err - } - - sdc.cacheEndpoints(serviceId, endpts) - - return endpts, nil -} - -func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, service *model.Service) (err error) { - sdc.log.Info("creating a new service", "namespace", service.Namespace, "name", service.Name) +func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName string, svcName string) (err error) { + sdc.log.Info("creating a new service", "namespace", nsName, "name", svcName) - nsId, err := sdc.getNamespaceId(ctx, service.Namespace) + nsId, err := sdc.getNamespaceId(ctx, nsName) if err != nil { return err } if nsId == "" { - nsId, err = sdc.createNamespace(ctx, service.Namespace) + nsId, err = sdc.createNamespace(ctx, nsName) + } + if err != nil { + return err } //TODO: Handle non-http namespaces - svcId, err := sdc.sdApi.CreateService(ctx, nsId, service.Name) + svcId, err := sdc.sdApi.CreateService(ctx, nsId, svcName) if err != nil { return err } - sdc.cacheServiceId(service.Namespace, service.Name, svcId) + sdc.cacheServiceId(nsName, svcName, svcId) - return sdc.RegisterEndpoints(ctx, service) + return nil } func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string, svcName string) (svc *model.Service, err error) { @@ -145,7 +129,7 @@ func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string return nil, nil } - endpts, err := sdc.ListEndpoints(ctx, svcId) + endpts, err := sdc.listEndpoints(ctx, svcId) if err != nil { return nil, err @@ -160,23 +144,22 @@ func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string return svc, nil } -func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, service *model.Service) (err error) { - if len(service.Endpoints) == 0 { - sdc.log.Info("skipping endpoint registration for empty endpoint list", "serviceName", service.Name) +func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName string, svcName string, endpts []*model.Endpoint) (err error) { + if len(endpts) == 0 { + sdc.log.Info("skipping endpoint registration for empty endpoint list", "serviceName", svcName) return nil } - sdc.log.Info("registering endpoints", "namespaceName", service.Namespace, - "serviceName", service.Name, "endpoints", service.Endpoints) + sdc.log.Info("registering endpoints", "namespaceName", nsName, "serviceName", svcName, "endpoints", endpts) - svcId, err := sdc.getServiceId(ctx, service.Namespace, service.Name) + svcId, err := sdc.getServiceId(ctx, nsName, svcName) if err != nil { return err } opCollector := NewOperationCollector() - for _, endpt := range service.Endpoints { + for _, endpt := range endpts { opCollector.Add(func() (opId string, err error) { return sdc.sdApi.RegisterInstance(ctx, svcId, endpt.Id, endpt.GetCloudMapAttributes()) }) @@ -198,23 +181,23 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, servic return nil } -func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, service *model.Service) (err error) { - if len(service.Endpoints) == 0 { - sdc.log.Info("skipping endpoint deletion for empty endpoint list", "serviceName", service.Name) +func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName string, svcName string, endpts []*model.Endpoint) (err error) { + if len(endpts) == 0 { + sdc.log.Info("skipping endpoint deletion for empty endpoint list", "serviceName", svcName) return nil } - sdc.log.Info("deleting endpoints", "namespaceName", service.Namespace, - "serviceName", service.Name, "endpoints", service.Endpoints) + sdc.log.Info("deleting endpoints", "namespaceName", nsName, + "serviceName", svcName, "endpoints", endpts) - svcId, err := sdc.getServiceId(ctx, service.Namespace, service.Name) + svcId, err := sdc.getServiceId(ctx, nsName, svcName) if err != nil { return err } opCollector := NewOperationCollector() - for _, endpt := range service.Endpoints { + for _, endpt := range endpts { opCollector.Add(func() (opId string, err error) { return sdc.sdApi.DeregisterInstance(ctx, svcId, endpt.Id) }) @@ -236,6 +219,23 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, service return nil } +func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId string) (endpts []*model.Endpoint, err error) { + + if cachedValue, exists := sdc.endpointCache.Get(serviceId); exists { + return cachedValue.([]*model.Endpoint), nil + } + + endpts, err = sdc.sdApi.ListInstances(ctx, serviceId) + + if err != nil { + return nil, err + } + + sdc.cacheEndpoints(serviceId, endpts) + + return endpts, nil +} + func (sdc *serviceDiscoveryClient) getNamespaceId(ctx context.Context, nsName string) (nsId string, err error) { // We are assuming a unique namespace name per account if cachedValue, exists := sdc.namespaceIdCache.Get(nsName); exists { @@ -307,10 +307,6 @@ func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName s return "", err } - if nsId == "" { - return "", fmt.Errorf("failed to create namespace") - } - sdc.cacheNamespaceId(nsName, nsId) return nsId, nil diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go new file mode 100644 index 00000000..98eae0cf --- /dev/null +++ b/pkg/cloudmap/client_test.go @@ -0,0 +1,278 @@ +package cloudmap + +import ( + "context" + "errors" + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/aws/aws-sdk-go-v2/aws" + testing2 "github.com/go-logr/logr/testing" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/cache" + "testing" + "time" +) + +func TestNewServiceDiscoveryClient(t *testing.T) { + sdc := NewServiceDiscoveryClient(&aws.Config{}) + assert.NotNil(t, sdc) +} + +func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) + sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). + Return([]*model.Endpoint{test.GetTestEndpoint()}, nil) + + sdc := getTestSdClient(t, sdApi) + svcs, err := sdc.ListServices(context.TODO(), test.NsName) + assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) + assert.Nil(t, err, "No error for happy case") + + cachedNs, _ := sdc.namespaceIdCache.Get(test.NsName) + assert.Equal(t, test.NsId, cachedNs, "Happy case caches namespace ID") + cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) + assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") + cachedEndpts, _ := sdc.endpointCache.Get(test.SvcId) + assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint()}, cachedEndpts, "Happy case caches endpoints") +} + +func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) + + sdc := getTestSdClient(t, sdApi) + sdc.namespaceIdCache.Add(test.NsName, test.NsId, time.Minute) + sdc.endpointCache.Add(test.SvcId, []*model.Endpoint{test.GetTestEndpoint()}, time.Minute) + + svcs, err := sdc.ListServices(context.TODO(), test.NsName) + assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) + assert.Nil(t, err, "No error for happy case") +} + +func TestServiceDiscoveryClient_ListServices_NamespaceError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + nsErr := errors.New("error listing namespaces") + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{}, nsErr) + + sdc := getTestSdClient(t, sdApi) + svcs, err := sdc.ListServices(context.TODO(), test.NsName) + assert.Equal(t, nsErr, err) + assert.Empty(t, svcs) +} + +func TestServiceDiscoveryClient_ListServices_ServiceError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + svcErr := errors.New("error listing services") + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + Return([]*model.Resource{}, svcErr) + + sdc := getTestSdClient(t, sdApi) + svcs, err := sdc.ListServices(context.TODO(), test.NsName) + assert.Equal(t, svcErr, err) + assert.Empty(t, svcs) +} + +func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + endptErr := errors.New("error listing endpoints") + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) + sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). + Return([]*model.Endpoint{}, endptErr) + + sdc := getTestSdClient(t, sdApi) + svcs, err := sdc.ListServices(context.TODO(), test.NsName) + assert.Equal(t, endptErr, err) + assert.Empty(t, svcs) +} + +func TestServiceDiscoveryClient_ListServices_NamespaceNotFound(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{}, nil) + + sdc := getTestSdClient(t, sdApi) + svcs, err := sdc.ListServices(context.TODO(), test.NsName) + assert.Empty(t, svcs) + assert.Nil(t, err, "No error for namespace not found") + + cachedNs, found := sdc.namespaceIdCache.Get(test.NsName) + assert.True(t, found) + assert.Equal(t, "", cachedNs, "Namespace not found caches empty ID") +} + +func TestServiceDiscoveryClient_CreateService_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + Return(test.SvcId, nil) + + sdc := getTestSdClient(t, sdApi) + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Nil(t, err, "No error for happy case") + + cachedNs, _ := sdc.namespaceIdCache.Get(test.NsName) + assert.Equal(t, test.NsId, cachedNs, "Happy case caches namespace ID") + cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) + assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") +} + +func TestServiceDiscoveryClient_CreateService_HappyCaseCachedResults(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + Return(test.SvcId, nil) + + sdc := getTestSdClient(t, sdApi) + sdc.namespaceIdCache.Add(test.NsName, test.NsId, time.Minute) + + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Nil(t, err, "No error for happy case") +} + +func TestServiceDiscoveryClient_CreateService_NamespaceError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + nsErr := errors.New("error listing namespaces") + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{}, nsErr) + + sdc := getTestSdClient(t, sdApi) + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Equal(t, nsErr, err) +} + +func TestServiceDiscoveryClient_CreateService_CreateServiceError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + svcErr := errors.New("error creating service") + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + Return("", svcErr) + + sdc := getTestSdClient(t, sdApi) + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Equal(t, err, svcErr) +} + +func TestServiceDiscoveryClient_CreateService_CreatesNamespace_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdc := getTestSdClient(t, sdApi) + + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{}, nil) + sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + Return(test.OpId1, nil) + sdApi.EXPECT().PollCreateNamespace(context.TODO(), test.OpId1). + Return(test.NsId, nil) + sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + Return(test.SvcId, nil) + + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Nil(t, err, "No error for happy case") + + cachedNs, _ := sdc.namespaceIdCache.Get(test.NsName) + assert.Equal(t, test.NsId, cachedNs, "Create namespace caches namespace ID") +} + +func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + pollErr := errors.New("polling error") + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{}, nil) + sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + Return(test.OpId1, nil) + sdApi.EXPECT().PollCreateNamespace(context.TODO(), test.OpId1). + Return("", pollErr) + + sdc := getTestSdClient(t, sdApi) + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Equal(t, pollErr, err) +} + +func TestServiceDiscoveryClient_CreateService_CreatesNamespace_CreateNsError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + nsErr := errors.New("create namespace error") + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Resource{}, nil) + sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + Return("", nsErr) + + sdc := getTestSdClient(t, sdApi) + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Equal(t, nsErr, err) +} + +func TestServiceDiscoveryClient_GetService(t *testing.T) { + // TODO: Add unit tests +} + +func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { + // TODO: Add unit tests +} + +func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { + // TODO: Add unit tests +} + +func getTestSdClient(t *testing.T, sdApi ServiceDiscoveryApi) serviceDiscoveryClient { + return serviceDiscoveryClient{ + log: testing2.TestLogger{T: t}, + sdApi: sdApi, + namespaceIdCache: cache.NewLRUExpireCache(1024), + serviceIdCache: cache.NewLRUExpireCache(1024), + endpointCache: cache.NewLRUExpireCache(1024), + } +} diff --git a/pkg/cloudmap/operation_poller_test.go b/pkg/cloudmap/operation_poller_test.go index 45cf2926..ccff3d93 100644 --- a/pkg/cloudmap/operation_poller_test.go +++ b/pkg/cloudmap/operation_poller_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" @@ -13,13 +14,6 @@ import ( "time" ) -const ( - svcId = "test-svc-id" - opId1 = "operation-id-1" - opId2 = "operation-id-2" - startTime = 1 -) - func TestOperationPoller_HappyCases(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() @@ -32,13 +26,13 @@ func TestOperationPoller_HappyCases(t *testing.T) { }{ { constructor: func() OperationPoller { - return NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + return NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) }, expectedOpType: types.OperationTypeRegisterInstance, }, { constructor: func() OperationPoller { - return NewDeregisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + return NewDeregisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) }, expectedOpType: types.OperationTypeDeregisterInstance, }, @@ -55,7 +49,7 @@ func TestOperationPoller_HappyCases(t *testing.T) { assert.Contains(t, filters, types.OperationFilter{ Name: types.OperationFilterNameServiceId, - Values: []string{svcId}, + Values: []string{test.SvcId}, }) assert.Contains(t, filters, types.OperationFilter{ @@ -78,7 +72,7 @@ func TestOperationPoller_HappyCases(t *testing.T) { assert.Equal(t, 2, len(timeFilter.Values)) filterStart, _ := strconv.Atoi(timeFilter.Values[0]) - assert.Equal(t, startTime, filterStart) + assert.Equal(t, test.OpStart, filterStart) firstEnd, _ = strconv.Atoi(timeFilter.Values[1]) @@ -94,8 +88,8 @@ func TestOperationPoller_HappyCases(t *testing.T) { "Filter time frame for operations must increase between invocations of ListOperations") return map[string]types.OperationStatus{ - opId1: types.OperationStatusSuccess, - opId2: types.OperationStatusSuccess, + test.OpId1: types.OperationStatusSuccess, + test.OpId2: types.OperationStatusSuccess, }, nil }) @@ -111,7 +105,7 @@ func TestOperationPoller_PollEmpty(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - p := NewRegisterInstancePoller(sdApi, svcId, []string{}, startTime) + p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{}, test.OpStart) err := p.Poll(context.TODO()) assert.Nil(t, err) } @@ -122,7 +116,7 @@ func TestOperationPoller_PollFailure(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - p := NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) pollErr := errors.New("error polling operations") @@ -140,20 +134,20 @@ func TestOperationPoller_PollOpFailure(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - p := NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) sdApi.EXPECT(). ListOperations(gomock.Any(), gomock.Any()). Return( map[string]types.OperationStatus{ - opId1: types.OperationStatusSuccess, - opId2: types.OperationStatusFail, + test.OpId1: types.OperationStatusSuccess, + test.OpId2: types.OperationStatusFail, }, nil) opErr := "operation failure message" sdApi.EXPECT(). - GetOperation(gomock.Any(), opId2). + GetOperation(gomock.Any(), test.OpId2). Return(&types.Operation{ErrorMessage: &opErr}, nil) err := p.Poll(context.TODO()) @@ -166,18 +160,18 @@ func TestOperationPoller_PollOpFailureAndMessageFailure(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - p := NewRegisterInstancePoller(sdApi, svcId, []string{opId1, opId2}, startTime) + p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) sdApi.EXPECT(). ListOperations(gomock.Any(), gomock.Any()). Return( map[string]types.OperationStatus{ - opId1: types.OperationStatusFail, - opId2: types.OperationStatusSuccess, + test.OpId1: types.OperationStatusFail, + test.OpId2: types.OperationStatusSuccess, }, nil) sdApi.EXPECT(). - GetOperation(gomock.Any(), opId1). + GetOperation(gomock.Any(), test.OpId1). Return(nil, errors.New("failed to retrieve operation failure reason")) err := p.Poll(context.TODO()) @@ -194,7 +188,7 @@ func TestOperationPoller_PollTimeout(t *testing.T) { log: testing2.TestLogger{T: t}, sdApi: sdApi, timeout: 2 * time.Millisecond, - opIds: []string{opId1, opId2}, + opIds: []string{test.OpId1, test.OpId2}, } sdApi.EXPECT(). diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index 8aaf0660..c9efb541 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -89,11 +89,6 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, log logr.Log return ctrl.Result{}, err } - cloudMapService := &model.Service{ - Namespace: svc.Namespace, - Name: svc.Name, - } - changes := model.Changes{ Create: endpoints, } @@ -105,7 +100,7 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, log logr.Log return ctrl.Result{}, err } if srv == nil { - if err := r.Cloudmap.CreateService(ctx, cloudMapService); err != nil { + if err := r.Cloudmap.CreateService(ctx, svc.Namespace, svc.Name); err != nil { log.Error(err, "error when creating new service in Cloud Map", "namespace", svc.Namespace, "name", svc.Name) return ctrl.Result{}, err } @@ -124,10 +119,9 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, log logr.Log if createRequired || updateRequired { // merge creates and updates (Cloud Map RegisterEndpoints can handle both) - cloudMapService.Endpoints = changes.Create - cloudMapService.Endpoints = append(cloudMapService.Endpoints, changes.Update...) + upserts := append(changes.Create, changes.Update...) - if err := r.Cloudmap.RegisterEndpoints(ctx, cloudMapService); err != nil { + if err := r.Cloudmap.RegisterEndpoints(ctx, svc.Namespace, svc.Name, upserts); err != nil { log.Error(err, "error when registering endpoints to Cloud Map", "namespace", svc.Namespace, "name", svc.Name) return ctrl.Result{}, err @@ -135,9 +129,7 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, log logr.Log } if deleteRequired { - cloudMapService.Endpoints = changes.Delete - - if err := r.Cloudmap.DeleteEndpoints(ctx, cloudMapService); err != nil { + if err := r.Cloudmap.DeleteEndpoints(ctx, svc.Namespace, svc.Name, changes.Delete); err != nil { log.Error(err, "error when deleting endpoints from Cloud Map", "namespace", srv.Namespace, "name", srv.Name) return ctrl.Result{}, err @@ -163,7 +155,7 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, log logr.Log return ctrl.Result{}, err } if srv != nil { - if err := r.Cloudmap.DeleteEndpoints(ctx, srv); err != nil { + if err := r.Cloudmap.DeleteEndpoints(ctx, srv.Namespace, srv.Name, srv.Endpoints); err != nil { log.Error(err, "error when deleting endpoints from Cloud Map", "namespace", srv.Namespace, "name", srv.Name) return ctrl.Result{}, err diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index 89358de5..cb351ecc 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -37,9 +37,10 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { cloudmapMock := cloudmapmock.NewMockServiceDiscoveryClient(mockController) // expected interactions with the Cloud Map client - cloudmapMock.EXPECT().GetService(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - cloudmapMock.EXPECT().CreateService(gomock.Any(), gomock.Any()).Return(nil).Times(1) - cloudmapMock.EXPECT().RegisterEndpoints(gomock.Any(), gomock.Eq(&expectedService)).Return(nil).Times(1) + cloudmapMock.EXPECT().GetService(gomock.Any(), expectedService.Namespace, expectedService.Name).Return(nil, nil) + cloudmapMock.EXPECT().CreateService(gomock.Any(), expectedService.Namespace, expectedService.Name).Return(nil).Times(1) + cloudmapMock.EXPECT().RegisterEndpoints(gomock.Any(), expectedService.Namespace, expectedService.Name, + expectedService.Endpoints).Return(nil).Times(1) reconciler := setupServiceExportReconciler(t, cloudmapMock) @@ -82,7 +83,8 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceNewEndpoint(t *testing // expected interactions with the Cloud Map client cloudmapMock.EXPECT().GetService(gomock.Any(), gomock.Any(), gomock.Any()).Return(&emptyService, nil) - cloudmapMock.EXPECT().RegisterEndpoints(gomock.Any(), gomock.Eq(&expectedService)).Return(nil).Times(1) + cloudmapMock.EXPECT().RegisterEndpoints(gomock.Any(), expectedService.Namespace, expectedService.Name, + expectedService.Endpoints).Return(nil).Times(1) request := ctrl.Request{ NamespacedName: types.NamespacedName{ diff --git a/pkg/model/types.go b/pkg/model/types.go index 78ba392c..d2be9d73 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -10,18 +10,20 @@ import ( "strings" ) -// Resource encapsulates a ID/name pair +// Resource encapsulates a ID/name pair. type Resource struct { Id string Name string } +// Service holds namespace and endpoint state for a named service. type Service struct { Namespace string Name string Endpoints []*Endpoint } +// Endpoint holds basic values and attributes for an endpoint. type Endpoint struct { Id string IP string @@ -34,7 +36,7 @@ const ( PortAttr = "AWS_INSTANCE_PORT" ) -// NewEndpointFromInstance converts a Cloud Map InstanceSummary to an endpoint +// NewEndpointFromInstance converts a Cloud Map InstanceSummary to an endpoint. func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { endpoint := Endpoint{ Id: *inst.Id, @@ -68,7 +70,7 @@ func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { return &endpoint, nil } -// GetCloudMapAttributes extracts endpoint attributes for Cloud Map service instance registration +// GetCloudMapAttributes extracts endpoint attributes for Cloud Map service instance registration. func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs := make(map[string]string, 0) @@ -84,11 +86,12 @@ func (e *Endpoint) GetCloudMapAttributes() map[string]string { return attrs } -// Equals evaluates if two Endpoints are "deeply equal" (including all fields) +// Equals evaluates if two Endpoints are "deeply equal" (including all fields). func (e *Endpoint) Equals(other *Endpoint) bool { return reflect.DeepEqual(e, other) } +// String gives a string representation for an endpoint. func (e *Endpoint) String() string { bytes, err := json.Marshal(e) if err != nil { @@ -98,7 +101,7 @@ func (e *Endpoint) String() string { return string(bytes) } -// EndpointIdFromIPAddress converts an IP address to human readable identifier +// EndpointIdFromIPAddress converts an IP address to human readable identifier. func EndpointIdFromIPAddress(address string) string { return strings.Replace(address, ".", "_", -1) } diff --git a/test/test-constants.go b/test/test-constants.go new file mode 100644 index 00000000..f0699062 --- /dev/null +++ b/test/test-constants.go @@ -0,0 +1,34 @@ +package test + +import "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + +const ( + NsName = "ns-name" + NsId = "ns-id" + SvcName = "svc-name" + SvcId = "svc-id" + EndptId1 = "endpoint-id-1" + EndptIp1 = "endpoint-ip-1" + EndptPort1 = 2 + OpId1 = "operation-id-1" + OpId2 = "operation-id-2" + OpStart = 1 +) + +func GetTestService() *model.Service { + endPt := GetTestEndpoint() + return &model.Service{ + Namespace: NsName, + Name: SvcName, + Endpoints: []*model.Endpoint{endPt}, + } +} + +func GetTestEndpoint() *model.Endpoint { + return &model.Endpoint{ + Id: EndptId1, + IP: EndptIp1, + Port: EndptPort1, + Attributes: make(map[string]string, 0), + } +} From 6cfd1c1963b045e28951d41641d73b584a767684 Mon Sep 17 00:00:00 2001 From: hendoncr Date: Fri, 22 Oct 2021 14:16:51 -0400 Subject: [PATCH 017/163] Add additional permissions to the ClusterRole (#52) --- config/rbac/role.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 565b958e..6035e516 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -28,7 +28,31 @@ rules: resources: - serviceimports verbs: + - create - get - list - patch - update + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - endpoints + verbs: + - get + - list + - watch + - create + - update +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get + - create + - update From ba9b31ca87969c5bd4e3cdf6259551c9216e36ad Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Mon, 25 Oct 2021 14:48:47 -0700 Subject: [PATCH 018/163] Add the support for creating service with DNS namespace type (#48) --- pkg/cloudmap/api.go | 52 ++++++++++--- pkg/cloudmap/api_test.go | 137 ++++++++++++++++++++++++++++++++++ pkg/cloudmap/client.go | 144 ++++++++++++++++++++---------------- pkg/cloudmap/client_test.go | 117 +++++++++++++++++++++-------- pkg/model/types.go | 31 ++++++++ test/test-constants.go | 16 ++++ 6 files changed, 392 insertions(+), 105 deletions(-) create mode 100644 pkg/cloudmap/api_test.go diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index f529c99b..5f1df066 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -13,11 +13,15 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) +const ( + defaultServiceTTLInSeconds int64 = 60 +) + // ServiceDiscoveryApi handles the AWS Cloud Map API request and response processing logic, and converts results to // internal data structures. It manages all interactions with the AWS SDK. type ServiceDiscoveryApi interface { // ListNamespaces returns a list of all namespaces. - ListNamespaces(ctx context.Context) (namespaces []*model.Resource, err error) + ListNamespaces(ctx context.Context) (namespaces []*model.Namespace, err error) // ListServices returns a list of services for a given namespace. ListServices(ctx context.Context, namespaceId string) (services []*model.Resource, err error) @@ -35,7 +39,7 @@ type ServiceDiscoveryApi interface { CreateHttpNamespace(ctx context.Context, namespaceName string) (operationId string, err error) // CreateService creates a named service in AWS Cloud Map under the given namespace. - CreateService(ctx context.Context, namespaceId string, serviceName string) (serviceId string, err error) + CreateService(ctx context.Context, namespace model.Namespace, serviceName string) (serviceId string, err error) // RegisterInstance registers a service instance in AWS Cloud Map. RegisterInstance(ctx context.Context, serviceId string, instanceId string, instanceAttrs map[string]string) (operationId string, err error) @@ -60,8 +64,8 @@ func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { } } -func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*model.Resource, error) { - namespaces := make([]*model.Resource, 0) +func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*model.Namespace, error) { + namespaces := make([]*model.Namespace, 0) pages := sd.NewListNamespacesPaginator(sdApi.awsFacade, &sd.ListNamespacesInput{}) for pages.HasMorePages() { @@ -71,10 +75,13 @@ func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*model. } for _, ns := range output.Namespaces { - namespaces = append(namespaces, &model.Resource{ - Id: aws.ToString(ns.Id), - Name: aws.ToString(ns.Name), - }) + if namespaceType := model.ConvertNamespaceType(ns.Type); !namespaceType.IsUnsupported() { + namespaces = append(namespaces, &model.Namespace{ + Id: aws.ToString(ns.Id), + Name: aws.ToString(ns.Name), + Type: namespaceType, + }) + } } } @@ -178,10 +185,19 @@ func (sdApi *serviceDiscoveryApi) CreateHttpNamespace(ctx context.Context, nsNam return aws.ToString(output.OperationId), nil } -func (sdApi *serviceDiscoveryApi) CreateService(ctx context.Context, nsId string, svcName string) (svcId string, err error) { - output, err := sdApi.awsFacade.CreateService(ctx, &sd.CreateServiceInput{ - NamespaceId: &nsId, - Name: &svcName}) +func (sdApi *serviceDiscoveryApi) CreateService(ctx context.Context, namespace model.Namespace, svcName string) (svcId string, err error) { + var output *sd.CreateServiceOutput + if namespace.Type == model.DnsPrivateNamespaceType { + dnsConfig := sdApi.getDnsConfig() + output, err = sdApi.awsFacade.CreateService(ctx, &sd.CreateServiceInput{ + NamespaceId: &namespace.Id, + DnsConfig: &dnsConfig, + Name: &svcName}) + } else { + output, err = sdApi.awsFacade.CreateService(ctx, &sd.CreateServiceInput{ + NamespaceId: &namespace.Id, + Name: &svcName}) + } if err != nil { return "", err @@ -192,6 +208,18 @@ func (sdApi *serviceDiscoveryApi) CreateService(ctx context.Context, nsId string return svcId, nil } +func (sdApi *serviceDiscoveryApi) getDnsConfig() types.DnsConfig { + dnsConfig := types.DnsConfig{ + DnsRecords: []types.DnsRecord{ + { + TTL: aws.Int64(defaultServiceTTLInSeconds), + Type: "SRV", + }, + }, + } + return dnsConfig +} + func (sdApi *serviceDiscoveryApi) RegisterInstance(ctx context.Context, svcId string, instId string, instAttrs map[string]string) (opId string, err error) { regResp, err := sdApi.awsFacade.RegisterInstance(ctx, &sd.RegisterInstanceInput{ Attributes: instAttrs, diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go new file mode 100644 index 00000000..b194d979 --- /dev/null +++ b/pkg/cloudmap/api_test.go @@ -0,0 +1,137 @@ +package cloudmap + +import ( + "context" + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/aws/aws-sdk-go-v2/aws" + sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" + testingLogger "github.com/go-logr/logr/testing" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewServiceDiscoveryApi(t *testing.T) { + sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}) + assert.NotNil(t, sdc) +} + +func TestServiceDiscoveryApi_ListNamespaces_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + id, name := test.NsId, test.NsName + ns := types.NamespaceSummary{ + Name: &name, + Id: &id, + Type: types.NamespaceTypeDnsPrivate, + } + awsFacade.EXPECT().ListNamespaces(context.TODO(), &sd.ListNamespacesInput{}). + Return(&sd.ListNamespacesOutput{Namespaces: []types.NamespaceSummary{ns}}, nil) + + namespaces, _ := sdApi.ListNamespaces(context.TODO()) + assert.True(t, len(namespaces) == 1) + assert.Equal(t, test.GetTestDnsNamespace(), namespaces[0], "") +} + +func TestServiceDiscoveryApi_ListNamespaces_SkipPublicDNSNotSupported(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + id, name := test.NsId, test.NsName + ns := types.NamespaceSummary{ + Name: &name, + Id: &id, + Type: types.NamespaceTypeDnsPublic, + } + awsFacade.EXPECT().ListNamespaces(context.TODO(), &sd.ListNamespacesInput{}). + Return(&sd.ListNamespacesOutput{Namespaces: []types.NamespaceSummary{ns}}, nil) + + namespaces, _ := sdApi.ListNamespaces(context.TODO()) + assert.True(t, len(namespaces) == 0, "Successfully skipped DNS_PUBLIC from the output") +} + +func TestServiceDiscoveryApi_CreateService_CreateForHttpNamespace(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + nsId, svcId, svcName := test.NsId, test.SvcId, test.SvcName + awsFacade.EXPECT().CreateService(context.TODO(), &sd.CreateServiceInput{ + Name: &svcName, + NamespaceId: &nsId, + }). + Return(&sd.CreateServiceOutput{ + Service: &types.Service{ + Id: &svcId, + }, + }, nil) + + retSvcId, _ := sdApi.CreateService(context.TODO(), *test.GetTestHttpNamespace(), svcName) + assert.Equal(t, svcId, retSvcId, "Successfully created service") +} + +func TestServiceDiscoveryApi_CreateService_CreateForDnsNamespace(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + nsId, svcId, svcName := test.NsId, test.SvcId, test.SvcName + awsFacade.EXPECT().CreateService(context.TODO(), &sd.CreateServiceInput{ + Name: &svcName, + NamespaceId: &nsId, + DnsConfig: &types.DnsConfig{ + DnsRecords: []types.DnsRecord{{ + TTL: aws.Int64(60), + Type: "SRV", + }}, + }, + }). + Return(&sd.CreateServiceOutput{ + Service: &types.Service{ + Id: &svcId, + }, + }, nil) + + retSvcId, _ := sdApi.CreateService(context.TODO(), *test.GetTestDnsNamespace(), svcName) + assert.Equal(t, svcId, retSvcId, "Successfully created service") +} + +func TestServiceDiscoveryApi_CreateService_ThrowError(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + nsId, svcName := test.NsId, test.SvcName + awsFacade.EXPECT().CreateService(context.TODO(), &sd.CreateServiceInput{ + Name: &svcName, + NamespaceId: &nsId, + }). + Return(nil, fmt.Errorf("dummy error")) + + retSvcId, err := sdApi.CreateService(context.TODO(), *test.GetTestHttpNamespace(), svcName) + assert.Empty(t, retSvcId) + assert.Equal(t, "dummy error", fmt.Sprint(err), "Got error") +} + +func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmap.MockAwsFacade) serviceDiscoveryApi { + return serviceDiscoveryApi{ + log: testingLogger.TestLogger{T: t}, + awsFacade: awsFacade, + } +} diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 815a4857..7515a178 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -12,12 +12,12 @@ import ( ) const ( - defaultNamespaceIdCacheTTL = 2 * time.Minute - defaultNamespaceIdCacheSize = 100 - defaultServiceIdCacheTTL = 2 * time.Minute - defaultServiceIdCacheSize = 1024 - defaultEndpointsCacheTTL = 5 * time.Second - defaultEndpointsCacheSize = 1024 + defaultNamespaceCacheTTL = 2 * time.Minute + defaultNamespaceCacheSize = 100 + defaultServiceIdCacheTTL = 2 * time.Minute + defaultServiceIdCacheSize = 1024 + defaultEndpointsCacheTTL = 5 * time.Second + defaultEndpointsCacheSize = 1024 ) // ServiceDiscoveryClient provides the service endpoint management functionality required by the AWS Cloud Map @@ -40,32 +40,31 @@ type ServiceDiscoveryClient interface { } type serviceDiscoveryClient struct { - log logr.Logger - sdApi ServiceDiscoveryApi - namespaceIdCache *cache.LRUExpireCache - serviceIdCache *cache.LRUExpireCache - endpointCache *cache.LRUExpireCache + log logr.Logger + sdApi ServiceDiscoveryApi + namespaceCache *cache.LRUExpireCache + serviceIdCache *cache.LRUExpireCache + endpointCache *cache.LRUExpireCache } // NewServiceDiscoveryClient creates a new service discovery client for AWS Cloud Map from a given AWS client config. func NewServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: ctrl.Log.WithName("cloudmap"), - sdApi: NewServiceDiscoveryApiFromConfig(cfg), - namespaceIdCache: cache.NewLRUExpireCache(defaultNamespaceIdCacheSize), - serviceIdCache: cache.NewLRUExpireCache(defaultServiceIdCacheSize), - endpointCache: cache.NewLRUExpireCache(defaultEndpointsCacheSize), + log: ctrl.Log.WithName("cloudmap"), + sdApi: NewServiceDiscoveryApiFromConfig(cfg), + namespaceCache: cache.NewLRUExpireCache(defaultNamespaceCacheSize), + serviceIdCache: cache.NewLRUExpireCache(defaultServiceIdCacheSize), + endpointCache: cache.NewLRUExpireCache(defaultEndpointsCacheSize), } } func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName string) (svcs []*model.Service, err error) { - nsId, err := sdc.getNamespaceId(ctx, nsName) - if err != nil || nsId == "" { + namespace, err := sdc.getNamespace(ctx, nsName) + if err != nil || namespace == nil { return svcs, err } - svcSums, err := sdc.sdApi.ListServices(ctx, nsId) - + svcSums, err := sdc.sdApi.ListServices(ctx, namespace.Id) if err != nil { return svcs, err } @@ -74,7 +73,6 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri sdc.cacheServiceId(nsName, svcSum.Name, svcSum.Id) endpts, endptsErr := sdc.listEndpoints(ctx, svcSum.Id) - if endptsErr != nil { return svcs, endptsErr } @@ -92,21 +90,20 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName string, svcName string) (err error) { sdc.log.Info("creating a new service", "namespace", nsName, "name", svcName) - nsId, err := sdc.getNamespaceId(ctx, nsName) + namespace, err := sdc.getNamespace(ctx, nsName) if err != nil { return err } - if nsId == "" { - nsId, err = sdc.createNamespace(ctx, nsName) - } - if err != nil { - return err + if namespace == nil { + // Create HttpNamespace if the namespace is not present in the CloudMap + namespace, err = sdc.createNamespace(ctx, nsName) + if err != nil { + return err + } } - //TODO: Handle non-http namespaces - svcId, err := sdc.sdApi.CreateService(ctx, nsId, svcName) - + svcId, err := sdc.sdApi.CreateService(ctx, *namespace, svcName) if err != nil { return err } @@ -117,7 +114,7 @@ func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName str } func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string, svcName string) (svc *model.Service, err error) { - sdc.log.Info("fetching a service", "nsName", nsName, "svcName", svcName) + sdc.log.Info("fetching a service", "namespace", nsName, "name", svcName) svcId, err := sdc.getServiceId(ctx, nsName, svcName) @@ -207,7 +204,6 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s // Evict cache entry so next list call reflects changes sdc.evictEndpoints(svcId) - if err != nil { return err } @@ -226,7 +222,6 @@ func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId } endpts, err = sdc.sdApi.ListInstances(ctx, serviceId) - if err != nil { return nil, err } @@ -236,28 +231,35 @@ func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId return endpts, nil } -func (sdc *serviceDiscoveryClient) getNamespaceId(ctx context.Context, nsName string) (nsId string, err error) { +func (sdc *serviceDiscoveryClient) getNamespace(ctx context.Context, nsName string) (namespace *model.Namespace, err error) { // We are assuming a unique namespace name per account - if cachedValue, exists := sdc.namespaceIdCache.Get(nsName); exists { - return cachedValue.(string), nil + if namespace, exists, err := sdc.getCachedNamespace(nsName); exists { + return namespace, err + } + if err != nil { + return nil, err } namespaces, err := sdc.sdApi.ListNamespaces(ctx) - if err != nil { - return "", err + return nil, err } for _, ns := range namespaces { - sdc.cacheNamespaceId(ns.Name, ns.Id) + sdc.cacheNamespace(*ns) + // Set the return namespace if nsName == ns.Name { - nsId = ns.Id + namespace = ns } } - // This will cache empty namespace IDs for namespaces not in Cloud Map - sdc.cacheNamespaceId(nsName, nsId) - return nsId, nil + if namespace == nil { + // This will cache empty namespace for namespaces not in Cloud Map + // This is so that we can avoid ListNamespaces call + sdc.cacheNilNamespace(nsName) + } + + return namespace, nil } func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName string, svcName string) (svcId string, err error) { @@ -267,23 +269,17 @@ func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName stri return cachedValue.(string), nil } - nsId, err := sdc.getNamespaceId(ctx, nsName) - - if err != nil { + namespace, err := sdc.getNamespace(ctx, nsName) + if err != nil || namespace == nil { return "", err } - if nsId == "" { - return "", nil - } - - svcs, err := sdc.sdApi.ListServices(ctx, nsId) - + services, err := sdc.sdApi.ListServices(ctx, namespace.Id) if err != nil { return "", err } - for _, svc := range svcs { + for _, svc := range services { sdc.cacheServiceId(nsName, svcName, svc.Id) if svc.Name == svcName { svcId = svc.Id @@ -293,27 +289,49 @@ func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName stri return svcId, nil } -func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName string) (nsId string, err error) { +func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName string) (namespace *model.Namespace, err error) { sdc.log.Info("creating a new namespace", "namespace", nsName) opId, err := sdc.sdApi.CreateHttpNamespace(ctx, nsName) - if err != nil { - return "", err + return nil, err } - nsId, err = sdc.sdApi.PollCreateNamespace(ctx, opId) - + nsId, err := sdc.sdApi.PollCreateNamespace(ctx, opId) if err != nil { - return "", err + return nil, err + } + + // Cache the Namespace, by default we always create namespace of type HTTP + namespace = &model.Namespace{ + Id: nsId, + Name: nsName, + Type: model.HttpNamespaceType, } + sdc.cacheNamespace(*namespace) + + return namespace, nil +} - sdc.cacheNamespaceId(nsName, nsId) +func (sdc *serviceDiscoveryClient) getCachedNamespace(nsName string) (namespace *model.Namespace, exists bool, err error) { + if cachedValue, exists := sdc.namespaceCache.Get(nsName); exists { + if cachedValue == nil { + return nil, exists, nil + } + ns, ok := cachedValue.(model.Namespace) + if !ok { + return nil, exists, fmt.Errorf("failed to cast the cached value for the namespace %s", nsName) + } + return &ns, exists, nil + } + return nil, exists, nil +} - return nsId, nil +func (sdc *serviceDiscoveryClient) cacheNamespace(namespace model.Namespace) { + sdc.namespaceCache.Add(namespace.Name, namespace, defaultNamespaceCacheTTL) } -func (sdc *serviceDiscoveryClient) cacheNamespaceId(nsName string, nsId string) { - sdc.namespaceIdCache.Add(nsName, nsId, defaultNamespaceIdCacheTTL) +func (sdc *serviceDiscoveryClient) cacheNilNamespace(nsName string) { + sdc.namespaceCache.Add(nsName, nil, defaultNamespaceCacheTTL) } func (sdc *serviceDiscoveryClient) cacheServiceId(nsName string, svcName string, svcId string) { diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 98eae0cf..9c641e6d 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -27,7 +27,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) sdApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). @@ -38,8 +38,8 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) assert.Nil(t, err, "No error for happy case") - cachedNs, _ := sdc.namespaceIdCache.Get(test.NsName) - assert.Equal(t, test.NsId, cachedNs, "Happy case caches namespace ID") + cachedNs, _ := sdc.namespaceCache.Get(test.NsName) + assert.Equal(t, *test.GetTestHttpNamespace(), cachedNs, "Happy case caches namespace ID") cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") cachedEndpts, _ := sdc.endpointCache.Get(test.SvcId) @@ -55,7 +55,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) sdc := getTestSdClient(t, sdApi) - sdc.namespaceIdCache.Add(test.NsName, test.NsId, time.Minute) + sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) sdc.endpointCache.Add(test.SvcId, []*model.Endpoint{test.GetTestEndpoint()}, time.Minute) svcs, err := sdc.ListServices(context.TODO(), test.NsName) @@ -70,7 +70,7 @@ func TestServiceDiscoveryClient_ListServices_NamespaceError(t *testing.T) { nsErr := errors.New("error listing namespaces") sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{}, nsErr) + Return([]*model.Namespace{}, nsErr) sdc := getTestSdClient(t, sdApi) svcs, err := sdc.ListServices(context.TODO(), test.NsName) @@ -86,7 +86,7 @@ func TestServiceDiscoveryClient_ListServices_ServiceError(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) sdApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{}, svcErr) @@ -103,7 +103,7 @@ func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { endptErr := errors.New("error listing endpoints") sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) + Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) sdApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). @@ -121,16 +121,16 @@ func TestServiceDiscoveryClient_ListServices_NamespaceNotFound(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{}, nil) + Return([]*model.Namespace{}, nil) sdc := getTestSdClient(t, sdApi) svcs, err := sdc.ListServices(context.TODO(), test.NsName) assert.Empty(t, svcs) assert.Nil(t, err, "No error for namespace not found") - cachedNs, found := sdc.namespaceIdCache.Get(test.NsName) + cachedNs, found := sdc.namespaceCache.Get(test.NsName) assert.True(t, found) - assert.Equal(t, "", cachedNs, "Namespace not found caches empty ID") + assert.Nil(t, cachedNs, "Namespace not found in the cache") } func TestServiceDiscoveryClient_CreateService_HappyCase(t *testing.T) { @@ -139,16 +139,36 @@ func TestServiceDiscoveryClient_CreateService_HappyCase(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) - sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) + sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) sdc := getTestSdClient(t, sdApi) err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err, "No error for happy case") - cachedNs, _ := sdc.namespaceIdCache.Get(test.NsName) - assert.Equal(t, test.NsId, cachedNs, "Happy case caches namespace ID") + cachedNs, _ := sdc.namespaceCache.Get(test.NsName) + assert.Equal(t, *test.GetTestHttpNamespace(), cachedNs, "Happy case caches namespace") + cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) + assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") +} + +func TestServiceDiscoveryClient_CreateService_HappyCaseForDNSNamespace(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Namespace{test.GetTestDnsNamespace()}, nil) + sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). + Return(test.SvcId, nil) + + sdc := getTestSdClient(t, sdApi) + err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + assert.Nil(t, err, "No error for happy case") + + cachedNs, _ := sdc.namespaceCache.Get(test.NsName) + assert.Equal(t, *test.GetTestDnsNamespace(), cachedNs, "Happy case caches namespace") cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") } @@ -158,11 +178,11 @@ func TestServiceDiscoveryClient_CreateService_HappyCaseCachedResults(t *testing. defer mockController.Finish() sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) sdc := getTestSdClient(t, sdApi) - sdc.namespaceIdCache.Add(test.NsName, test.NsId, time.Minute) + sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err, "No error for happy case") @@ -175,7 +195,7 @@ func TestServiceDiscoveryClient_CreateService_NamespaceError(t *testing.T) { nsErr := errors.New("error listing namespaces") sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{}, nsErr) + Return([]*model.Namespace{}, nsErr) sdc := getTestSdClient(t, sdApi) err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) @@ -189,8 +209,8 @@ func TestServiceDiscoveryClient_CreateService_CreateServiceError(t *testing.T) { svcErr := errors.New("error creating service") sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{{Name: test.NsName, Id: test.NsId}}, nil) - sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) + sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return("", svcErr) sdc := getTestSdClient(t, sdApi) @@ -206,19 +226,19 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_HappyCase(t *test sdc := getTestSdClient(t, sdApi) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{}, nil) + Return([]*model.Namespace{}, nil) sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return(test.OpId1, nil) sdApi.EXPECT().PollCreateNamespace(context.TODO(), test.OpId1). Return(test.NsId, nil) - sdApi.EXPECT().CreateService(context.TODO(), test.NsId, test.SvcName). + sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err, "No error for happy case") - cachedNs, _ := sdc.namespaceIdCache.Get(test.NsName) - assert.Equal(t, test.NsId, cachedNs, "Create namespace caches namespace ID") + cachedNs, _ := sdc.namespaceCache.Get(test.NsName) + assert.Equal(t, *test.GetTestHttpNamespace(), cachedNs, "Create namespace caches namespace ID") } func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *testing.T) { @@ -228,7 +248,7 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *test pollErr := errors.New("polling error") sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{}, nil) + Return([]*model.Namespace{}, nil) sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return(test.OpId1, nil) sdApi.EXPECT().PollCreateNamespace(context.TODO(), test.OpId1). @@ -246,7 +266,7 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_CreateNsError(t * nsErr := errors.New("create namespace error") sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Resource{}, nil) + Return([]*model.Namespace{}, nil) sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return("", nsErr) @@ -267,12 +287,49 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { // TODO: Add unit tests } +func TestServiceDiscoveryClient_getNamespace_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + sdc := getTestSdClient(t, sdApi) + sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) + + namespace, _ := sdc.getNamespace(context.TODO(), test.NsName) + assert.Equal(t, test.GetTestHttpNamespace(), namespace, "Namespace found in the cache") +} + +func TestServiceDiscoveryClient_getNamespace_GetEmptyNamespace(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + sdc := getTestSdClient(t, sdApi) + sdc.namespaceCache.Add(test.NsName, nil, time.Minute) + + namespace, err := sdc.getNamespace(context.TODO(), test.NsName) + assert.Nil(t, namespace, "Namespace not found in the cache") + assert.Nil(t, err, "No errors with empty namespace") +} + +func TestServiceDiscoveryClient_getCachedNamespace_ErrorCasting(t *testing.T) { + sdc := getTestSdClient(t, nil) + sdc.namespaceCache.Add(test.NsName, struct{ dummy string }{"dummy"}, time.Minute) + + namespace, exists, err := sdc.getCachedNamespace(test.NsName) + assert.True(t, exists, "Cache exists") + assert.Nil(t, namespace, "No corresponding cached value found") + assert.Equal(t, fmt.Sprintf("failed to cast the cached value for the namespace %s", test.NsName), fmt.Sprint(err), "Got the error for improper casting") +} + func getTestSdClient(t *testing.T, sdApi ServiceDiscoveryApi) serviceDiscoveryClient { return serviceDiscoveryClient{ - log: testing2.TestLogger{T: t}, - sdApi: sdApi, - namespaceIdCache: cache.NewLRUExpireCache(1024), - serviceIdCache: cache.NewLRUExpireCache(1024), - endpointCache: cache.NewLRUExpireCache(1024), + log: testing2.TestLogger{T: t}, + sdApi: sdApi, + namespaceCache: cache.NewLRUExpireCache(1024), + serviceIdCache: cache.NewLRUExpireCache(1024), + endpointCache: cache.NewLRUExpireCache(1024), } } diff --git a/pkg/model/types.go b/pkg/model/types.go index d2be9d73..2669b6ea 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -16,6 +16,22 @@ type Resource struct { Name string } +const ( + HttpNamespaceType NamespaceType = "HTTP" + DnsPrivateNamespaceType NamespaceType = "DNS_PRIVATE" + // UnsupportedNamespaceType Placeholder NamespaceType to denote not supported values + UnsupportedNamespaceType NamespaceType = "" +) + +type NamespaceType string + +// Namespace hold namespace attributes +type Namespace struct { + Id string + Name string + Type NamespaceType +} + // Service holds namespace and endpoint state for a named service. type Service struct { Namespace string @@ -105,3 +121,18 @@ func (e *Endpoint) String() string { func EndpointIdFromIPAddress(address string) string { return strings.Replace(address, ".", "_", -1) } + +func ConvertNamespaceType(nsType types.NamespaceType) (namespaceType NamespaceType) { + switch nsType { + case types.NamespaceTypeDnsPrivate: + return DnsPrivateNamespaceType + case types.NamespaceTypeHttp: + return HttpNamespaceType + default: + return UnsupportedNamespaceType + } +} + +func (namespaceType *NamespaceType) IsUnsupported() bool { + return *namespaceType == UnsupportedNamespaceType +} diff --git a/test/test-constants.go b/test/test-constants.go index f0699062..095d421f 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -15,6 +15,22 @@ const ( OpStart = 1 ) +func GetTestHttpNamespace() *model.Namespace { + return &model.Namespace{ + Id: NsId, + Name: NsName, + Type: model.HttpNamespaceType, + } +} + +func GetTestDnsNamespace() *model.Namespace { + return &model.Namespace{ + Id: NsId, + Name: NsName, + Type: model.DnsPrivateNamespaceType, + } +} + func GetTestService() *model.Service { endPt := GetTestEndpoint() return &model.Service{ From 8f8df953a1b53be67112686e3985ebc26865fb20 Mon Sep 17 00:00:00 2001 From: Jaromir Vanek Date: Tue, 26 Oct 2021 14:32:19 -0700 Subject: [PATCH 019/163] Fix controllers RBAC permissions (#54) --- config/default/kustomization.yaml | 4 +- config/rbac/role.yaml | 46 +++++++++++---------- pkg/controllers/cloudmap_controller.go | 5 ++- pkg/controllers/serviceexport_controller.go | 2 + 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 757b1dd2..08158e36 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,12 +1,12 @@ # Adds namespace to all resources. -namespace: migration-system +namespace: cloud-map-mcs-system # Value of this field is prepended to the # names of all resources, e.g. a deployment named # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. -namePrefix: migration- +namePrefix: cloud-map-mcs- # Labels to add to all resources and selectors. #commonLabels: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6035e516..b6b86bea 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -7,52 +7,56 @@ metadata: name: manager-role rules: - apiGroups: - - multicluster.x-k8s.io + - "" resources: - - serviceexports + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - services verbs: + - create - get - list - - patch - - update - watch - apiGroups: - - multicluster.x-k8s.io + - discovery.k8s.io resources: - - serviceexports/finalizers + - endpointslices verbs: + - create - get - - update + - list + - watch - apiGroups: - multicluster.x-k8s.io resources: - - serviceimports + - serviceexports verbs: - - create - get - list - patch - update - watch - apiGroups: - - "" + - multicluster.x-k8s.io resources: - - namespaces - - services - - endpoints + - serviceexports/finalizers verbs: - get - - list - - watch - - create - update - apiGroups: - - discovery.k8s.io + - multicluster.x-k8s.io resources: - - endpointslices + - serviceimports verbs: - - list - - watch - - get - create + - delete + - get + - list + - patch - update + - watch diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 1c53fc33..2355c7d1 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -34,7 +34,10 @@ type CloudMapReconciler struct { logr.Logger } -// +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceimports,verbs=get;list;update;patch +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch +// +kubebuilder:rbac:groups="",resources=services,verbs=create;get;list;watch +// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;create;watch +// +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceimports,verbs=create;get;list;watch;update;patch;delete // Start implements manager.Runnable func (r *CloudMapReconciler) Start(ctx context.Context) error { diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index c9efb541..51fdd338 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -47,6 +47,8 @@ type ServiceExportReconciler struct { Cloudmap cloudmap.ServiceDiscoveryClient } +// +kubebuilder:rbac:groups="",resources=services,verbs=get +// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;watch;create // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceexports,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceexports/finalizers,verbs=get;update From 49091408f817ac45d2a6df7844621b563c00f27b Mon Sep 17 00:00:00 2001 From: Jaromir Vanek Date: Wed, 27 Oct 2021 21:04:21 -0700 Subject: [PATCH 020/163] Annotate imported EndpointSlices with multicluster label (#56) --- pkg/controllers/cloudmap_controller.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 2355c7d1..fe48c4ab 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -24,7 +24,12 @@ const ( // TODO move to configuration syncPeriod = 2 * time.Second + // DerivedServiceAnnotation annotates a ServiceImport with derived Service name DerivedServiceAnnotation = "multicluster.k8s.aws/derived-service" + + // LabelServiceName is used to indicate the name of multi-cluster service + // that an EndpointSlice belongs to. + LabelServiceImportName = "multicluster.kubernetes.io/service-name" ) // CloudMapReconciler reconciles state of Cloud Map services with local ServiceImport objects @@ -148,7 +153,7 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se return err } - err = r.updateEndpointSlices(ctx, svc, existingService) + err = r.updateEndpointSlices(ctx, svcImport, svc, existingService) if err != nil { return err } @@ -203,7 +208,7 @@ func (r *CloudMapReconciler) createDerivedService(ctx context.Context, svc *mode return nil } -func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, cloudMapService *model.Service, svc *v1.Service) error { +func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *v1alpha1.ServiceImport, cloudMapService *model.Service, svc *v1.Service) error { existingSlicesList := v1beta1.EndpointSliceList{} var existingSlices []*v1beta1.EndpointSlice if err := r.Client.List(ctx, &existingSlicesList, @@ -212,7 +217,7 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, cloudMapS } if len(existingSlicesList.Items) == 0 { // create new endpoint slice - existingSlices = createEndpointSlicesStruct(cloudMapService, svc) + existingSlices = createEndpointSlicesStruct(svcImport, cloudMapService, svc) for _, slice := range existingSlices { if err := r.Client.Create(ctx, slice); err != nil { return err @@ -252,7 +257,7 @@ func createDerivedServiceStruct(svc *model.Service, svcImport *v1alpha1.ServiceI } } -func createEndpointSlicesStruct(cloudMapSvc *model.Service, svc *v1.Service) []*v1beta1.EndpointSlice { +func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc *model.Service, svc *v1.Service) []*v1beta1.EndpointSlice { slices := make([]*v1beta1.EndpointSlice, 0) t := true @@ -279,7 +284,10 @@ func createEndpointSlicesStruct(cloudMapSvc *model.Service, svc *v1.Service) []* slices = append(slices, &v1beta1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{v1beta1.LabelServiceName: svc.Name}, + Labels: map[string]string{ + v1beta1.LabelServiceName: svc.Name, // derived Service name + LabelServiceImportName: svcImport.Name, // original ServiceImport name + }, GenerateName: svc.Name + "-", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(svc, schema.GroupVersionKind{ Version: svc.TypeMeta.APIVersion, From 075bf73941f0fe6ce1f967158b3507ecc6d5cabb Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 28 Oct 2021 18:27:54 -0700 Subject: [PATCH 021/163] Add API happy case unit tests (#58) --- pkg/cloudmap/api_test.go | 114 ++++++++++++++++++++++++++++++++++++++- test/test-constants.go | 21 ++++---- 2 files changed, 124 insertions(+), 11 deletions(-) diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index b194d979..8a4455e2 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" @@ -37,7 +38,7 @@ func TestServiceDiscoveryApi_ListNamespaces_HappyCase(t *testing.T) { namespaces, _ := sdApi.ListNamespaces(context.TODO()) assert.True(t, len(namespaces) == 1) - assert.Equal(t, test.GetTestDnsNamespace(), namespaces[0], "") + assert.Equal(t, test.GetTestDnsNamespace(), namespaces[0], "No error for happy case") } func TestServiceDiscoveryApi_ListNamespaces_SkipPublicDNSNotSupported(t *testing.T) { @@ -60,6 +61,105 @@ func TestServiceDiscoveryApi_ListNamespaces_SkipPublicDNSNotSupported(t *testing assert.True(t, len(namespaces) == 0, "Successfully skipped DNS_PUBLIC from the output") } +func TestServiceDiscoveryApi_ListServices_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + filter := types.ServiceFilter{ + Name: types.ServiceFilterNameNamespaceId, + Values: []string{test.NsId}, + } + + awsFacade.EXPECT().ListServices(context.TODO(), &sd.ListServicesInput{Filters: []types.ServiceFilter{filter}}). + Return(&sd.ListServicesOutput{Services: []types.ServiceSummary{ + {Id: aws.String(test.SvcId), Name: aws.String(test.SvcName)}, + }}, nil) + + svcs, err := sdApi.ListServices(context.TODO(), test.NsId) + assert.Nil(t, err, "No error for happy case") + assert.True(t, len(svcs) == 1) + assert.Equal(t, svcs[0], &model.Resource{Id: test.SvcId, Name: test.SvcName}) +} + +func TestServiceDiscoveryApi_ListInstances_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + awsFacade.EXPECT().ListInstances(context.TODO(), gomock.Any()). + Return(&sd.ListInstancesOutput{ + Instances: []types.InstanceSummary{{ + Id: aws.String(test.EndptId1), + Attributes: map[string]string{ + model.Ipv4Attr: test.EndptIp1, + model.PortAttr: test.EndptPortStr1, + }}}, + }, nil) + + insts, err := sdApi.ListInstances(context.TODO(), test.SvcId) + assert.Nil(t, err, "No error for happy case") + assert.True(t, len(insts) == 1) + assert.Equal(t, insts[0], test.GetTestEndpoint()) +} + +func TestServiceDiscoveryApi_ListOperations_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + filters := make([]types.OperationFilter, 0) + awsFacade.EXPECT().ListOperations(context.TODO(), &sd.ListOperationsInput{Filters: filters}). + Return(&sd.ListOperationsOutput{ + Operations: []types.OperationSummary{ + {Id: aws.String(test.OpId1), Status: types.OperationStatusSuccess}, + }}, nil) + + ops, err := sdApi.ListOperations(context.TODO(), filters) + assert.Nil(t, err, "No error for happy case") + assert.True(t, len(ops) == 1) + assert.Equal(t, ops[test.OpId1], types.OperationStatusSuccess) + +} + +func TestServiceDiscoveryApi_GetOperation_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + expectedOp := &types.Operation{Id: aws.String(test.OpId1), Status: types.OperationStatusPending} + awsFacade.EXPECT().GetOperation(context.TODO(), &sd.GetOperationInput{OperationId: aws.String(test.OpId1)}). + Return(&sd.GetOperationOutput{Operation: expectedOp}, nil) + + op, err := sdApi.GetOperation(context.TODO(), test.OpId1) + assert.Nil(t, err, "No error for happy case") + assert.Equal(t, expectedOp, op) +} + +func TestServiceDiscoveryApi_CreateHttNamespace_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + sdApi := getServiceDiscoveryApi(t, awsFacade) + + awsFacade.EXPECT().CreateHttpNamespace(context.TODO(), &sd.CreateHttpNamespaceInput{Name: aws.String(test.NsName)}). + Return(&sd.CreateHttpNamespaceOutput{OperationId: aws.String(test.OpId1)}, nil) + + opId, err := sdApi.CreateHttpNamespace(context.TODO(), test.NsName) + assert.Nil(t, err, "No error for happy case") + assert.Equal(t, test.OpId1, opId) + +} + func TestServiceDiscoveryApi_CreateService_CreateForHttpNamespace(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() @@ -129,6 +229,18 @@ func TestServiceDiscoveryApi_CreateService_ThrowError(t *testing.T) { assert.Equal(t, "dummy error", fmt.Sprint(err), "Got error") } +func TestServiceDiscoveryApi_RegisterInstance_HappyCase(t *testing.T) { + // TODO: Add unit tests +} + +func TestServiceDiscoveryApi_DeregisterInstance_HappyCase(t *testing.T) { + // TODO: Add unit tests +} + +func TestServiceDiscoveryApi_PollCreateNamespace_HappyCase(t *testing.T) { + // TODO: Add unit tests +} + func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmap.MockAwsFacade) serviceDiscoveryApi { return serviceDiscoveryApi{ log: testingLogger.TestLogger{T: t}, diff --git a/test/test-constants.go b/test/test-constants.go index 095d421f..86614c93 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -3,16 +3,17 @@ package test import "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" const ( - NsName = "ns-name" - NsId = "ns-id" - SvcName = "svc-name" - SvcId = "svc-id" - EndptId1 = "endpoint-id-1" - EndptIp1 = "endpoint-ip-1" - EndptPort1 = 2 - OpId1 = "operation-id-1" - OpId2 = "operation-id-2" - OpStart = 1 + NsName = "ns-name" + NsId = "ns-id" + SvcName = "svc-name" + SvcId = "svc-id" + EndptId1 = "endpoint-id-1" + EndptIp1 = "192.168.0.1" + EndptPort1 = 2 + EndptPortStr1 = "2" + OpId1 = "operation-id-1" + OpId2 = "operation-id-2" + OpStart = 1 ) func GetTestHttpNamespace() *model.Namespace { From a1328237970d9f95a3e80c5d7e5b121d89b227a2 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Mon, 1 Nov 2021 10:27:16 -0700 Subject: [PATCH 022/163] Add janitor to clean up Cloud Map integration test resources (#57) --- Makefile | 2 + integration/janitor/api.go | 40 ++++++++++ integration/janitor/api_test.go | 51 +++++++++++++ integration/janitor/aws_facade.go | 30 ++++++++ integration/janitor/janitor.go | 109 ++++++++++++++++++++++++++++ integration/janitor/janitor_test.go | 76 +++++++++++++++++++ integration/janitor/runner/main.go | 15 ++++ pkg/cloudmap/api.go | 34 +++------ pkg/cloudmap/api_test.go | 19 +++-- pkg/cloudmap/client.go | 15 +++- pkg/cloudmap/client_test.go | 17 ++++- test/test-constants.go | 5 +- 12 files changed, 372 insertions(+), 41 deletions(-) create mode 100644 integration/janitor/api.go create mode 100644 integration/janitor/api_test.go create mode 100644 integration/janitor/aws_facade.go create mode 100644 integration/janitor/janitor.go create mode 100644 integration/janitor/janitor_test.go create mode 100644 integration/janitor/runner/main.go diff --git a/Makefile b/Makefile index 76f36b10..f831e0eb 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,8 @@ generate-mocks: mockgen $(MOCKGEN) --source pkg/cloudmap/operation_collector.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_collector_mock.go --package cloudmap $(MOCKGEN) --source pkg/cloudmap/api.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/api_mock.go --package cloudmap $(MOCKGEN) --source pkg/cloudmap/aws_facade.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/aws_facade_mock.go --package cloudmap + $(MOCKGEN) --source integration/janitor/api.go --destination $(MOCKS_DESTINATION)/integration/janitor/api_mock.go --package janitor + $(MOCKGEN) --source integration/janitor/aws_facade.go --destination $(MOCKS_DESTINATION)/integration/janitor/aws_facade_mock.go --package janitor CONTROLLER_GEN = $(shell pwd)/bin/controller-gen diff --git a/integration/janitor/api.go b/integration/janitor/api.go new file mode 100644 index 00000000..d74561d7 --- /dev/null +++ b/integration/janitor/api.go @@ -0,0 +1,40 @@ +package janitor + +import ( + "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-sdk-go-v2/aws" + sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" +) + +type ServiceDiscoveryJanitorApi interface { + DeleteNamespace(ctx context.Context, namespaceId string) (operationId string, err error) + DeleteService(ctx context.Context, serviceId string) error + cloudmap.ServiceDiscoveryApi +} + +type serviceDiscoveryJanitorApi struct { + cloudmap.ServiceDiscoveryApi + janitorFacade SdkJanitorFacade +} + +func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config) ServiceDiscoveryJanitorApi { + return &serviceDiscoveryJanitorApi{ + ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg), + janitorFacade: NewSdkJanitorFacadeFromConfig(cfg), + } +} + +func (api *serviceDiscoveryJanitorApi) DeleteNamespace(ctx context.Context, nsId string) (opId string, err error) { + out, err := api.janitorFacade.DeleteNamespace(ctx, &sd.DeleteNamespaceInput{Id: &nsId}) + if err != nil { + return "", err + } + + return aws.ToString(out.OperationId), nil +} + +func (api *serviceDiscoveryJanitorApi) DeleteService(ctx context.Context, svcId string) error { + _, err := api.janitorFacade.DeleteService(ctx, &sd.DeleteServiceInput{Id: &svcId}) + return err +} diff --git a/integration/janitor/api_test.go b/integration/janitor/api_test.go new file mode 100644 index 00000000..f0d84fbc --- /dev/null +++ b/integration/janitor/api_test.go @@ -0,0 +1,51 @@ +package janitor + +import ( + "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/aws/aws-sdk-go-v2/aws" + sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewServiceDiscoveryJanitorApiFromConfig(t *testing.T) { + assert.NotNil(t, NewServiceDiscoveryJanitorApiFromConfig(&aws.Config{})) +} + +func TestServiceDiscoveryJanitorApi_DeleteNamespace_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + mocksdk := janitor.NewMockSdkJanitorFacade(mockController) + jApi := getJanitorApi(t, mocksdk) + + mocksdk.EXPECT().DeleteNamespace(context.TODO(), &sd.DeleteNamespaceInput{Id: aws.String(test.NsId)}). + Return(&sd.DeleteNamespaceOutput{OperationId: aws.String(test.OpId1)}, nil) + + opId, err := jApi.DeleteNamespace(context.TODO(), test.NsId) + assert.Nil(t, err, "No error for happy case") + assert.Equal(t, test.OpId1, opId) +} + +func TestServiceDiscoveryJanitorApi_DeleteService_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + mocksdk := janitor.NewMockSdkJanitorFacade(mockController) + jApi := getJanitorApi(t, mocksdk) + + mocksdk.EXPECT().DeleteService(context.TODO(), &sd.DeleteServiceInput{Id: aws.String(test.SvcId)}). + Return(&sd.DeleteServiceOutput{}, nil) + + err := jApi.DeleteService(context.TODO(), test.SvcId) + assert.Nil(t, err, "No error for happy case") +} + +func getJanitorApi(t *testing.T, sdk *janitor.MockSdkJanitorFacade) ServiceDiscoveryJanitorApi { + return &serviceDiscoveryJanitorApi{ + janitorFacade: sdk, + } +} diff --git a/integration/janitor/aws_facade.go b/integration/janitor/aws_facade.go new file mode 100644 index 00000000..00752cd0 --- /dev/null +++ b/integration/janitor/aws_facade.go @@ -0,0 +1,30 @@ +package janitor + +import ( + "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-sdk-go-v2/aws" + sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" +) + +// SdkJanitorFacade extends the minimal surface area of ServiceDiscovery API calls of the client +// for integration test janitor operations. +type SdkJanitorFacade interface { + // DeleteNamespace provides ServiceDiscovery DeleteNamespace wrapper interface. + DeleteNamespace(context.Context, *sd.DeleteNamespaceInput, ...func(*sd.Options)) (*sd.DeleteNamespaceOutput, error) + + // DeleteService provides ServiceDiscovery DeleteService wrapper interface. + DeleteService(context.Context, *sd.DeleteServiceInput, ...func(*sd.Options)) (*sd.DeleteServiceOutput, error) + + cloudmap.AwsFacade +} + +type sdkJanitorFacade struct { + *sd.Client +} + +// NewSdkJanitorFacadeFromConfig creates a new AWS facade from an AWS client config +// extended for integration test janitor operations. +func NewSdkJanitorFacadeFromConfig(cfg *aws.Config) SdkJanitorFacade { + return &sdkJanitorFacade{sd.NewFromConfig(*cfg)} +} diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go new file mode 100644 index 00000000..f2b55e02 --- /dev/null +++ b/integration/janitor/janitor.go @@ -0,0 +1,109 @@ +package janitor + +import ( + "context" + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "os" +) + +// CloudMapJanitor handles AWS Cloud Map resource cleanup during integration tests. +type CloudMapJanitor interface { + // Cleanup removes all instances, services and the namespace from AWS Cloud Map for a given namespace name. + Cleanup(ctx context.Context, nsName string) +} + +type cloudMapJanitor struct { + sdApi ServiceDiscoveryJanitorApi + fail func() +} + +// NewDefaultJanitor returns a new janitor object. +func NewDefaultJanitor() CloudMapJanitor { + awsCfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(os.Getenv("AWS_REGION")), + ) + + if err != nil { + fmt.Printf("unable to configure AWS session: %s", err.Error()) + os.Exit(1) + } + + return &cloudMapJanitor{ + sdApi: NewServiceDiscoveryJanitorApiFromConfig(&awsCfg), + fail: func() { os.Exit(1) }, + } +} + +func (j *cloudMapJanitor) Cleanup(ctx context.Context, nsName string) { + fmt.Printf("Cleaning up all test resources in Cloud Map for namespace : %s\n", nsName) + + nsList, err := j.sdApi.ListNamespaces(ctx) + j.checkOrFail(err, "", "could not find namespace to clean") + + var nsId string + for _, ns := range nsList { + if ns.Name == nsName { + nsId = ns.Id + } + } + + if nsId == "" { + fmt.Println("namespace does not exist in account, nothing to clean") + return + } + + fmt.Printf("found namespace to clean: %s\n", nsId) + + svcs, err := j.sdApi.ListServices(ctx, nsId) + j.checkOrFail(err, + fmt.Sprintf("namespace has %d services to clean", len(svcs)), + "could not find services to clean") + + for _, svc := range svcs { + fmt.Printf("found service to clean: %s\n", svc.Id) + j.deregisterInstances(ctx, svc.Id) + + delSvcErr := j.sdApi.DeleteService(ctx, svc.Id) + j.checkOrFail(delSvcErr, "service deleted", "could not cleanup service") + } + + opId, err := j.sdApi.DeleteNamespace(ctx, nsId) + if err == nil { + fmt.Println("namespace delete in progress") + _, err = j.sdApi.PollNamespaceOperation(ctx, opId) + } + j.checkOrFail(err, "clean up successful", "could not cleanup namespace") +} + +func (j *cloudMapJanitor) deregisterInstances(ctx context.Context, svcId string) { + insts, err := j.sdApi.ListInstances(ctx, svcId) + j.checkOrFail(err, + fmt.Sprintf("service has %d instances to clean", len(insts)), + "could not list instances to cleanup") + + opColl := cloudmap.NewOperationCollector() + for _, inst := range insts { + instId := aws.ToString(inst.Id) + fmt.Printf("found instance to clean: %s\n", instId) + opColl.Add(func() (opId string, err error) { + return j.sdApi.DeregisterInstance(ctx, svcId, instId) + }) + } + + opErr := cloudmap.NewDeregisterInstancePoller(j.sdApi, svcId, opColl.Collect(), opColl.GetStartTime()).Poll(ctx) + j.checkOrFail(opErr, "instances de-registered", "could not cleanup instances") +} + +func (j *cloudMapJanitor) checkOrFail(err error, successMsg string, failMsg string) { + if err != nil { + fmt.Printf("%s: %s\n", failMsg, err.Error()) + j.fail() + } + + if successMsg != "" { + fmt.Println(successMsg) + } +} diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go new file mode 100644 index 00000000..9f44c2d8 --- /dev/null +++ b/integration/janitor/janitor_test.go @@ -0,0 +1,76 @@ +package janitor + +import ( + "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +type testJanitor struct { + janitor *cloudMapJanitor + mockApi *janitor.MockServiceDiscoveryJanitorApi + failed *bool + close func() +} + +func TestNewDefaultJanitor(t *testing.T) { + assert.NotNil(t, NewDefaultJanitor()) +} + +func TestCleanupHappyCase(t *testing.T) { + tj := getTestJanitor(t) + defer tj.close() + + tj.mockApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Namespace{{Id: test.NsId, Name: test.NsName}}, nil) + tj.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). + Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) + tj.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). + Return([]types.InstanceSummary{{Id: aws.String(test.EndptId1)}}, nil) + + tj.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1). + Return(test.OpId1, nil) + tj.mockApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). + Return(map[string]types.OperationStatus{test.OpId1: types.OperationStatusSuccess}, nil) + tj.mockApi.EXPECT().DeleteService(context.TODO(), test.SvcId). + Return(nil) + tj.mockApi.EXPECT().DeleteNamespace(context.TODO(), test.NsId). + Return(test.OpId2, nil) + tj.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId2). + Return(test.NsId, nil) + + tj.janitor.Cleanup(context.TODO(), test.NsName) + assert.False(t, *tj.failed) +} + +func TestCleanupNothingToClean(t *testing.T) { + tj := getTestJanitor(t) + defer tj.close() + + tj.mockApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Namespace{}, nil) + + tj.janitor.Cleanup(context.TODO(), test.NsName) + assert.False(t, *tj.failed) +} + +func getTestJanitor(t *testing.T) *testJanitor { + mockController := gomock.NewController(t) + api := janitor.NewMockServiceDiscoveryJanitorApi(mockController) + failed := false + return &testJanitor{ + janitor: &cloudMapJanitor{ + sdApi: api, + fail: func() { failed = true }, + }, + mockApi: api, + failed: &failed, + close: func() { mockController.Finish() }, + } +} diff --git a/integration/janitor/runner/main.go b/integration/janitor/runner/main.go new file mode 100644 index 00000000..9a2dafad --- /dev/null +++ b/integration/janitor/runner/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/janitor" +) + +const ( + e2eNs = "aws-cloud-map-mcs-e2e" +) + +func main() { + j := janitor.NewDefaultJanitor() + j.Cleanup(context.TODO(), e2eNs) +} diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index 5f1df066..11181b06 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -27,7 +27,7 @@ type ServiceDiscoveryApi interface { ListServices(ctx context.Context, namespaceId string) (services []*model.Resource, err error) // ListInstances returns a list of service instances registered to a given service. - ListInstances(ctx context.Context, serviceId string) ([]*model.Endpoint, error) + ListInstances(ctx context.Context, serviceId string) ([]types.InstanceSummary, error) // ListOperations returns a map of operations to their status matching a list of filters. ListOperations(ctx context.Context, opFilters []types.OperationFilter) (operationStatusMap map[string]types.OperationStatus, err error) @@ -47,8 +47,8 @@ type ServiceDiscoveryApi interface { // DeregisterInstance de-registers a service instance in Cloud Map. DeregisterInstance(ctx context.Context, serviceId string, instanceId string) (operationId string, err error) - // PollCreateNamespace polls a create namespace operation, and returns the namespace ID. - PollCreateNamespace(ctx context.Context, operationId string) (namespaceId string, err error) + // PollNamespaceOperation polls a namespace operation, and returns the namespace ID. + PollNamespaceOperation(ctx context.Context, operationId string) (namespaceId string, err error) } type serviceDiscoveryApi struct { @@ -64,8 +64,7 @@ func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { } } -func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*model.Namespace, error) { - namespaces := make([]*model.Namespace, 0) +func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) (namespaces []*model.Namespace, err error) { pages := sd.NewListNamespacesPaginator(sdApi.awsFacade, &sd.ListNamespacesInput{}) for pages.HasMorePages() { @@ -88,8 +87,7 @@ func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) ([]*model. return namespaces, nil } -func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) ([]*model.Resource, error) { - svcs := make([]*model.Resource, 0) +func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) (svcs []*model.Resource, err error) { filter := types.ServiceFilter{ Name: types.ServiceFilterNameNamespaceId, @@ -115,30 +113,19 @@ func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) return svcs, nil } -func (sdApi *serviceDiscoveryApi) ListInstances(ctx context.Context, svcId string) ([]*model.Endpoint, error) { - endpts := make([]*model.Endpoint, 0) - +func (sdApi *serviceDiscoveryApi) ListInstances(ctx context.Context, svcId string) (insts []types.InstanceSummary, err error) { pages := sd.NewListInstancesPaginator(sdApi.awsFacade, &sd.ListInstancesInput{ServiceId: &svcId}) for pages.HasMorePages() { output, err := pages.NextPage(ctx) if err != nil { - return endpts, err + return insts, err } - for _, inst := range output.Instances { - endpt, endptErr := model.NewEndpointFromInstance(&inst) - - if endptErr != nil { - sdApi.log.Info(fmt.Sprintf("skipping instance %s to endpoint conversion: %s", *inst.Id, endptErr.Error())) - continue - } - - endpts = append(endpts, endpt) - } + insts = append(insts, output.Instances...) } - return endpts, nil + return insts, nil } func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters []types.OperationFilter) (opStatusMap map[string]types.OperationStatus, err error) { @@ -248,7 +235,7 @@ func (sdApi *serviceDiscoveryApi) DeregisterInstance(ctx context.Context, svcId } -func (sdApi *serviceDiscoveryApi) PollCreateNamespace(ctx context.Context, opId string) (nsId string, err error) { +func (sdApi *serviceDiscoveryApi) PollNamespaceOperation(ctx context.Context, opId string) (nsId string, err error) { err = wait.Poll(defaultOperationPollInterval, defaultOperationPollTimeout, func() (done bool, err error) { sdApi.log.Info("polling operation", "opId", opId) op, err := sdApi.GetOperation(ctx, opId) @@ -263,7 +250,6 @@ func (sdApi *serviceDiscoveryApi) PollCreateNamespace(ctx context.Context, opId if op.Status == types.OperationStatusSuccess { nsId = op.Targets[string(types.OperationTargetTypeNamespace)] - sdApi.log.Info("namespace created", "nsId", nsId) return true, nil } diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 8a4455e2..58ccfd33 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -93,18 +93,17 @@ func TestServiceDiscoveryApi_ListInstances_HappyCase(t *testing.T) { awsFacade.EXPECT().ListInstances(context.TODO(), gomock.Any()). Return(&sd.ListInstancesOutput{ - Instances: []types.InstanceSummary{{ - Id: aws.String(test.EndptId1), - Attributes: map[string]string{ - model.Ipv4Attr: test.EndptIp1, - model.PortAttr: test.EndptPortStr1, - }}}, + Instances: []types.InstanceSummary{ + {Id: aws.String(test.EndptId1)}, + {Id: aws.String(test.EndptId2)}, + }, }, nil) insts, err := sdApi.ListInstances(context.TODO(), test.SvcId) assert.Nil(t, err, "No error for happy case") - assert.True(t, len(insts) == 1) - assert.Equal(t, insts[0], test.GetTestEndpoint()) + assert.True(t, len(insts) == 2) + assert.Equal(t, test.EndptId1, *insts[0].Id) + assert.Equal(t, test.EndptId2, *insts[1].Id) } func TestServiceDiscoveryApi_ListOperations_HappyCase(t *testing.T) { @@ -241,8 +240,8 @@ func TestServiceDiscoveryApi_PollCreateNamespace_HappyCase(t *testing.T) { // TODO: Add unit tests } -func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmap.MockAwsFacade) serviceDiscoveryApi { - return serviceDiscoveryApi{ +func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmap.MockAwsFacade) ServiceDiscoveryApi { + return &serviceDiscoveryApi{ log: testingLogger.TestLogger{T: t}, awsFacade: awsFacade, } diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 7515a178..323afadd 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -221,11 +221,20 @@ func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId return cachedValue.([]*model.Endpoint), nil } - endpts, err = sdc.sdApi.ListInstances(ctx, serviceId) + insts, err := sdc.sdApi.ListInstances(ctx, serviceId) if err != nil { return nil, err } + for _, inst := range insts { + endpt, endptErr := model.NewEndpointFromInstance(&inst) + if endptErr != nil { + sdc.log.Info(fmt.Sprintf("skipping instance %s to endpoint conversion: %s", *inst.Id, endptErr.Error())) + continue + } + endpts = append(endpts, endpt) + } + sdc.cacheEndpoints(serviceId, endpts) return endpts, nil @@ -296,11 +305,13 @@ func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName s return nil, err } - nsId, err := sdc.sdApi.PollCreateNamespace(ctx, opId) + nsId, err := sdc.sdApi.PollNamespaceOperation(ctx, opId) if err != nil { return nil, err } + sdc.log.Info("namespace created", "nsId", nsId) + // Cache the Namespace, by default we always create namespace of type HTTP namespace = &model.Namespace{ Id: nsId, diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 9c641e6d..ecb91a2e 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -31,7 +32,15 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { sdApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). - Return([]*model.Endpoint{test.GetTestEndpoint()}, nil) + Return([]types.InstanceSummary{ + { + Id: aws.String(test.EndptId1), + Attributes: map[string]string{ + model.Ipv4Attr: test.EndptIp1, + model.PortAttr: test.EndptPortStr1, + }, + }, + }, nil) sdc := getTestSdClient(t, sdApi) svcs, err := sdc.ListServices(context.TODO(), test.NsName) @@ -107,7 +116,7 @@ func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { sdApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). - Return([]*model.Endpoint{}, endptErr) + Return([]types.InstanceSummary{}, endptErr) sdc := getTestSdClient(t, sdApi) svcs, err := sdc.ListServices(context.TODO(), test.NsName) @@ -229,7 +238,7 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_HappyCase(t *test Return([]*model.Namespace{}, nil) sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return(test.OpId1, nil) - sdApi.EXPECT().PollCreateNamespace(context.TODO(), test.OpId1). + sdApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). Return(test.NsId, nil) sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) @@ -251,7 +260,7 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *test Return([]*model.Namespace{}, nil) sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return(test.OpId1, nil) - sdApi.EXPECT().PollCreateNamespace(context.TODO(), test.OpId1). + sdApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). Return("", pollErr) sdc := getTestSdClient(t, sdApi) diff --git a/test/test-constants.go b/test/test-constants.go index 86614c93..227aeda6 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -1,6 +1,8 @@ package test -import "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" +import ( + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" +) const ( NsName = "ns-name" @@ -8,6 +10,7 @@ const ( SvcName = "svc-name" SvcId = "svc-id" EndptId1 = "endpoint-id-1" + EndptId2 = "endpoint-id-2" EndptIp1 = "192.168.0.1" EndptPort1 = 2 EndptPortStr1 = "2" From b49417a90ee18cb649a1b5fef183aea2c95fbc32 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Wed, 3 Nov 2021 20:03:06 -0700 Subject: [PATCH 023/163] Fix multiple instance registration/deregistration (#60) --- pkg/cloudmap/client.go | 7 ++- pkg/cloudmap/client_test.go | 62 +++++++++++++++++++++++++- pkg/controllers/cloudmap_controller.go | 3 +- test/test-constants.go | 7 ++- 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 323afadd..5e6e9fbc 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -157,8 +157,10 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName opCollector := NewOperationCollector() for _, endpt := range endpts { + endptId := endpt.Id + endptAttrs := endpt.GetCloudMapAttributes() opCollector.Add(func() (opId string, err error) { - return sdc.sdApi.RegisterInstance(ctx, svcId, endpt.Id, endpt.GetCloudMapAttributes()) + return sdc.sdApi.RegisterInstance(ctx, svcId, endptId, endptAttrs) }) } @@ -195,8 +197,9 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s opCollector := NewOperationCollector() for _, endpt := range endpts { + endptId := endpt.Id opCollector.Add(func() (opId string, err error) { - return sdc.sdApi.DeregisterInstance(ctx, svcId, endpt.Id) + return sdc.sdApi.DeregisterInstance(ctx, svcId, endptId) }) } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index ecb91a2e..bcd80213 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -288,12 +288,70 @@ func TestServiceDiscoveryClient_GetService(t *testing.T) { // TODO: Add unit tests } -func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { +func TestServiceDiscoveryClient_GetService_CachedValues(t *testing.T) { // TODO: Add unit tests } +func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + sdc := getTestSdClient(t, sdApi) + sdc.namespaceCache.Add(test.NsName, test.NsId, time.Minute) + sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) + + attrs1 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp1, "AWS_INSTANCE_PORT": test.EndptPortStr1} + attrs2 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp2, "AWS_INSTANCE_PORT": test.EndptPortStr2} + + sdApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). + Return(test.OpId1, nil) + sdApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId2, attrs2). + Return(test.OpId2, nil) + sdApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). + Return(map[string]types.OperationStatus{ + test.OpId1: types.OperationStatusSuccess, + test.OpId2: types.OperationStatusSuccess}, nil) + + err := sdc.RegisterEndpoints(context.TODO(), test.NsName, test.SvcName, + []*model.Endpoint{ + { + Id: test.EndptId1, + IP: test.EndptIp1, + Port: test.EndptPort1, + }, + { + Id: test.EndptId2, + IP: test.EndptIp2, + Port: test.EndptPort2, + }, + }) + + assert.Nil(t, err) +} + func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { - // TODO: Add unit tests + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + + sdc := getTestSdClient(t, sdApi) + sdc.namespaceCache.Add(test.NsName, test.NsId, time.Minute) + sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) + + sdApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).Return(test.OpId1, nil) + sdApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2).Return(test.OpId2, nil) + sdApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). + Return(map[string]types.OperationStatus{ + test.OpId1: types.OperationStatusSuccess, + test.OpId2: types.OperationStatusSuccess}, nil) + + err := sdc.DeleteEndpoints(context.TODO(), test.NsName, test.SvcName, + []*model.Endpoint{{Id: test.EndptId1}, {Id: test.EndptId2}}) + + assert.Nil(t, err) } func TestServiceDiscoveryClient_getNamespace_HappyCase(t *testing.T) { diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index fe48c4ab..2da2dfc0 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -27,8 +27,7 @@ const ( // DerivedServiceAnnotation annotates a ServiceImport with derived Service name DerivedServiceAnnotation = "multicluster.k8s.aws/derived-service" - // LabelServiceName is used to indicate the name of multi-cluster service - // that an EndpointSlice belongs to. + // LabelServiceImportName indicates the name of multi-cluster service that an EndpointSlice belongs to. LabelServiceImportName = "multicluster.kubernetes.io/service-name" ) diff --git a/test/test-constants.go b/test/test-constants.go index 227aeda6..4001c673 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -12,8 +12,11 @@ const ( EndptId1 = "endpoint-id-1" EndptId2 = "endpoint-id-2" EndptIp1 = "192.168.0.1" - EndptPort1 = 2 - EndptPortStr1 = "2" + EndptIp2 = "192.168.0.2" + EndptPort1 = 1 + EndptPortStr1 = "1" + EndptPort2 = 2 + EndptPortStr2 = "2" OpId1 = "operation-id-1" OpId2 = "operation-id-2" OpStart = 1 From 15f818986c312f84e839e54e1ac892ebc6a33b04 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 4 Nov 2021 13:01:21 -0700 Subject: [PATCH 024/163] Improve unit test coverage (#61) --- pkg/cloudmap/api_test.go | 79 +++++++++++++++++++++++++++++++++++-- pkg/cloudmap/client_test.go | 65 ++++++++++++++++++++++++++---- test/test-constants.go | 12 +++++- 3 files changed, 142 insertions(+), 14 deletions(-) diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 58ccfd33..7d583f4b 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -2,6 +2,7 @@ package cloudmap import ( "context" + "errors" "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -229,15 +230,85 @@ func TestServiceDiscoveryApi_CreateService_ThrowError(t *testing.T) { } func TestServiceDiscoveryApi_RegisterInstance_HappyCase(t *testing.T) { - // TODO: Add unit tests + mockController := gomock.NewController(t) + defer mockController.Finish() + + attrs := map[string]string{"a": "b"} + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade.EXPECT().RegisterInstance(context.TODO(), + &sd.RegisterInstanceInput{ + ServiceId: aws.String(test.SvcId), + InstanceId: aws.String(test.EndptId1), + Attributes: attrs}). + Return(&sd.RegisterInstanceOutput{OperationId: aws.String(test.OpId1)}, nil) + + sdApi := getServiceDiscoveryApi(t, awsFacade) + opId, err := sdApi.RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs) + assert.Nil(t, err) + assert.Equal(t, test.OpId1, opId) +} + +func TestServiceDiscoveryApi_RegisterInstance_Error(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdkErr := errors.New("fail") + awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade.EXPECT().RegisterInstance(context.TODO(), gomock.Any()).Return(nil, sdkErr) + + sdApi := getServiceDiscoveryApi(t, awsFacade) + _, err := sdApi.RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, map[string]string{}) + assert.Equal(t, sdkErr, err) } func TestServiceDiscoveryApi_DeregisterInstance_HappyCase(t *testing.T) { - // TODO: Add unit tests + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade.EXPECT().DeregisterInstance(context.TODO(), + &sd.DeregisterInstanceInput{ + ServiceId: aws.String(test.SvcId), + InstanceId: aws.String(test.EndptId1)}). + Return(&sd.DeregisterInstanceOutput{OperationId: aws.String(test.OpId1)}, nil) + + sdApi := getServiceDiscoveryApi(t, awsFacade) + opId, err := sdApi.DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1) + assert.Nil(t, err) + assert.Equal(t, test.OpId1, opId) +} + +func TestServiceDiscoveryApi_DeregisterInstance_Error(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdkErr := errors.New("fail") + awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade.EXPECT().DeregisterInstance(context.TODO(), gomock.Any()).Return(nil, sdkErr) + + sdApi := getServiceDiscoveryApi(t, awsFacade) + _, err := sdApi.DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1) + assert.Equal(t, sdkErr, err) } -func TestServiceDiscoveryApi_PollCreateNamespace_HappyCase(t *testing.T) { - // TODO: Add unit tests +func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade.EXPECT().GetOperation(context.TODO(), &sd.GetOperationInput{OperationId: aws.String(test.OpId1)}). + Return(&sd.GetOperationOutput{Operation: &types.Operation{Status: types.OperationStatusPending}}, nil) + + awsFacade.EXPECT().GetOperation(context.TODO(), &sd.GetOperationInput{OperationId: aws.String(test.OpId1)}). + Return(&sd.GetOperationOutput{Operation: &types.Operation{Status: types.OperationStatusSuccess, + Targets: map[string]string{string(types.OperationTargetTypeNamespace): test.NsId}}}, nil) + + sdApi := getServiceDiscoveryApi(t, awsFacade) + + nsId, err := sdApi.PollNamespaceOperation(context.TODO(), test.OpId1) + assert.Nil(t, err) + assert.Equal(t, test.NsId, nsId) } func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmap.MockAwsFacade) ServiceDiscoveryApi { diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index bcd80213..3b0c13b4 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -40,6 +40,13 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { model.PortAttr: test.EndptPortStr1, }, }, + { + Id: aws.String(test.EndptId2), + Attributes: map[string]string{ + model.Ipv4Attr: test.EndptIp2, + model.PortAttr: test.EndptPortStr2, + }, + }, }, nil) sdc := getTestSdClient(t, sdApi) @@ -52,7 +59,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") cachedEndpts, _ := sdc.endpointCache.Get(test.SvcId) - assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint()}, cachedEndpts, "Happy case caches endpoints") + assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, cachedEndpts, "Happy case caches endpoints") } func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T) { @@ -65,7 +72,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T sdc := getTestSdClient(t, sdApi) sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) - sdc.endpointCache.Add(test.SvcId, []*model.Endpoint{test.GetTestEndpoint()}, time.Minute) + sdc.endpointCache.Add(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, time.Minute) svcs, err := sdc.ListServices(context.TODO(), test.NsName) assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) @@ -284,12 +291,51 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_CreateNsError(t * assert.Equal(t, nsErr, err) } -func TestServiceDiscoveryClient_GetService(t *testing.T) { - // TODO: Add unit tests +func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi.EXPECT().ListNamespaces(context.TODO()).Return([]*model.Namespace{{Id: test.NsId, Name: test.NsName}}, nil) + sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) + sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). + Return([]types.InstanceSummary{ + { + Id: aws.String(test.EndptId1), + Attributes: map[string]string{ + model.Ipv4Attr: test.EndptIp1, + model.PortAttr: test.EndptPortStr1, + }, + }, + { + Id: aws.String(test.EndptId2), + Attributes: map[string]string{ + model.Ipv4Attr: test.EndptIp2, + model.PortAttr: test.EndptPortStr2, + }, + }, + }, nil) + sdc := getTestSdClient(t, sdApi) + + svc, err := sdc.GetService(context.TODO(), test.NsName, test.SvcName) + assert.Nil(t, err) + assert.Equal(t, test.GetTestService(), svc) } func TestServiceDiscoveryClient_GetService_CachedValues(t *testing.T) { - // TODO: Add unit tests + mockController := gomock.NewController(t) + defer mockController.Finish() + + sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdc := getTestSdClient(t, sdApi) + sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) + sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) + sdc.endpointCache.Add(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, time.Minute) + + svc, err := sdc.GetService(context.TODO(), test.NsName, test.SvcName) + assert.Nil(t, err) + assert.Equal(t, test.GetTestService(), svc) } func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { @@ -297,10 +343,9 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { defer mockController.Finish() sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdc := getTestSdClient(t, sdApi) - sdc.namespaceCache.Add(test.NsName, test.NsId, time.Minute) sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) + sdc.endpointCache.Add(test.SvcId, model.Endpoint{}, time.Minute) attrs1 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp1, "AWS_INSTANCE_PORT": test.EndptPortStr1} attrs2 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp2, "AWS_INSTANCE_PORT": test.EndptPortStr2} @@ -329,6 +374,8 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { }) assert.Nil(t, err) + _, entryCached := sdc.endpointCache.Get(test.SvcId) + assert.False(t, entryCached, "Cache entry evicted after register") } func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { @@ -338,8 +385,8 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) sdc := getTestSdClient(t, sdApi) - sdc.namespaceCache.Add(test.NsName, test.NsId, time.Minute) sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) + sdc.endpointCache.Add(test.SvcId, model.Endpoint{}, time.Minute) sdApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).Return(test.OpId1, nil) sdApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2).Return(test.OpId2, nil) @@ -352,6 +399,8 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { []*model.Endpoint{{Id: test.EndptId1}, {Id: test.EndptId2}}) assert.Nil(t, err) + _, entryCached := sdc.endpointCache.Get(test.SvcId) + assert.False(t, entryCached, "Cache entry evicted after de-register") } func TestServiceDiscoveryClient_getNamespace_HappyCase(t *testing.T) { diff --git a/test/test-constants.go b/test/test-constants.go index 4001c673..6954f4e6 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -39,11 +39,10 @@ func GetTestDnsNamespace() *model.Namespace { } func GetTestService() *model.Service { - endPt := GetTestEndpoint() return &model.Service{ Namespace: NsName, Name: SvcName, - Endpoints: []*model.Endpoint{endPt}, + Endpoints: []*model.Endpoint{GetTestEndpoint(), GetTestEndpoint2()}, } } @@ -55,3 +54,12 @@ func GetTestEndpoint() *model.Endpoint { Attributes: make(map[string]string, 0), } } + +func GetTestEndpoint2() *model.Endpoint { + return &model.Endpoint{ + Id: EndptId2, + IP: EndptIp2, + Port: EndptPort2, + Attributes: make(map[string]string, 0), + } +} From 4382eff0643ba64e6ad9c62f0270b122f03d5ddf Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Fri, 5 Nov 2021 17:29:06 -0700 Subject: [PATCH 025/163] Set AWS_REGION (#63) --- CONTRIBUTING.md | 117 ++++++++++++++---- Makefile | 8 +- README.md | 10 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- config/default/kustomization.yaml | 2 +- config/manager/aws.properties | 1 + config/manager/kustomization.yaml | 6 +- config/manager/manager.yaml | 6 + main.go | 11 +- ...eployment.yaml => example-deployment.yaml} | 2 +- ...demo-service.yaml => example-service.yaml} | 4 +- ...export.yaml => example-serviceexport.yaml} | 4 +- 13 files changed, 129 insertions(+), 46 deletions(-) create mode 100644 config/manager/aws.properties rename samples/{demo-deployment.yaml => example-deployment.yaml} (94%) rename samples/{demo-service.yaml => example-service.yaml} (74%) rename samples/{demo-export.yaml => example-serviceexport.yaml} (64%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7236b8ce..65f1fb48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,24 @@ # Contributing Guidelines +* [Getting Started](#getting-started) + + [Local Setup](#local-setup) + - [Prerequisites](#prerequisites) + - [Cluster Setup](#cluster-setup) + - [Run the controller from outside the cluster](#run-the-controller-from-outside-the-cluster) + - [Build and deploy to the cluster](#build-and-deploy-to-the-cluster) + - [Run unit tests](#run-unit-tests) + - [Cleanup](#cleanup) + + [Deploying to a cluster](#deploying-to-a-cluster) + + [Build and push docker image to ECR](#build-and-push-docker-image-to-ecr) + - [Deployment](#deployment) + - [Uninstallation](#uninstallation) +* [Reporting Bugs/Feature Requests](#reporting-bugsfeature-requests) +* [Contributing via Pull Requests](#contributing-via-pull-requests) +* [Finding contributions to work on](#finding-contributions-to-work-on) +* [Code of Conduct](#code-of-conduct) +* [Security issue notifications](#security-issue-notifications) +* [Licensing](#licensing) + Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community. @@ -8,7 +27,7 @@ information to effectively respond to your bug report or contribution. ## Getting Started -### Build and Run locally +### Local Setup #### Prerequisites @@ -16,7 +35,7 @@ In order to build and run locally: * Make sure to have `kubectl` [installed](https://kubernetes.io/docs/tasks/tools/install-kubectl/), at least version `1.15` or above. * Make sure to have `kind` [installed](https://kind.sigs.k8s.io/docs/user/quick-start/#installation). -* Make sure, you have created a [HttpNamespace](https://docs.aws.amazon.com/cloud-map/latest/api/API_CreateHttpNamespace.html) in AWS Cloud Map. The examples below assumes the namespace name to be `demo` +* Make sure, you have access to AWS Cloud Map. As per exercise below, AWS Cloud Map namespace `example` of the type [HttpNamespace](https://docs.aws.amazon.com/cloud-map/latest/api/API_CreateHttpNamespace.html) will be automatically created. Note that this walk-through assumes throughout to operate in the `us-west-2` region. @@ -24,9 +43,10 @@ Note that this walk-through assumes throughout to operate in the `us-west-2` reg export AWS_REGION=us-west-2 ``` -#### Cluster provisioning +#### Cluster Setup Spin up a local Kubernetes cluster using `kind` + ```sh kind create cluster --name my-cluster # Creating cluster "my-cluster" ... @@ -34,12 +54,23 @@ kind create cluster --name my-cluster ``` When completed, set the kubectl context + ```sh kind export kubeconfig --name my-cluster # Set kubectl context to "kind-my-cluster" ``` +Create `example` namespace in the cluster + +```sh +kubectl create namespace example +# namespace/example created +``` + +#### Run the controller from outside the cluster + To register the custom CRDs (`ServiceImport`, `ServiceExport`) in the cluster and create installers + ```sh make install # ... @@ -48,36 +79,68 @@ make install ``` To run the controller, run the following command. The controller runs in an infinite loop so open another terminal to create CRDs. + ```sh make run ``` -Create `demo` namespace -```sh -kubectl create namespace demo -# namespace/demo created -``` +Apply deployment, service and serviceexport configs -Apply deployment, service and export configs ```sh -kubectl apply -f samples/demo-deployment.yaml +kubectl apply -f samples/example-deployment.yaml # deployment.apps/nginx-deployment created -kubectl apply -f samples/demo-service.yaml -# service/demo-service created -kubectl apply -f samples/demo-export.yaml -# serviceexport.multicluster.x-k8s.io/demo-service created +kubectl apply -f samples/example-service.yaml +# service/my-service created +kubectl apply -f samples/example-serviceexport.yaml +# serviceexport.multicluster.x-k8s.io/my-service created ``` Check running controller if it correctly detects newly created resources + +``` +controllers.ServiceExport updating Cloud Map service {"serviceexport": "example/my-service", "namespace": "example", "name": "my-service"} +cloudmap fetching a service {"namespaceName": "example", "serviceName": "my-service"} +cloudmap creating a new service {"namespace": "example", "name": "my-service"} +``` + +#### Build and deploy to the cluster + +Build a `controller:local` (`--no-cache` args can be set to update the existing docker image) + +```shell +make docker-build IMG=controller:local ARGS="--no-cache" +# ... +# docker build --no-cache -t controller:local . +# ... +# +``` + +Load the controller docker image into the kind cluster `my-cluster` +```shell +kind load docker-image controller:local --name my-cluster +# Image: "controller:local" with ID "sha256:xxx" not yet present on node "my-cluster-control-plane", loading... +``` + +> ⚠ **The controller still needs credentials to interact to AWS SDK.** We are not supporting this configuration out of box. There are multiple ways to achieve this within the cluster. + +Finally, create the controller resources in the cluster. +```shell +make deploy IMG=controller:local AWS_REGION=us-west-2 +# customresourcedefinition.apiextensions.k8s.io/serviceexports.multicluster.x-k8s.io configured +# customresourcedefinition.apiextensions.k8s.io/serviceimports.multicluster.x-k8s.io created +# ... +# deployment.apps/cloud-map-mcs-controller-manager created ``` -controllers.ServiceExport updating Cloud Map service {"serviceexport": "demo/demo-service", "namespace": "demo", "name": "demo-service"} -cloudmap fetching a service {"namespaceName": "demo", "serviceName": "demo-service"} -cloudmap creating a new service {"namespace": "demo", "name": "demo-service"} + +Stream the controller logs +```shell +kubectl logs -f -l control-plane=controller-manager -c manager -n cloud-map-mcs-system ``` #### Run unit tests Use command below to run the unit test + ```sh make test ``` @@ -85,11 +148,19 @@ make test #### Cleanup Use the command below to clean all the generated files + ```sh make clean ``` +Use the command below to remove the CRDs from the cluster + +```sh +make uninstall +``` + Use the command below to delete the cluster `my-cluster` + ```sh kind delete cluster --name my-cluster ``` @@ -108,23 +179,17 @@ make docker-build docker-push IMG=.dkr.ecr..amazona #### Deployment -You must specify AWS access credentials for the operator. You can do so via environment variables, or by creating them. +You must setup the AWS access credentials. Also, set the AWS_REGION environment variable like `export AWS_REGION=us-west-2` -Any one of below three options will work: ```sh # With an IAM user. make deploy - -# Use an existing access key -OPERATOR_AWS_ACCESS_KEY_ID=xxx OPERATOR_AWS_SECRET_KEY=yyy make deploy - -# Use an AWS profile -OPERATOR_AWS_PROFILE=default make deploy ``` #### Uninstallation -To remove the operator from your cluster, run +To remove the controller from your cluster, run + ```sh make undeploy ``` diff --git a/Makefile b/Makefile index f831e0eb..0b31c20d 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ PKG:=github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version IMG ?= controller:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" +# AWS Region +AWS_REGION ?= us-east-1 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -67,10 +69,10 @@ run: manifests generate generate-mocks fmt vet ## Run a controller from your hos go run ./main.go docker-build: test ## Build docker image with the manager. - docker build -t ${IMG} . + docker build $(ARGS) -t ${IMG} . docker-push: ## Push docker image with the manager. - docker push ${IMG} + docker push $(ARGS) ${IMG} clean: rm -rf $(MOCKS_DESTINATION) bin/ testbin/ cover.out @@ -85,7 +87,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | kubectl apply -f - + AWS_REGION=${AWS_REGION} $(KUSTOMIZE) build config/default | kubectl apply -f - undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete -f - diff --git a/README.md b/README.md index 3af46301..e77aed12 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ AWS Cloud Map multi-cluster service discovery for Kubernetes (K8s) is a controll First, install the controller with latest release on at least 2 AWS EKS clusters. Nodes must have sufficient IAM permissions to perform CloudMap operations. +> **_NOTE:_** AWS region environment variable should be set like `export AWS_REGION=us-west-2` + ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release" ``` @@ -51,9 +53,9 @@ metadata: *See the `samples` directory for a set of example yaml files to set up a service and export it. To apply the sample files run* ```sh kubectl create namespace demo -kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/demo-deployment.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/demo-service.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/demo-export.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-deployment.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-service.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-serviceexport.yaml ``` ### Import services @@ -67,6 +69,8 @@ kubectl get ServiceImport -A AWS Cloud Map MCS Controller for K8s adheres to the [SemVer](https://semver.org/) specification. Each release updates the major version tag (eg. `vX`), a major/minor version tag (eg. `vX.Y`) and a major/minor/patch version tag (eg. `vX.Y.Z`). To see a full list of all releases, refer to our [Github releases page](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/releases). +> **_NOTE:_** AWS region environment variable should be set like `export AWS_REGION=us-west-2` + To install from a release run ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release[?ref=*git version tag*]" diff --git a/config/controller_install_latest/kustomization.yaml b/config/controller_install_latest/kustomization.yaml index 1fbcbf37..38076ff6 100644 --- a/config/controller_install_latest/kustomization.yaml +++ b/config/controller_install_latest/kustomization.yaml @@ -1,4 +1,4 @@ -bases: +resources: - ../default images: diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index bc1bf2e1..98ff571e 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -1,4 +1,4 @@ -bases: +resources: - ../default images: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 08158e36..e196a2b7 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -12,7 +12,7 @@ namePrefix: cloud-map-mcs- #commonLabels: # someName: someValue -bases: +resources: - ../crd - ../rbac - ../manager diff --git a/config/manager/aws.properties b/config/manager/aws.properties new file mode 100644 index 00000000..2c72a849 --- /dev/null +++ b/config/manager/aws.properties @@ -0,0 +1 @@ +AWS_REGION diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5e793dd1..b01ba653 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -8,8 +8,10 @@ configMapGenerator: - files: - controller_manager_config.yaml name: manager-config -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization +- envs: + - aws.properties + name: aws-config + images: - name: controller newName: controller diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 79adfe72..a75d3168 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -52,5 +52,11 @@ spec: requests: cpu: 100m memory: 20Mi + env: + - name: AWS_REGION + valueFrom: + configMapKeyRef: + name: aws-config + key: AWS_REGION serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/main.go b/main.go index 741d5826..c1b448da 100644 --- a/main.go +++ b/main.go @@ -86,19 +86,22 @@ func main() { } // TODO: configure session + awsRegion := os.Getenv("AWS_REGION") + setupLog.Info("configuring AWS session", "AWS_REGION", awsRegion) awsCfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(os.Getenv("AWS_REGION")), + config.WithRegion(awsRegion), ) if err != nil { - setupLog.Error(err, "unable to configure AWS session") + setupLog.Error(err, "unable to configure AWS session", "AWS_REGION", awsRegion) os.Exit(1) } + serviceDiscoveryClient := cloudmap.NewServiceDiscoveryClient(&awsCfg) if err = (&controllers.ServiceExportReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("ServiceExport"), Scheme: mgr.GetScheme(), - Cloudmap: cloudmap.NewServiceDiscoveryClient(&awsCfg), + Cloudmap: serviceDiscoveryClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ServiceExport") os.Exit(1) @@ -106,7 +109,7 @@ func main() { cloudMapReconciler := &controllers.CloudMapReconciler{ Client: mgr.GetClient(), - Cloudmap: cloudmap.NewServiceDiscoveryClient(&awsCfg), + Cloudmap: serviceDiscoveryClient, Logger: ctrl.Log.WithName("controllers").WithName("CloudMap"), } diff --git a/samples/demo-deployment.yaml b/samples/example-deployment.yaml similarity index 94% rename from samples/demo-deployment.yaml rename to samples/example-deployment.yaml index 3063ef78..79dbcd7c 100644 --- a/samples/demo-deployment.yaml +++ b/samples/example-deployment.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - namespace: demo + namespace: example name: nginx-deployment labels: app: nginx diff --git a/samples/demo-service.yaml b/samples/example-service.yaml similarity index 74% rename from samples/demo-service.yaml rename to samples/example-service.yaml index 07bfeb7b..b204accb 100644 --- a/samples/demo-service.yaml +++ b/samples/example-service.yaml @@ -1,8 +1,8 @@ kind: Service apiVersion: v1 metadata: - namespace: demo - name: demo-service + namespace: example + name: my-service spec: selector: app: nginx diff --git a/samples/demo-export.yaml b/samples/example-serviceexport.yaml similarity index 64% rename from samples/demo-export.yaml rename to samples/example-serviceexport.yaml index 1930db08..8c094161 100644 --- a/samples/demo-export.yaml +++ b/samples/example-serviceexport.yaml @@ -1,5 +1,5 @@ kind: ServiceExport apiVersion: multicluster.x-k8s.io/v1alpha1 metadata: - namespace: demo - name: demo-service + namespace: example + name: my-service From 7509127f764916f1696dcc6cf0db633cdcac972c Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:13:35 -0700 Subject: [PATCH 026/163] Update controller docker version to v0.1.1 (#64) --- config/controller_install_release/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index 98ff571e..45fec726 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -4,4 +4,4 @@ resources: images: - name: controller newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s - newTag: v0.1.0 + newTag: v0.1.1 From b6a739c2a93e4b047f681bd30b8186cc38525436 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Mon, 8 Nov 2021 12:44:28 -0800 Subject: [PATCH 027/163] Extract Cloud Map client cache (#67) --- Makefile | 1 + pkg/cloudmap/cache.go | 149 ++++++++++++++ pkg/cloudmap/cache_test.go | 114 +++++++++++ pkg/cloudmap/client.go | 107 +++------- pkg/cloudmap/client_test.go | 398 +++++++++++++++--------------------- 5 files changed, 457 insertions(+), 312 deletions(-) create mode 100644 pkg/cloudmap/cache.go create mode 100644 pkg/cloudmap/cache_test.go diff --git a/Makefile b/Makefile index 0b31c20d..b6c40baf 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi MOCKS_DESTINATION=mocks generate-mocks: mockgen $(MOCKGEN) --source pkg/cloudmap/client.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/client_mock.go --package cloudmap + $(MOCKGEN) --source pkg/cloudmap/cache.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/cache_mock.go --package cloudmap $(MOCKGEN) --source pkg/cloudmap/operation_poller.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_poller_mock.go --package cloudmap $(MOCKGEN) --source pkg/cloudmap/operation_collector.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_collector_mock.go --package cloudmap $(MOCKGEN) --source pkg/cloudmap/api.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/api_mock.go --package cloudmap diff --git a/pkg/cloudmap/cache.go b/pkg/cloudmap/cache.go new file mode 100644 index 00000000..99b4826e --- /dev/null +++ b/pkg/cloudmap/cache.go @@ -0,0 +1,149 @@ +package cloudmap + +import ( + "errors" + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/cache" + ctrl "sigs.k8s.io/controller-runtime" + "time" +) + +const ( + nsKeyPrefix = "ns" + svcKeyPrefix = "svc" + endptKeyPrefix = "endpt" + + defaultCacheSize = 1024 + defaultNsTTL = 2 * time.Minute + defaultSvcTTL = 2 * time.Minute + defaultEndptTTL = 5 * time.Second +) + +type ServiceDiscoveryClientCache interface { + GetNamespace(namespaceName string) (namespace *model.Namespace, found bool) + CacheNamespace(namespace *model.Namespace) + CacheNilNamespace(namespaceName string) + GetServiceId(namespaceName string, serviceName string) (serviceId string, found bool) + CacheServiceId(namespaceName string, serviceName string, serviceId string) + GetEndpoints(serviceId string) (endpoints []*model.Endpoint, found bool) + CacheEndpoints(serviceId string, endpoints []*model.Endpoint) + EvictEndpoints(serviceId string) +} + +type sdCache struct { + log logr.Logger + cache *cache.LRUExpireCache + config sdCacheConfig +} + +type sdCacheConfig struct { + nsTTL time.Duration + svcTTL time.Duration + endptTTL time.Duration +} + +func NewDefaultServiceDiscoveryClientCache() ServiceDiscoveryClientCache { + return &sdCache{ + log: ctrl.Log.WithName("cloudmap"), + cache: cache.NewLRUExpireCache(defaultCacheSize), + config: sdCacheConfig{ + nsTTL: defaultNsTTL, + svcTTL: defaultSvcTTL, + endptTTL: defaultEndptTTL, + }} +} + +func (sdCache *sdCache) GetNamespace(nsName string) (ns *model.Namespace, found bool) { + key := sdCache.buildNsKey(nsName) + entry, exists := sdCache.cache.Get(key) + if !exists { + return nil, false + } + + if entry == nil { + return nil, true + } + + nsEntry, ok := entry.(model.Namespace) + if !ok { + sdCache.log.Error(errors.New("failed to retrieve namespace from cache"), "", "nsName", nsName) + sdCache.cache.Remove(key) + return nil, false + } + + return &nsEntry, true +} + +func (sdCache *sdCache) CacheNamespace(namespace *model.Namespace) { + key := sdCache.buildNsKey(namespace.Name) + sdCache.cache.Add(key, *namespace, sdCache.config.nsTTL) +} + +func (sdCache *sdCache) CacheNilNamespace(nsName string) { + key := sdCache.buildNsKey(nsName) + sdCache.cache.Add(key, nil, sdCache.config.nsTTL) +} + +func (sdCache *sdCache) GetServiceId(nsName string, svcName string) (svcId string, found bool) { + key := sdCache.buildSvcKey(nsName, svcName) + entry, exists := sdCache.cache.Get(key) + if !exists { + return "", false + } + + svcId, ok := entry.(string) + if !ok { + sdCache.log.Error(errors.New("failed to retrieve service ID from cache"), "", + "nsName", nsName, "svcName", svcName) + sdCache.cache.Remove(key) + return "", false + } + + return svcId, true +} + +func (sdCache *sdCache) CacheServiceId(nsName string, svcName string, svcId string) { + key := sdCache.buildSvcKey(nsName, svcName) + sdCache.cache.Add(key, svcId, sdCache.config.svcTTL) +} + +func (sdCache *sdCache) GetEndpoints(svcId string) (endpts []*model.Endpoint, found bool) { + key := sdCache.buildEndptsKey(svcId) + entry, exists := sdCache.cache.Get(key) + if !exists { + return nil, false + } + + endpts, ok := entry.([]*model.Endpoint) + if !ok { + sdCache.log.Error(errors.New("failed to retrieve endpoints from cache"), "", "svcId", svcId) + sdCache.cache.Remove(key) + return nil, false + } + + return endpts, true +} + +func (sdCache *sdCache) CacheEndpoints(svcId string, endpts []*model.Endpoint) { + key := sdCache.buildEndptsKey(svcId) + sdCache.cache.Add(key, endpts, sdCache.config.endptTTL) +} + +func (sdCache *sdCache) EvictEndpoints(svcId string) { + key := sdCache.buildEndptsKey(svcId) + sdCache.cache.Remove(key) +} + +func (sdCache *sdCache) buildNsKey(nsName string) (cacheKey string) { + return fmt.Sprintf("%s:%s", nsKeyPrefix, nsName) +} + +func (sdCache *sdCache) buildSvcKey(nsName string, svcName string) (cacheKey string) { + return fmt.Sprintf("%s:%s:%s", svcKeyPrefix, nsName, svcName) +} + +func (sdCache *sdCache) buildEndptsKey(svcId string) string { + return fmt.Sprintf("%s:%s", endptKeyPrefix, svcId) +} diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go new file mode 100644 index 00000000..3a840a76 --- /dev/null +++ b/pkg/cloudmap/cache_test.go @@ -0,0 +1,114 @@ +package cloudmap + +import ( + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestNewDefaultServiceDiscoveryClientCache(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) + + assert.Equal(t, defaultNsTTL, sdc.config.nsTTL) + assert.Equal(t, defaultSvcTTL, sdc.config.svcTTL) + assert.Equal(t, defaultEndptTTL, sdc.config.endptTTL) +} + +func TestServiceDiscoveryClientCacheGetNamespace_Found(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + sdc.CacheNamespace(test.GetTestHttpNamespace()) + + ns, found := sdc.GetNamespace(test.NsName) + assert.True(t, found) + assert.Equal(t, test.GetTestHttpNamespace(), ns) +} + +func TestServiceDiscoveryClientCacheGetNamespace_NotFound(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + + ns, found := sdc.GetNamespace(test.NsName) + assert.False(t, found) + assert.Nil(t, ns) +} + +func TestServiceDiscoveryClientCacheGetNamespace_Nil(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + sdc.CacheNilNamespace(test.NsName) + + ns, found := sdc.GetNamespace(test.NsName) + assert.True(t, found) + assert.Nil(t, ns) +} + +func TestServiceDiscoveryClientCacheGetNamespace_Corrupt(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) + sdc.cache.Add(sdc.buildNsKey(test.NsName), &model.Resource{}, time.Minute) + + ns, found := sdc.GetNamespace(test.NsName) + assert.False(t, found) + assert.Nil(t, ns) +} + +func TestServiceDiscoveryClientCacheGetServiceId_Found(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + sdc.CacheServiceId(test.NsName, test.SvcName, test.SvcId) + + svcId, found := sdc.GetServiceId(test.NsName, test.SvcName) + assert.True(t, found) + assert.Equal(t, test.SvcId, svcId) +} + +func TestServiceDiscoveryClientCacheGetServiceId_NotFound(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + + svcId, found := sdc.GetServiceId(test.NsName, test.SvcName) + assert.False(t, found) + assert.Empty(t, svcId) +} + +func TestServiceDiscoveryClientCacheGetServiceId_Corrupt(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) + + sdc.cache.Add(sdc.buildSvcKey(test.NsName, test.SvcName), &model.Resource{}, time.Minute) + svcId, found := sdc.GetServiceId(test.NsName, test.SvcName) + assert.False(t, found) + assert.Empty(t, svcId) +} + +func TestServiceDiscoveryClientCacheGetEndpoints_Found(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + sdc.CacheEndpoints(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + + endpts, found := sdc.GetEndpoints(test.SvcId) + assert.True(t, found) + assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, endpts) +} + +func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + + endpts, found := sdc.GetEndpoints(test.SvcId) + assert.False(t, found) + assert.Nil(t, endpts) +} + +func TestServiceDiscoveryClientCacheGetEndpoints_Corrupt(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) + + sdc.cache.Add(sdc.buildEndptsKey(test.SvcId), &model.Resource{}, time.Minute) + endpts, found := sdc.GetEndpoints(test.SvcId) + assert.False(t, found) + assert.Nil(t, endpts) +} + +func TestServiceDiscoveryClientEvictEndpoints(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + sdc.CacheEndpoints(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + sdc.EvictEndpoints(test.SvcId) + + endpts, found := sdc.GetEndpoints(test.SvcId) + assert.False(t, found) + assert.Nil(t, endpts) +} diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 5e6e9fbc..56eb1ca4 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -6,18 +6,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/util/cache" ctrl "sigs.k8s.io/controller-runtime" - "time" -) - -const ( - defaultNamespaceCacheTTL = 2 * time.Minute - defaultNamespaceCacheSize = 100 - defaultServiceIdCacheTTL = 2 * time.Minute - defaultServiceIdCacheSize = 1024 - defaultEndpointsCacheTTL = 5 * time.Second - defaultEndpointsCacheSize = 1024 ) // ServiceDiscoveryClient provides the service endpoint management functionality required by the AWS Cloud Map @@ -40,21 +29,17 @@ type ServiceDiscoveryClient interface { } type serviceDiscoveryClient struct { - log logr.Logger - sdApi ServiceDiscoveryApi - namespaceCache *cache.LRUExpireCache - serviceIdCache *cache.LRUExpireCache - endpointCache *cache.LRUExpireCache + log logr.Logger + sdApi ServiceDiscoveryApi + cache ServiceDiscoveryClientCache } // NewServiceDiscoveryClient creates a new service discovery client for AWS Cloud Map from a given AWS client config. func NewServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: ctrl.Log.WithName("cloudmap"), - sdApi: NewServiceDiscoveryApiFromConfig(cfg), - namespaceCache: cache.NewLRUExpireCache(defaultNamespaceCacheSize), - serviceIdCache: cache.NewLRUExpireCache(defaultServiceIdCacheSize), - endpointCache: cache.NewLRUExpireCache(defaultEndpointsCacheSize), + log: ctrl.Log.WithName("cloudmap"), + sdApi: NewServiceDiscoveryApiFromConfig(cfg), + cache: NewDefaultServiceDiscoveryClientCache(), } } @@ -64,13 +49,14 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri return svcs, err } + // TODO: Cache list svcSums, err := sdc.sdApi.ListServices(ctx, namespace.Id) if err != nil { return svcs, err } for _, svcSum := range svcSums { - sdc.cacheServiceId(nsName, svcSum.Name, svcSum.Id) + sdc.cache.CacheServiceId(nsName, svcSum.Name, svcSum.Id) endpts, endptsErr := sdc.listEndpoints(ctx, svcSum.Id) if endptsErr != nil { @@ -108,7 +94,7 @@ func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName str return err } - sdc.cacheServiceId(nsName, svcName, svcId) + sdc.cache.CacheServiceId(nsName, svcName, svcId) return nil } @@ -167,7 +153,7 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName err = NewRegisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) // Evict cache entry so next list call reflects changes - sdc.evictEndpoints(svcId) + sdc.cache.EvictEndpoints(svcId) if err != nil { return err @@ -206,7 +192,7 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s err = NewDeregisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) // Evict cache entry so next list call reflects changes - sdc.evictEndpoints(svcId) + sdc.cache.EvictEndpoints(svcId) if err != nil { return err } @@ -219,9 +205,8 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s } func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId string) (endpts []*model.Endpoint, err error) { - - if cachedValue, exists := sdc.endpointCache.Get(serviceId); exists { - return cachedValue.([]*model.Endpoint), nil + if endpts, found := sdc.cache.GetEndpoints(serviceId); found { + return endpts, nil } insts, err := sdc.sdApi.ListInstances(ctx, serviceId) @@ -238,18 +223,16 @@ func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId endpts = append(endpts, endpt) } - sdc.cacheEndpoints(serviceId, endpts) + sdc.cache.CacheEndpoints(serviceId, endpts) return endpts, nil } func (sdc *serviceDiscoveryClient) getNamespace(ctx context.Context, nsName string) (namespace *model.Namespace, err error) { // We are assuming a unique namespace name per account - if namespace, exists, err := sdc.getCachedNamespace(nsName); exists { - return namespace, err - } - if err != nil { - return nil, err + namespace, exists := sdc.cache.GetNamespace(nsName) + if exists { + return namespace, nil } namespaces, err := sdc.sdApi.ListNamespaces(ctx) @@ -258,7 +241,7 @@ func (sdc *serviceDiscoveryClient) getNamespace(ctx context.Context, nsName stri } for _, ns := range namespaces { - sdc.cacheNamespace(*ns) + sdc.cache.CacheNamespace(ns) // Set the return namespace if nsName == ns.Name { namespace = ns @@ -268,17 +251,16 @@ func (sdc *serviceDiscoveryClient) getNamespace(ctx context.Context, nsName stri if namespace == nil { // This will cache empty namespace for namespaces not in Cloud Map // This is so that we can avoid ListNamespaces call - sdc.cacheNilNamespace(nsName) + sdc.cache.CacheNilNamespace(nsName) } return namespace, nil } func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName string, svcName string) (svcId string, err error) { - cacheKey := sdc.buildServiceIdCacheKey(nsName, svcName) - - if cachedValue, exists := sdc.serviceIdCache.Get(cacheKey); exists { - return cachedValue.(string), nil + svcId, found := sdc.cache.GetServiceId(nsName, svcName) + if found { + return svcId, nil } namespace, err := sdc.getNamespace(ctx, nsName) @@ -292,7 +274,7 @@ func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName stri } for _, svc := range services { - sdc.cacheServiceId(nsName, svcName, svc.Id) + sdc.cache.CacheServiceId(nsName, svcName, svc.Id) if svc.Name == svcName { svcId = svc.Id } @@ -315,52 +297,13 @@ func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName s sdc.log.Info("namespace created", "nsId", nsId) - // Cache the Namespace, by default we always create namespace of type HTTP + // Default namespace type HTTP namespace = &model.Namespace{ Id: nsId, Name: nsName, Type: model.HttpNamespaceType, } - sdc.cacheNamespace(*namespace) + sdc.cache.CacheNamespace(namespace) return namespace, nil } - -func (sdc *serviceDiscoveryClient) getCachedNamespace(nsName string) (namespace *model.Namespace, exists bool, err error) { - if cachedValue, exists := sdc.namespaceCache.Get(nsName); exists { - if cachedValue == nil { - return nil, exists, nil - } - ns, ok := cachedValue.(model.Namespace) - if !ok { - return nil, exists, fmt.Errorf("failed to cast the cached value for the namespace %s", nsName) - } - return &ns, exists, nil - } - return nil, exists, nil -} - -func (sdc *serviceDiscoveryClient) cacheNamespace(namespace model.Namespace) { - sdc.namespaceCache.Add(namespace.Name, namespace, defaultNamespaceCacheTTL) -} - -func (sdc *serviceDiscoveryClient) cacheNilNamespace(nsName string) { - sdc.namespaceCache.Add(nsName, nil, defaultNamespaceCacheTTL) -} - -func (sdc *serviceDiscoveryClient) cacheServiceId(nsName string, svcName string, svcId string) { - cacheKey := sdc.buildServiceIdCacheKey(nsName, svcName) - sdc.serviceIdCache.Add(cacheKey, svcId, defaultServiceIdCacheTTL) -} - -func (sdc *serviceDiscoveryClient) cacheEndpoints(svcId string, endpts []*model.Endpoint) { - sdc.endpointCache.Add(svcId, endpts, defaultEndpointsCacheTTL) -} - -func (sdc *serviceDiscoveryClient) evictEndpoints(svcId string) { - sdc.endpointCache.Remove(svcId) -} - -func (sdc *serviceDiscoveryClient) buildServiceIdCacheKey(nsName string, svcName string) (cacheKey string) { - return fmt.Sprintf("%s/%s", nsName, svcName) -} diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 3b0c13b4..263fe94e 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -3,7 +3,6 @@ package cloudmap import ( "context" "errors" - "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -12,26 +11,36 @@ import ( testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/util/cache" "testing" - "time" ) +type testSdClient struct { + client *serviceDiscoveryClient + mockApi cloudmap.MockServiceDiscoveryApi + mockCache cloudmap.MockServiceDiscoveryClientCache + close func() +} + func TestNewServiceDiscoveryClient(t *testing.T) { sdc := NewServiceDiscoveryClient(&aws.Config{}) assert.NotNil(t, sdc) } func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) - sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) + + tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). + tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + + tc.mockCache.EXPECT().GetEndpoints(test.SvcId).Return(nil, false) + tc.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). Return([]types.InstanceSummary{ { Id: aws.String(test.EndptId1), @@ -48,258 +57,226 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { }, }, }, nil) + tc.mockCache.EXPECT().CacheEndpoints(test.SvcId, + []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) - sdc := getTestSdClient(t, sdApi) - svcs, err := sdc.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) assert.Nil(t, err, "No error for happy case") - - cachedNs, _ := sdc.namespaceCache.Get(test.NsName) - assert.Equal(t, *test.GetTestHttpNamespace(), cachedNs, "Happy case caches namespace ID") - cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) - assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") - cachedEndpts, _ := sdc.endpointCache.Get(test.SvcId) - assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, cachedEndpts, "Happy case caches endpoints") } func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() + + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) + tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) - sdc := getTestSdClient(t, sdApi) - sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) - sdc.endpointCache.Add(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, time.Minute) + tc.mockCache.EXPECT().GetEndpoints(test.SvcId). + Return([]*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, true) - svcs, err := sdc.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) assert.Nil(t, err, "No error for happy case") } func TestServiceDiscoveryClient_ListServices_NamespaceError(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() nsErr := errors.New("error listing namespaces") - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{}, nsErr) + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). + Return(nil, nsErr) - sdc := getTestSdClient(t, sdApi) - svcs, err := sdc.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, nsErr, err) assert.Empty(t, svcs) } func TestServiceDiscoveryClient_ListServices_ServiceError(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - svcErr := errors.New("error listing services") + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) - sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + svcErr := errors.New("error listing services") + tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{}, svcErr) - sdc := getTestSdClient(t, sdApi) - svcs, err := sdc.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, svcErr, err) assert.Empty(t, svcs) } func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - endptErr := errors.New("error listing endpoints") - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) - sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) + + tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). + tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + + endptErr := errors.New("error listing endpoints") + tc.mockCache.EXPECT().GetEndpoints(test.SvcId).Return(nil, false) + tc.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). Return([]types.InstanceSummary{}, endptErr) - sdc := getTestSdClient(t, sdApi) - svcs, err := sdc.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, endptErr, err) assert.Empty(t, svcs) } func TestServiceDiscoveryClient_ListServices_NamespaceNotFound(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) + tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) - sdc := getTestSdClient(t, sdApi) - svcs, err := sdc.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Empty(t, svcs) assert.Nil(t, err, "No error for namespace not found") - - cachedNs, found := sdc.namespaceCache.Get(test.NsName) - assert.True(t, found) - assert.Nil(t, cachedNs, "Namespace not found in the cache") } func TestServiceDiscoveryClient_CreateService_HappyCase(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) - sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) + + tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) + tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) - sdc := getTestSdClient(t, sdApi) - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err, "No error for happy case") - - cachedNs, _ := sdc.namespaceCache.Get(test.NsName) - assert.Equal(t, *test.GetTestHttpNamespace(), cachedNs, "Happy case caches namespace") - cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) - assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") } func TestServiceDiscoveryClient_CreateService_HappyCaseForDNSNamespace(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{test.GetTestDnsNamespace()}, nil) - sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). - Return(test.SvcId, nil) - - sdc := getTestSdClient(t, sdApi) - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) - assert.Nil(t, err, "No error for happy case") + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestDnsNamespace(), true) - cachedNs, _ := sdc.namespaceCache.Get(test.NsName) - assert.Equal(t, *test.GetTestDnsNamespace(), cachedNs, "Happy case caches namespace") - cachedSvc, _ := sdc.serviceIdCache.Get(fmt.Sprintf("%s/%s", test.NsName, test.SvcName)) - assert.Equal(t, test.SvcId, cachedSvc, "Happy case caches service ID") -} - -func TestServiceDiscoveryClient_CreateService_HappyCaseCachedResults(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() - - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). + tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). Return(test.SvcId, nil) + tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) - sdc := getTestSdClient(t, sdApi) - sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) - - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err, "No error for happy case") } func TestServiceDiscoveryClient_CreateService_NamespaceError(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() nsErr := errors.New("error listing namespaces") - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nsErr) - sdc := getTestSdClient(t, sdApi) - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Equal(t, nsErr, err) } func TestServiceDiscoveryClient_CreateService_CreateServiceError(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() + + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestDnsNamespace(), true) svcErr := errors.New("error creating service") - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) - sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). + tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). Return("", svcErr) - sdc := getTestSdClient(t, sdApi) - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Equal(t, err, svcErr) } func TestServiceDiscoveryClient_CreateService_CreatesNamespace_HappyCase(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() - - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdc := getTestSdClient(t, sdApi) + tc := getTestSdClient(t) + defer tc.close() - sdApi.EXPECT().ListNamespaces(context.TODO()). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) + + tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return(test.OpId1, nil) - sdApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). + tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). Return(test.NsId, nil) - sdApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). + tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) + + tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) + tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err, "No error for happy case") - - cachedNs, _ := sdc.namespaceCache.Get(test.NsName) - assert.Equal(t, *test.GetTestHttpNamespace(), cachedNs, "Create namespace caches namespace ID") } func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - pollErr := errors.New("polling error") - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) + + pollErr := errors.New("polling error") + tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return(test.OpId1, nil) - sdApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). + tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). Return("", pollErr) - sdc := getTestSdClient(t, sdApi) - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Equal(t, pollErr, err) } func TestServiceDiscoveryClient_CreateService_CreatesNamespace_CreateNsError(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - nsErr := errors.New("create namespace error") - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - sdApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) + + nsErr := errors.New("create namespace error") + tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). Return("", nsErr) - sdc := getTestSdClient(t, sdApi) - err := sdc.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) assert.Equal(t, nsErr, err) } func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() + + tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName) - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdApi.EXPECT().ListNamespaces(context.TODO()).Return([]*model.Namespace{{Id: test.NsId, Name: test.NsName}}, nil) - sdApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockApi.EXPECT().ListNamespaces(context.TODO()). + Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) + tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) + + tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) - sdApi.EXPECT().ListInstances(context.TODO(), test.SvcId). + tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + + tc.mockCache.EXPECT().GetEndpoints(test.SvcId).Return([]*model.Endpoint{}, false) + tc.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). Return([]types.InstanceSummary{ { Id: aws.String(test.EndptId1), @@ -316,50 +293,48 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { }, }, }, nil) - sdc := getTestSdClient(t, sdApi) + tc.mockCache.EXPECT().CacheEndpoints(test.SvcId, + []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) - svc, err := sdc.GetService(context.TODO(), test.NsName, test.SvcName) + svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err) assert.Equal(t, test.GetTestService(), svc) } func TestServiceDiscoveryClient_GetService_CachedValues(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdc := getTestSdClient(t, sdApi) - sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) - sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) - sdc.endpointCache.Add(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, time.Minute) + tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName).Return(test.SvcId, true) + tc.mockCache.EXPECT().GetEndpoints(test.SvcId). + Return([]*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, true) - svc, err := sdc.GetService(context.TODO(), test.NsName, test.SvcName) + svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err) assert.Equal(t, test.GetTestService(), svc) } func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - sdc := getTestSdClient(t, sdApi) - sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) - sdc.endpointCache.Add(test.SvcId, model.Endpoint{}, time.Minute) + tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName).Return(test.SvcId, true) attrs1 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp1, "AWS_INSTANCE_PORT": test.EndptPortStr1} attrs2 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp2, "AWS_INSTANCE_PORT": test.EndptPortStr2} - sdApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). + tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). Return(test.OpId1, nil) - sdApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId2, attrs2). + tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId2, attrs2). Return(test.OpId2, nil) - sdApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). + tc.mockApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). Return(map[string]types.OperationStatus{ test.OpId1: types.OperationStatusSuccess, test.OpId2: types.OperationStatusSuccess}, nil) - err := sdc.RegisterEndpoints(context.TODO(), test.NsName, test.SvcName, + tc.mockCache.EXPECT().EvictEndpoints(test.SvcId) + + err := tc.client.RegisterEndpoints(context.TODO(), test.NsName, test.SvcName, []*model.Endpoint{ { Id: test.EndptId1, @@ -374,78 +349,41 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { }) assert.Nil(t, err) - _, entryCached := sdc.endpointCache.Get(test.SvcId) - assert.False(t, entryCached, "Cache entry evicted after register") } func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + tc := getTestSdClient(t) + defer tc.close() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName).Return(test.SvcId, true) - sdc := getTestSdClient(t, sdApi) - sdc.serviceIdCache.Add(fmt.Sprintf("%s/%s", test.NsName, test.SvcName), test.SvcId, time.Minute) - sdc.endpointCache.Add(test.SvcId, model.Endpoint{}, time.Minute) - - sdApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).Return(test.OpId1, nil) - sdApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2).Return(test.OpId2, nil) - sdApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). + tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).Return(test.OpId1, nil) + tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2).Return(test.OpId2, nil) + tc.mockApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). Return(map[string]types.OperationStatus{ test.OpId1: types.OperationStatusSuccess, test.OpId2: types.OperationStatusSuccess}, nil) - err := sdc.DeleteEndpoints(context.TODO(), test.NsName, test.SvcName, + tc.mockCache.EXPECT().EvictEndpoints(test.SvcId) + + err := tc.client.DeleteEndpoints(context.TODO(), test.NsName, test.SvcName, []*model.Endpoint{{Id: test.EndptId1}, {Id: test.EndptId2}}) assert.Nil(t, err) - _, entryCached := sdc.endpointCache.Get(test.SvcId) - assert.False(t, entryCached, "Cache entry evicted after de-register") } -func TestServiceDiscoveryClient_getNamespace_HappyCase(t *testing.T) { +func getTestSdClient(t *testing.T) *testSdClient { mockController := gomock.NewController(t) - defer mockController.Finish() - - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - - sdc := getTestSdClient(t, sdApi) - sdc.namespaceCache.Add(test.NsName, *test.GetTestHttpNamespace(), time.Minute) - - namespace, _ := sdc.getNamespace(context.TODO(), test.NsName) - assert.Equal(t, test.GetTestHttpNamespace(), namespace, "Namespace found in the cache") -} - -func TestServiceDiscoveryClient_getNamespace_GetEmptyNamespace(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() - - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) - - sdc := getTestSdClient(t, sdApi) - sdc.namespaceCache.Add(test.NsName, nil, time.Minute) - - namespace, err := sdc.getNamespace(context.TODO(), test.NsName) - assert.Nil(t, namespace, "Namespace not found in the cache") - assert.Nil(t, err, "No errors with empty namespace") -} - -func TestServiceDiscoveryClient_getCachedNamespace_ErrorCasting(t *testing.T) { - sdc := getTestSdClient(t, nil) - sdc.namespaceCache.Add(test.NsName, struct{ dummy string }{"dummy"}, time.Minute) - - namespace, exists, err := sdc.getCachedNamespace(test.NsName) - assert.True(t, exists, "Cache exists") - assert.Nil(t, namespace, "No corresponding cached value found") - assert.Equal(t, fmt.Sprintf("failed to cast the cached value for the namespace %s", test.NsName), fmt.Sprint(err), "Got the error for improper casting") -} - -func getTestSdClient(t *testing.T, sdApi ServiceDiscoveryApi) serviceDiscoveryClient { - return serviceDiscoveryClient{ - log: testing2.TestLogger{T: t}, - sdApi: sdApi, - namespaceCache: cache.NewLRUExpireCache(1024), - serviceIdCache: cache.NewLRUExpireCache(1024), - endpointCache: cache.NewLRUExpireCache(1024), + mockCache := cloudmap.NewMockServiceDiscoveryClientCache(mockController) + mockApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + return &testSdClient{ + client: &serviceDiscoveryClient{ + log: testing2.TestLogger{T: t}, + sdApi: mockApi, + cache: mockCache, + }, + mockApi: *mockApi, + mockCache: *mockCache, + close: func() { mockController.Finish() }, } } From 3f8cbd8727d540c24734f2f3075cfdb6820e6d76 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Tue, 9 Nov 2021 13:58:47 -0800 Subject: [PATCH 028/163] Replace ListInstances API calls with DiscoverInstances (#70) --- integration/janitor/janitor.go | 8 +++--- integration/janitor/janitor_test.go | 4 +-- pkg/cloudmap/api.go | 24 ++++++++--------- pkg/cloudmap/api_test.go | 24 ++++++++++------- pkg/cloudmap/aws_facade.go | 6 ++--- pkg/cloudmap/cache.go | 25 +++++++++--------- pkg/cloudmap/cache_test.go | 16 +++++------ pkg/cloudmap/client.go | 33 ++++++++++++++--------- pkg/cloudmap/client_test.go | 41 +++++++++++++++-------------- pkg/model/types.go | 12 +++++---- pkg/model/types_test.go | 18 ++++++------- 11 files changed, 114 insertions(+), 97 deletions(-) diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index f2b55e02..9f00beda 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -64,7 +64,7 @@ func (j *cloudMapJanitor) Cleanup(ctx context.Context, nsName string) { for _, svc := range svcs { fmt.Printf("found service to clean: %s\n", svc.Id) - j.deregisterInstances(ctx, svc.Id) + j.deregisterInstances(ctx, nsName, svc.Name, svc.Id) delSvcErr := j.sdApi.DeleteService(ctx, svc.Id) j.checkOrFail(delSvcErr, "service deleted", "could not cleanup service") @@ -78,15 +78,15 @@ func (j *cloudMapJanitor) Cleanup(ctx context.Context, nsName string) { j.checkOrFail(err, "clean up successful", "could not cleanup namespace") } -func (j *cloudMapJanitor) deregisterInstances(ctx context.Context, svcId string) { - insts, err := j.sdApi.ListInstances(ctx, svcId) +func (j *cloudMapJanitor) deregisterInstances(ctx context.Context, nsName string, svcName string, svcId string) { + insts, err := j.sdApi.DiscoverInstances(ctx, nsName, svcName) j.checkOrFail(err, fmt.Sprintf("service has %d instances to clean", len(insts)), "could not list instances to cleanup") opColl := cloudmap.NewOperationCollector() for _, inst := range insts { - instId := aws.ToString(inst.Id) + instId := aws.ToString(inst.InstanceId) fmt.Printf("found instance to clean: %s\n", instId) opColl.Add(func() (opId string, err error) { return j.sdApi.DeregisterInstance(ctx, svcId, instId) diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index 9f44c2d8..99c1b782 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -31,8 +31,8 @@ func TestCleanupHappyCase(t *testing.T) { Return([]*model.Namespace{{Id: test.NsId, Name: test.NsName}}, nil) tj.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) - tj.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). - Return([]types.InstanceSummary{{Id: aws.String(test.EndptId1)}}, nil) + tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + Return([]types.HttpInstanceSummary{{InstanceId: aws.String(test.EndptId1)}}, nil) tj.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1). Return(test.OpId1, nil) diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index 11181b06..1198adbb 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -26,8 +26,8 @@ type ServiceDiscoveryApi interface { // ListServices returns a list of services for a given namespace. ListServices(ctx context.Context, namespaceId string) (services []*model.Resource, err error) - // ListInstances returns a list of service instances registered to a given service. - ListInstances(ctx context.Context, serviceId string) ([]types.InstanceSummary, error) + // DiscoverInstances returns a list of service instances registered to a given service. + DiscoverInstances(ctx context.Context, nsName string, svcName string) (insts []types.HttpInstanceSummary, err error) // ListOperations returns a map of operations to their status matching a list of filters. ListOperations(ctx context.Context, opFilters []types.OperationFilter) (operationStatusMap map[string]types.OperationStatus, err error) @@ -113,19 +113,19 @@ func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) return svcs, nil } -func (sdApi *serviceDiscoveryApi) ListInstances(ctx context.Context, svcId string) (insts []types.InstanceSummary, err error) { - pages := sd.NewListInstancesPaginator(sdApi.awsFacade, &sd.ListInstancesInput{ServiceId: &svcId}) - - for pages.HasMorePages() { - output, err := pages.NextPage(ctx) - if err != nil { - return insts, err - } +func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string) (insts []types.HttpInstanceSummary, err error) { + out, err := sdApi.awsFacade.DiscoverInstances(ctx, &sd.DiscoverInstancesInput{ + NamespaceName: aws.String(nsName), + ServiceName: aws.String(svcName), + HealthStatus: types.HealthStatusFilterAll, + MaxResults: aws.Int32(1000), + }) - insts = append(insts, output.Instances...) + if err != nil { + return insts, err } - return insts, nil + return out.Instances, nil } func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters []types.OperationFilter) (opStatusMap map[string]types.OperationStatus, err error) { diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 7d583f4b..a36f5980 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -85,26 +85,32 @@ func TestServiceDiscoveryApi_ListServices_HappyCase(t *testing.T) { assert.Equal(t, svcs[0], &model.Resource{Id: test.SvcId, Name: test.SvcName}) } -func TestServiceDiscoveryApi_ListInstances_HappyCase(t *testing.T) { +func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() awsFacade := cloudmap.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) - awsFacade.EXPECT().ListInstances(context.TODO(), gomock.Any()). - Return(&sd.ListInstancesOutput{ - Instances: []types.InstanceSummary{ - {Id: aws.String(test.EndptId1)}, - {Id: aws.String(test.EndptId2)}, + awsFacade.EXPECT().DiscoverInstances(context.TODO(), + &sd.DiscoverInstancesInput{ + NamespaceName: aws.String(test.NsName), + ServiceName: aws.String(test.SvcName), + HealthStatus: types.HealthStatusFilterAll, + MaxResults: aws.Int32(1000), + }). + Return(&sd.DiscoverInstancesOutput{ + Instances: []types.HttpInstanceSummary{ + {InstanceId: aws.String(test.EndptId1)}, + {InstanceId: aws.String(test.EndptId2)}, }, }, nil) - insts, err := sdApi.ListInstances(context.TODO(), test.SvcId) + insts, err := sdApi.DiscoverInstances(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err, "No error for happy case") assert.True(t, len(insts) == 2) - assert.Equal(t, test.EndptId1, *insts[0].Id) - assert.Equal(t, test.EndptId2, *insts[1].Id) + assert.Equal(t, test.EndptId1, *insts[0].InstanceId) + assert.Equal(t, test.EndptId2, *insts[1].InstanceId) } func TestServiceDiscoveryApi_ListOperations_HappyCase(t *testing.T) { diff --git a/pkg/cloudmap/aws_facade.go b/pkg/cloudmap/aws_facade.go index 199ce1b3..fb174d43 100644 --- a/pkg/cloudmap/aws_facade.go +++ b/pkg/cloudmap/aws_facade.go @@ -15,9 +15,6 @@ type AwsFacade interface { // ListServices provides ServiceDiscovery ListServices wrapper interface for paginator. ListServices(context.Context, *sd.ListServicesInput, ...func(options *sd.Options)) (*sd.ListServicesOutput, error) - // ListInstances provides ServiceDiscovery ListInstances wrapper interface for paginator. - ListInstances(context.Context, *sd.ListInstancesInput, ...func(*sd.Options)) (*sd.ListInstancesOutput, error) - // ListOperations provides ServiceDiscovery ListOperations wrapper interface for paginator. ListOperations(context.Context, *sd.ListOperationsInput, ...func(*sd.Options)) (*sd.ListOperationsOutput, error) @@ -35,6 +32,9 @@ type AwsFacade interface { // DeregisterInstance provides ServiceDiscovery DeregisterInstance wrapper interface. DeregisterInstance(context.Context, *sd.DeregisterInstanceInput, ...func(*sd.Options)) (*sd.DeregisterInstanceOutput, error) + + // DiscoverInstances provides ServiceDiscovery DiscoverInstances wrapper interface. + DiscoverInstances(context.Context, *sd.DiscoverInstancesInput, ...func(*sd.Options)) (*sd.DiscoverInstancesOutput, error) } type awsFacade struct { diff --git a/pkg/cloudmap/cache.go b/pkg/cloudmap/cache.go index 99b4826e..5f9ab776 100644 --- a/pkg/cloudmap/cache.go +++ b/pkg/cloudmap/cache.go @@ -27,9 +27,9 @@ type ServiceDiscoveryClientCache interface { CacheNilNamespace(namespaceName string) GetServiceId(namespaceName string, serviceName string) (serviceId string, found bool) CacheServiceId(namespaceName string, serviceName string, serviceId string) - GetEndpoints(serviceId string) (endpoints []*model.Endpoint, found bool) - CacheEndpoints(serviceId string, endpoints []*model.Endpoint) - EvictEndpoints(serviceId string) + GetEndpoints(namespaceName string, serviceName string) (endpoints []*model.Endpoint, found bool) + CacheEndpoints(namespaceName string, serviceName string, endpoints []*model.Endpoint) + EvictEndpoints(namespaceName string, serviceName string) } type sdCache struct { @@ -109,8 +109,8 @@ func (sdCache *sdCache) CacheServiceId(nsName string, svcName string, svcId stri sdCache.cache.Add(key, svcId, sdCache.config.svcTTL) } -func (sdCache *sdCache) GetEndpoints(svcId string) (endpts []*model.Endpoint, found bool) { - key := sdCache.buildEndptsKey(svcId) +func (sdCache *sdCache) GetEndpoints(nsName string, svcName string) (endpts []*model.Endpoint, found bool) { + key := sdCache.buildEndptsKey(nsName, svcName) entry, exists := sdCache.cache.Get(key) if !exists { return nil, false @@ -118,7 +118,8 @@ func (sdCache *sdCache) GetEndpoints(svcId string) (endpts []*model.Endpoint, fo endpts, ok := entry.([]*model.Endpoint) if !ok { - sdCache.log.Error(errors.New("failed to retrieve endpoints from cache"), "", "svcId", svcId) + sdCache.log.Error(errors.New("failed to retrieve endpoints from cache"), "", + "ns", "nsName", "svc", svcName) sdCache.cache.Remove(key) return nil, false } @@ -126,13 +127,13 @@ func (sdCache *sdCache) GetEndpoints(svcId string) (endpts []*model.Endpoint, fo return endpts, true } -func (sdCache *sdCache) CacheEndpoints(svcId string, endpts []*model.Endpoint) { - key := sdCache.buildEndptsKey(svcId) +func (sdCache *sdCache) CacheEndpoints(nsName string, svcName string, endpts []*model.Endpoint) { + key := sdCache.buildEndptsKey(nsName, svcName) sdCache.cache.Add(key, endpts, sdCache.config.endptTTL) } -func (sdCache *sdCache) EvictEndpoints(svcId string) { - key := sdCache.buildEndptsKey(svcId) +func (sdCache *sdCache) EvictEndpoints(nsName string, svcName string) { + key := sdCache.buildEndptsKey(nsName, svcName) sdCache.cache.Remove(key) } @@ -144,6 +145,6 @@ func (sdCache *sdCache) buildSvcKey(nsName string, svcName string) (cacheKey str return fmt.Sprintf("%s:%s:%s", svcKeyPrefix, nsName, svcName) } -func (sdCache *sdCache) buildEndptsKey(svcId string) string { - return fmt.Sprintf("%s:%s", endptKeyPrefix, svcId) +func (sdCache *sdCache) buildEndptsKey(nsName string, svcName string) string { + return fmt.Sprintf("%s:%s:%s", endptKeyPrefix, nsName, svcName) } diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index 3a840a76..ebd8ba40 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -79,9 +79,9 @@ func TestServiceDiscoveryClientCacheGetServiceId_Corrupt(t *testing.T) { func TestServiceDiscoveryClientCacheGetEndpoints_Found(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheEndpoints(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) - endpts, found := sdc.GetEndpoints(test.SvcId) + endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) assert.True(t, found) assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, endpts) } @@ -89,7 +89,7 @@ func TestServiceDiscoveryClientCacheGetEndpoints_Found(t *testing.T) { func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - endpts, found := sdc.GetEndpoints(test.SvcId) + endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) assert.False(t, found) assert.Nil(t, endpts) } @@ -97,18 +97,18 @@ func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { func TestServiceDiscoveryClientCacheGetEndpoints_Corrupt(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) - sdc.cache.Add(sdc.buildEndptsKey(test.SvcId), &model.Resource{}, time.Minute) - endpts, found := sdc.GetEndpoints(test.SvcId) + sdc.cache.Add(sdc.buildEndptsKey(test.NsName, test.SvcName), &model.Resource{}, time.Minute) + endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) assert.False(t, found) assert.Nil(t, endpts) } func TestServiceDiscoveryClientEvictEndpoints(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheEndpoints(test.SvcId, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) - sdc.EvictEndpoints(test.SvcId) + sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + sdc.EvictEndpoints(test.NsName, test.SvcName) - endpts, found := sdc.GetEndpoints(test.SvcId) + endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) assert.False(t, found) assert.Nil(t, endpts) } diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 56eb1ca4..4ceb185f 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -58,7 +58,7 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri for _, svcSum := range svcSums { sdc.cache.CacheServiceId(nsName, svcSum.Name, svcSum.Id) - endpts, endptsErr := sdc.listEndpoints(ctx, svcSum.Id) + endpts, endptsErr := sdc.listEndpoints(ctx, nsName, svcSum.Name) if endptsErr != nil { return svcs, endptsErr } @@ -101,6 +101,15 @@ func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName str func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string, svcName string) (svc *model.Service, err error) { sdc.log.Info("fetching a service", "namespace", nsName, "name", svcName) + endpts, cacheHit := sdc.cache.GetEndpoints(nsName, svcName) + + if cacheHit { + return &model.Service{ + Namespace: nsName, + Name: svcName, + Endpoints: endpts, + }, nil + } svcId, err := sdc.getServiceId(ctx, nsName, svcName) @@ -112,19 +121,17 @@ func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string return nil, nil } - endpts, err := sdc.listEndpoints(ctx, svcId) + endpts, err = sdc.listEndpoints(ctx, nsName, svcName) if err != nil { return nil, err } - svc = &model.Service{ + return &model.Service{ Namespace: nsName, Name: svcName, Endpoints: endpts, - } - - return svc, nil + }, nil } func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName string, svcName string, endpts []*model.Endpoint) (err error) { @@ -153,7 +160,7 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName err = NewRegisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) // Evict cache entry so next list call reflects changes - sdc.cache.EvictEndpoints(svcId) + sdc.cache.EvictEndpoints(nsName, svcName) if err != nil { return err @@ -192,7 +199,7 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s err = NewDeregisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) // Evict cache entry so next list call reflects changes - sdc.cache.EvictEndpoints(svcId) + sdc.cache.EvictEndpoints(nsName, svcName) if err != nil { return err } @@ -204,12 +211,12 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s return nil } -func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId string) (endpts []*model.Endpoint, err error) { - if endpts, found := sdc.cache.GetEndpoints(serviceId); found { +func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, nsName string, svcName string) (endpts []*model.Endpoint, err error) { + if endpts, found := sdc.cache.GetEndpoints(nsName, svcName); found { return endpts, nil } - insts, err := sdc.sdApi.ListInstances(ctx, serviceId) + insts, err := sdc.sdApi.DiscoverInstances(ctx, nsName, svcName) if err != nil { return nil, err } @@ -217,13 +224,13 @@ func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, serviceId for _, inst := range insts { endpt, endptErr := model.NewEndpointFromInstance(&inst) if endptErr != nil { - sdc.log.Info(fmt.Sprintf("skipping instance %s to endpoint conversion: %s", *inst.Id, endptErr.Error())) + sdc.log.Info(fmt.Sprintf("skipping instance %s to endpoint conversion: %s", *inst.InstanceId, endptErr.Error())) continue } endpts = append(endpts, endpt) } - sdc.cache.CacheEndpoints(serviceId, endpts) + sdc.cache.CacheEndpoints(nsName, svcName, endpts) return endpts, nil } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 263fe94e..022e2c1f 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -39,25 +39,25 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) - tc.mockCache.EXPECT().GetEndpoints(test.SvcId).Return(nil, false) - tc.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). - Return([]types.InstanceSummary{ + tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return(nil, false) + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + Return([]types.HttpInstanceSummary{ { - Id: aws.String(test.EndptId1), + InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ model.Ipv4Attr: test.EndptIp1, model.PortAttr: test.EndptPortStr1, }, }, { - Id: aws.String(test.EndptId2), + InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ model.Ipv4Attr: test.EndptIp2, model.PortAttr: test.EndptPortStr2, }, }, }, nil) - tc.mockCache.EXPECT().CacheEndpoints(test.SvcId, + tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) svcs, err := tc.client.ListServices(context.TODO(), test.NsName) @@ -75,7 +75,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) - tc.mockCache.EXPECT().GetEndpoints(test.SvcId). + tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName). Return([]*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, true) svcs, err := tc.client.ListServices(context.TODO(), test.NsName) @@ -123,9 +123,9 @@ func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) endptErr := errors.New("error listing endpoints") - tc.mockCache.EXPECT().GetEndpoints(test.SvcId).Return(nil, false) - tc.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). - Return([]types.InstanceSummary{}, endptErr) + tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return(nil, false) + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + Return([]types.HttpInstanceSummary{}, endptErr) svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, endptErr, err) @@ -264,6 +264,8 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { tc := getTestSdClient(t) defer tc.close() + tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return([]*model.Endpoint{}, false) + tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName) tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) @@ -275,25 +277,25 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) - tc.mockCache.EXPECT().GetEndpoints(test.SvcId).Return([]*model.Endpoint{}, false) - tc.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId). - Return([]types.InstanceSummary{ + tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return([]*model.Endpoint{}, false) + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + Return([]types.HttpInstanceSummary{ { - Id: aws.String(test.EndptId1), + InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ model.Ipv4Attr: test.EndptIp1, model.PortAttr: test.EndptPortStr1, }, }, { - Id: aws.String(test.EndptId2), + InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ model.Ipv4Attr: test.EndptIp2, model.PortAttr: test.EndptPortStr2, }, }, }, nil) - tc.mockCache.EXPECT().CacheEndpoints(test.SvcId, + tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) @@ -305,8 +307,7 @@ func TestServiceDiscoveryClient_GetService_CachedValues(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName).Return(test.SvcId, true) - tc.mockCache.EXPECT().GetEndpoints(test.SvcId). + tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName). Return([]*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, true) svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) @@ -332,7 +333,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { test.OpId1: types.OperationStatusSuccess, test.OpId2: types.OperationStatusSuccess}, nil) - tc.mockCache.EXPECT().EvictEndpoints(test.SvcId) + tc.mockCache.EXPECT().EvictEndpoints(test.NsName, test.SvcName) err := tc.client.RegisterEndpoints(context.TODO(), test.NsName, test.SvcName, []*model.Endpoint{ @@ -364,7 +365,7 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { test.OpId1: types.OperationStatusSuccess, test.OpId2: types.OperationStatusSuccess}, nil) - tc.mockCache.EXPECT().EvictEndpoints(test.SvcId) + tc.mockCache.EXPECT().EvictEndpoints(test.NsName, test.SvcName) err := tc.client.DeleteEndpoints(context.TODO(), test.NsName, test.SvcName, []*model.Endpoint{{Id: test.EndptId1}, {Id: test.EndptId2}}) diff --git a/pkg/model/types.go b/pkg/model/types.go index 2669b6ea..1320f27a 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -52,17 +52,18 @@ const ( PortAttr = "AWS_INSTANCE_PORT" ) -// NewEndpointFromInstance converts a Cloud Map InstanceSummary to an endpoint. -func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { +// NewEndpointFromInstance converts a Cloud Map HttpInstanceSummary to an endpoint. +func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) { endpoint := Endpoint{ - Id: *inst.Id, + Id: *inst.InstanceId, Attributes: make(map[string]string, 0), } if ipv4, hasIp := inst.Attributes[Ipv4Attr]; hasIp { endpoint.IP = ipv4 } else { - return nil, errors.New(fmt.Sprintf("cannot convert service instance %s to endpoint without IP address", *inst.Id)) + return nil, errors.New(fmt.Sprintf("cannot convert service instance %s to endpoint without IP address", + *inst.InstanceId)) } if portStr, hasPort := inst.Attributes[PortAttr]; hasPort { @@ -74,7 +75,8 @@ func NewEndpointFromInstance(inst *types.InstanceSummary) (*Endpoint, error) { endpoint.Port = int32(port) } else { - return nil, errors.New(fmt.Sprintf("cannot convert service instance %s to endpoint without port", *inst.Id)) + return nil, errors.New(fmt.Sprintf("cannot convert service instance %s to endpoint without port", + *inst.InstanceId)) } for key, val := range inst.Attributes { diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index 1fa6a35d..0e8f45b6 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -12,14 +12,14 @@ var ip = "192.168.0.1" func TestNewEndpointFromInstance(t *testing.T) { tests := []struct { name string - inst *types.InstanceSummary + inst *types.HttpInstanceSummary want *Endpoint wantErr bool }{ { name: "happy case", - inst: &types.InstanceSummary{ - Id: &instId, + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, Attributes: map[string]string{ Ipv4Attr: ip, PortAttr: "65535", @@ -37,8 +37,8 @@ func TestNewEndpointFromInstance(t *testing.T) { }, { name: "invalid port", - inst: &types.InstanceSummary{ - Id: &instId, + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, Attributes: map[string]string{ Ipv4Attr: ip, PortAttr: "99999", @@ -49,8 +49,8 @@ func TestNewEndpointFromInstance(t *testing.T) { }, { name: "missing IP", - inst: &types.InstanceSummary{ - Id: &instId, + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, Attributes: map[string]string{ PortAttr: "80", "custom-attr": "custom-val", @@ -60,8 +60,8 @@ func TestNewEndpointFromInstance(t *testing.T) { }, { name: "missing port", - inst: &types.InstanceSummary{ - Id: &instId, + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, Attributes: map[string]string{ Ipv4Attr: ip, "custom-attr": "custom-val", From e7a9357a74824ef2af857e6be38915ac33a88e24 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Tue, 9 Nov 2021 14:15:26 -0800 Subject: [PATCH 029/163] Add integration test for service export (#69) --- .gitignore | 1 + Makefile | 10 ++ integration/configs/e2e-deployment.yaml | 22 ++++ integration/configs/e2e-export.yaml | 5 + integration/configs/e2e-service.yaml | 11 ++ integration/janitor/janitor.go | 4 +- integration/janitor/runner/main.go | 14 ++- integration/scenarios/export_service.go | 109 ++++++++++++++++++++ integration/scenarios/runner/main.go | 50 +++++++++ integration/scripts/cleanup-cloudmap.sh | 8 ++ integration/scripts/cleanup-kind.sh | 8 ++ integration/scripts/common.sh | 14 +++ integration/scripts/poll-endpoints.sh | 30 ++++++ integration/scripts/run-tests.sh | 22 ++++ integration/scripts/setup-kind.sh | 15 +++ main.go | 2 +- pkg/cloudmap/cache.go | 40 ++++--- pkg/cloudmap/cache_test.go | 18 +++- pkg/cloudmap/client.go | 13 ++- pkg/cloudmap/client_test.go | 2 +- pkg/controllers/cloudmap_controller.go | 2 +- pkg/controllers/serviceexport_controller.go | 3 +- 22 files changed, 369 insertions(+), 34 deletions(-) create mode 100644 integration/configs/e2e-deployment.yaml create mode 100644 integration/configs/e2e-export.yaml create mode 100644 integration/configs/e2e-service.yaml create mode 100644 integration/scenarios/export_service.go create mode 100644 integration/scenarios/runner/main.go create mode 100755 integration/scripts/cleanup-cloudmap.sh create mode 100755 integration/scripts/cleanup-kind.sh create mode 100755 integration/scripts/common.sh create mode 100755 integration/scripts/poll-endpoints.sh create mode 100755 integration/scripts/run-tests.sh create mode 100755 integration/scripts/setup-kind.sh diff --git a/.gitignore b/.gitignore index 56f77fe1..3748628d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ vendor/ bin/ testbin/ +testlog/ cover.out # Files generated by JetBrains IDEs, e.g. IntelliJ IDEA diff --git a/Makefile b/Makefile index b6c40baf..fdf15b6f 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,16 @@ test: manifests generate generate-mocks fmt vet ## Run tests. test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -covermode=atomic +integration-setup: ## Setup the integration test using kind clusters + @./integration/scripts/setup-kind.sh + +integration-run: ## Run the integration test controller + @./integration/scripts/run-tests.sh + +integration-cleanup: ## Cleanup integration test resources in Cloud Map and local kind cluster + @./integration/scripts/cleanup-cloudmap.sh + @./integration/scripts/cleanup-kind.sh + ##@ Build build: manifests generate generate-mocks fmt vet ## Build manager binary. diff --git a/integration/configs/e2e-deployment.yaml b/integration/configs/e2e-deployment.yaml new file mode 100644 index 00000000..d9eab694 --- /dev/null +++ b/integration/configs/e2e-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: aws-cloud-map-mcs-e2e + name: coredns-deployment + labels: + app: coredns +spec: + replicas: 5 + selector: + matchLabels: + app: coredns + template: + metadata: + labels: + app: coredns + spec: + containers: + - name: coredns + image: k8s.gcr.io/coredns:1.7.0 + ports: + - containerPort: 80 diff --git a/integration/configs/e2e-export.yaml b/integration/configs/e2e-export.yaml new file mode 100644 index 00000000..a23e9f4c --- /dev/null +++ b/integration/configs/e2e-export.yaml @@ -0,0 +1,5 @@ +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-service diff --git a/integration/configs/e2e-service.yaml b/integration/configs/e2e-service.yaml new file mode 100644 index 00000000..83669588 --- /dev/null +++ b/integration/configs/e2e-service.yaml @@ -0,0 +1,11 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-service +spec: + selector: + app: coredns + ports: + - port: 8080 + targetPort: 80 diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index 9f00beda..9a6252ce 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -22,9 +22,7 @@ type cloudMapJanitor struct { // NewDefaultJanitor returns a new janitor object. func NewDefaultJanitor() CloudMapJanitor { - awsCfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(os.Getenv("AWS_REGION")), - ) + awsCfg, err := config.LoadDefaultConfig(context.TODO()) if err != nil { fmt.Printf("unable to configure AWS session: %s", err.Error()) diff --git a/integration/janitor/runner/main.go b/integration/janitor/runner/main.go index 9a2dafad..66e3aa08 100644 --- a/integration/janitor/runner/main.go +++ b/integration/janitor/runner/main.go @@ -2,14 +2,18 @@ package main import ( "context" + "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/janitor" -) - -const ( - e2eNs = "aws-cloud-map-mcs-e2e" + "os" ) func main() { + if len(os.Args) != 2 { + fmt.Println("Expected single namespace name argument") + os.Exit(1) + } + j := janitor.NewDefaultJanitor() - j.Cleanup(context.TODO(), e2eNs) + nsName := os.Args[1] + j.Cleanup(context.TODO(), nsName) } diff --git a/integration/scenarios/export_service.go b/integration/scenarios/export_service.go new file mode 100644 index 00000000..46b2e386 --- /dev/null +++ b/integration/scenarios/export_service.go @@ -0,0 +1,109 @@ +package scenarios + +import ( + "context" + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-sdk-go-v2/aws" + "k8s.io/apimachinery/pkg/util/wait" + "strconv" + "strings" + "time" +) + +const ( + defaultScenarioPollInterval = 10 * time.Second + defaultScenarioPollTimeout = 2 * time.Minute +) + +// ExportServiceScenario defines an integration test against a service export to check creation of namespace, service, +// and endpoint export. +type ExportServiceScenario interface { + // Run executes the service export integration test scenario, returning any error. + Run() error +} + +type exportServiceScenario struct { + sdClient cloudmap.ServiceDiscoveryClient + expectedSvc model.Service +} + +func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, portStr string, ips string) (ExportServiceScenario, error) { + endpts := make([]*model.Endpoint, 0) + + port, parseError := strconv.ParseUint(portStr, 10, 16) + if parseError != nil { + return nil, parseError + } + + for _, ip := range strings.Split(ips, ",") { + endpts = append(endpts, &model.Endpoint{ + Id: model.EndpointIdFromIPAddress(ip), + IP: ip, + Port: int32(port), + Attributes: make(map[string]string, 0), + }) + } + + return &exportServiceScenario{ + sdClient: cloudmap.NewServiceDiscoveryClientWithCustomCache(cfg, + &cloudmap.SdCacheConfig{ + NsTTL: time.Second, + SvcTTL: time.Second, + EndptTTL: time.Second, + }), + expectedSvc: model.Service{ + Namespace: nsName, + Name: svcName, + Endpoints: endpts, + }, + }, nil +} + +func (e *exportServiceScenario) Run() error { + fmt.Printf("Seeking expected service: %v\n", e.expectedSvc) + + return wait.Poll(defaultScenarioPollInterval, defaultScenarioPollTimeout, func() (done bool, err error) { + fmt.Println("Polling service...") + cmSvc, err := e.sdClient.GetService(context.TODO(), e.expectedSvc.Namespace, e.expectedSvc.Name) + if err != nil { + return true, err + } + + if cmSvc == nil { + fmt.Println("Service not found.") + return false, nil + } + + fmt.Printf("Found service: %v\n", cmSvc) + return e.compareEndpoints(cmSvc.Endpoints), nil + }) +} + +func (e *exportServiceScenario) compareEndpoints(cmEndpoints []*model.Endpoint) bool { + if len(e.expectedSvc.Endpoints) != len(cmEndpoints) { + fmt.Println("Endpoints do not match.") + return false + } + + for _, expected := range e.expectedSvc.Endpoints { + match := false + for _, actual := range cmEndpoints { + // Ignore K8S instance attribute for the purpose of this test. + delete(actual.Attributes, controllers.K8sVersionAttr) + if expected.Equals(actual) { + match = true + break + } + } + if !match { + fmt.Println("Endpoints do not match.") + return false + } + } + + fmt.Println("Endpoints match.") + return true +} diff --git a/integration/scenarios/runner/main.go b/integration/scenarios/runner/main.go new file mode 100644 index 00000000..a82cefd6 --- /dev/null +++ b/integration/scenarios/runner/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/scenarios" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "os" +) + +func main() { + if len(os.Args) != 5 { + fmt.Println("Expected namespace, service, endpoint port, and endpoint IP list arguments") + os.Exit(1) + } + + nsName := os.Args[1] + svcName := os.Args[2] + port := os.Args[3] + ips := os.Args[4] + + testServiceExport(nsName, svcName, port, ips) +} + +func testServiceExport(nsName string, svcName string, port string, ips string) { + fmt.Printf("Testing service export integration for namespace %s and service %s\n", nsName, svcName) + + export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, port, ips) + if err != nil { + fmt.Printf("Failed to setup service export integration test scenario: %s", err.Error()) + os.Exit(1) + } + + if err := export.Run(); err != nil { + fmt.Printf("Service export integration test scenario failed: %s", err.Error()) + os.Exit(1) + } +} + +func getAwsConfig() *aws.Config { + awsCfg, err := config.LoadDefaultConfig(context.TODO()) + + if err != nil { + fmt.Printf("unable to configure AWS session: %s", err.Error()) + os.Exit(1) + } + + return &awsCfg +} diff --git a/integration/scripts/cleanup-cloudmap.sh b/integration/scripts/cleanup-cloudmap.sh new file mode 100755 index 00000000..00d25575 --- /dev/null +++ b/integration/scripts/cleanup-cloudmap.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Deletes all AWS Cloud Map resources used for integration test. + +set -eo pipefail +source ./integration/scripts/common.sh + +go run ./integration/janitor/runner/main.go "$NAMESPACE" \ No newline at end of file diff --git a/integration/scripts/cleanup-kind.sh b/integration/scripts/cleanup-kind.sh new file mode 100755 index 00000000..d1a9d446 --- /dev/null +++ b/integration/scripts/cleanup-kind.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Deletes Kind cluster used for integration test. + +set -eo pipefail +source ./integration/scripts/common.sh + +$KIND_BIN delete cluster --name "$KIND_SHORT" \ No newline at end of file diff --git a/integration/scripts/common.sh b/integration/scripts/common.sh new file mode 100755 index 00000000..f3cfbab6 --- /dev/null +++ b/integration/scripts/common.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +KIND_BIN='kind' +KUBECTL_BIN='./testbin/bin/kubectl' +LOGS='./integration/testlog' +CONFIGS='./integration/configs' +SCENARIOS='./integration/scenarios' +NAMESPACE='aws-cloud-map-mcs-e2e' +SERVICE='e2e-service' +ENDPT_PORT=80 +KIND_SHORT='cloud-map-e2e' +CLUSTER='kind-cloud-map-e2e' +IMAGE='kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729' +EXPECTED_ENDPOINT_COUNT=5 \ No newline at end of file diff --git a/integration/scripts/poll-endpoints.sh b/integration/scripts/poll-endpoints.sh new file mode 100755 index 00000000..4fa8338e --- /dev/null +++ b/integration/scripts/poll-endpoints.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Poll for endpoints to become active + +set -e +source ./integration/scripts/common.sh + +endpt_count=0 +poll_count=0 +while ((endpt_count < $1)) +do + if ((poll_count++ > 30)) ; then + echo "timed out polling for endpoints" + exit 1 + fi + + sleep 2s + if ! addresses=$($KUBECTL_BIN describe endpoints --namespace "$NAMESPACE" | grep " Addresses: ") + then + # no endpoints ready + continue + fi + + endpts=$(echo "$addresses" | tr -s " " "$addresses" | cut -f 3 -d " ") + + endpt_count=$(echo "$endpts" | tr ',' '\n' | wc -l | xargs) +done + +echo "$endpts" +exit 0 \ No newline at end of file diff --git a/integration/scripts/run-tests.sh b/integration/scripts/run-tests.sh new file mode 100755 index 00000000..7cd432a1 --- /dev/null +++ b/integration/scripts/run-tests.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Runs the AWS Cloud Map MCS Controller for K8s as a background process and tests services have been exported + +set -eo pipefail + +source ./integration/scripts/common.sh + +$KUBECTL_BIN apply -f "$CONFIGS/e2e-deployment.yaml" +$KUBECTL_BIN apply -f "$CONFIGS/e2e-service.yaml" +$KUBECTL_BIN apply -f "$CONFIGS/e2e-export.yaml" + +endpts=$(./integration/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") + +mkdir -p "$LOGS" +make +./bin/manager &> "$LOGS/ctl.log" & +CTL_PID=$! + +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT "$endpts" + +kill $CTL_PID \ No newline at end of file diff --git a/integration/scripts/setup-kind.sh b/integration/scripts/setup-kind.sh new file mode 100755 index 00000000..37fb6358 --- /dev/null +++ b/integration/scripts/setup-kind.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Builds the AWS Cloud Map MCS Controller for K8s, provisions a Kubernetes clusters with Kind, +# installs Cloud Map CRDs and controller into the cluster and applies export and deployment configs. + +set -e + +source ./integration/scripts/common.sh + +$KIND_BIN create cluster --name "$KIND_SHORT" --image "$IMAGE" +$KUBECTL_BIN config use-context "$CLUSTER" +$KUBECTL_BIN create namespace "$NAMESPACE" +make install + +exit 0 diff --git a/main.go b/main.go index c1b448da..a12c492b 100644 --- a/main.go +++ b/main.go @@ -96,7 +96,7 @@ func main() { os.Exit(1) } - serviceDiscoveryClient := cloudmap.NewServiceDiscoveryClient(&awsCfg) + serviceDiscoveryClient := cloudmap.NewDefaultServiceDiscoveryClient(&awsCfg) if err = (&controllers.ServiceExportReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("ServiceExport"), diff --git a/pkg/cloudmap/cache.go b/pkg/cloudmap/cache.go index 5f9ab776..640d157e 100644 --- a/pkg/cloudmap/cache.go +++ b/pkg/cloudmap/cache.go @@ -35,24 +35,30 @@ type ServiceDiscoveryClientCache interface { type sdCache struct { log logr.Logger cache *cache.LRUExpireCache - config sdCacheConfig + config *SdCacheConfig } -type sdCacheConfig struct { - nsTTL time.Duration - svcTTL time.Duration - endptTTL time.Duration +type SdCacheConfig struct { + NsTTL time.Duration + SvcTTL time.Duration + EndptTTL time.Duration } -func NewDefaultServiceDiscoveryClientCache() ServiceDiscoveryClientCache { +func NewServiceDiscoveryClientCache(cacheConfig *SdCacheConfig) ServiceDiscoveryClientCache { return &sdCache{ - log: ctrl.Log.WithName("cloudmap"), - cache: cache.NewLRUExpireCache(defaultCacheSize), - config: sdCacheConfig{ - nsTTL: defaultNsTTL, - svcTTL: defaultSvcTTL, - endptTTL: defaultEndptTTL, - }} + log: ctrl.Log.WithName("cloudmap"), + cache: cache.NewLRUExpireCache(defaultCacheSize), + config: cacheConfig, + } +} + +func NewDefaultServiceDiscoveryClientCache() ServiceDiscoveryClientCache { + return NewServiceDiscoveryClientCache( + &SdCacheConfig{ + NsTTL: defaultNsTTL, + SvcTTL: defaultSvcTTL, + EndptTTL: defaultEndptTTL, + }) } func (sdCache *sdCache) GetNamespace(nsName string) (ns *model.Namespace, found bool) { @@ -78,12 +84,12 @@ func (sdCache *sdCache) GetNamespace(nsName string) (ns *model.Namespace, found func (sdCache *sdCache) CacheNamespace(namespace *model.Namespace) { key := sdCache.buildNsKey(namespace.Name) - sdCache.cache.Add(key, *namespace, sdCache.config.nsTTL) + sdCache.cache.Add(key, *namespace, sdCache.config.NsTTL) } func (sdCache *sdCache) CacheNilNamespace(nsName string) { key := sdCache.buildNsKey(nsName) - sdCache.cache.Add(key, nil, sdCache.config.nsTTL) + sdCache.cache.Add(key, nil, sdCache.config.NsTTL) } func (sdCache *sdCache) GetServiceId(nsName string, svcName string) (svcId string, found bool) { @@ -106,7 +112,7 @@ func (sdCache *sdCache) GetServiceId(nsName string, svcName string) (svcId strin func (sdCache *sdCache) CacheServiceId(nsName string, svcName string, svcId string) { key := sdCache.buildSvcKey(nsName, svcName) - sdCache.cache.Add(key, svcId, sdCache.config.svcTTL) + sdCache.cache.Add(key, svcId, sdCache.config.SvcTTL) } func (sdCache *sdCache) GetEndpoints(nsName string, svcName string) (endpts []*model.Endpoint, found bool) { @@ -129,7 +135,7 @@ func (sdCache *sdCache) GetEndpoints(nsName string, svcName string) (endpts []*m func (sdCache *sdCache) CacheEndpoints(nsName string, svcName string, endpts []*model.Endpoint) { key := sdCache.buildEndptsKey(nsName, svcName) - sdCache.cache.Add(key, endpts, sdCache.config.endptTTL) + sdCache.cache.Add(key, endpts, sdCache.config.EndptTTL) } func (sdCache *sdCache) EvictEndpoints(nsName string, svcName string) { diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index ebd8ba40..e8377bcd 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -8,12 +8,24 @@ import ( "time" ) +func TestNewServiceDiscoveryClientCache(t *testing.T) { + sdc := NewServiceDiscoveryClientCache(&SdCacheConfig{ + NsTTL: 3 * time.Second, + SvcTTL: 3 * time.Second, + EndptTTL: 3 * time.Second, + }).(*sdCache) + + assert.Equal(t, 3*time.Second, sdc.config.NsTTL) + assert.Equal(t, 3*time.Second, sdc.config.SvcTTL) + assert.Equal(t, 3*time.Second, sdc.config.EndptTTL) +} + func TestNewDefaultServiceDiscoveryClientCache(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) - assert.Equal(t, defaultNsTTL, sdc.config.nsTTL) - assert.Equal(t, defaultSvcTTL, sdc.config.svcTTL) - assert.Equal(t, defaultEndptTTL, sdc.config.endptTTL) + assert.Equal(t, defaultNsTTL, sdc.config.NsTTL) + assert.Equal(t, defaultSvcTTL, sdc.config.SvcTTL) + assert.Equal(t, defaultEndptTTL, sdc.config.EndptTTL) } func TestServiceDiscoveryClientCacheGetNamespace_Found(t *testing.T) { diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 4ceb185f..0f32bdcc 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -34,8 +34,9 @@ type serviceDiscoveryClient struct { cache ServiceDiscoveryClientCache } -// NewServiceDiscoveryClient creates a new service discovery client for AWS Cloud Map from a given AWS client config. -func NewServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { +// NewDefaultServiceDiscoveryClient creates a new service discovery client for AWS Cloud Map with default resource cache +// from a given AWS client config. +func NewDefaultServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { return &serviceDiscoveryClient{ log: ctrl.Log.WithName("cloudmap"), sdApi: NewServiceDiscoveryApiFromConfig(cfg), @@ -43,6 +44,14 @@ func NewServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { } } +func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCacheConfig) ServiceDiscoveryClient { + return &serviceDiscoveryClient{ + log: ctrl.Log.WithName("cloudmap"), + sdApi: NewServiceDiscoveryApiFromConfig(cfg), + cache: NewServiceDiscoveryClientCache(cacheConfig), + } +} + func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName string) (svcs []*model.Service, err error) { namespace, err := sdc.getNamespace(ctx, nsName) if err != nil || namespace == nil { diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 022e2c1f..4aaf77c9 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -22,7 +22,7 @@ type testSdClient struct { } func TestNewServiceDiscoveryClient(t *testing.T) { - sdc := NewServiceDiscoveryClient(&aws.Config{}) + sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}) assert.NotNil(t, sdc) } diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 2da2dfc0..c5469b00 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -27,7 +27,7 @@ const ( // DerivedServiceAnnotation annotates a ServiceImport with derived Service name DerivedServiceAnnotation = "multicluster.k8s.aws/derived-service" - // LabelServiceImportName indicates the name of multi-cluster service that an EndpointSlice belongs to. + // LabelServiceImportName indicates the name of the multi-cluster service that an EndpointSlice belongs to. LabelServiceImportName = "multicluster.kubernetes.io/service-name" ) diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index 51fdd338..bdef8bcc 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -36,6 +36,7 @@ import ( ) const ( + K8sVersionAttr = "K8S_CONTROLLER" serviceExportFinalizer = "multicluster.k8s.aws/service-export-finalizer" ) @@ -195,7 +196,7 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. for _, IP := range ep.Addresses { attributes := make(map[string]string, 0) if version.GetVersion() != "" { - attributes["K8S_CONTROLLER"] = version.PackageName + " " + version.GetVersion() + attributes[K8sVersionAttr] = version.PackageName + " " + version.GetVersion() } // TODO extract attributes - pod, node and other useful details if possible From 26134dcdf5cef3c64aee42caa6cceda946b09f79 Mon Sep 17 00:00:00 2001 From: Jaromir Vanek Date: Wed, 10 Nov 2021 14:37:32 -0800 Subject: [PATCH 030/163] Add architecture architecture to documentation (#72) --- CONTRIBUTING.md | 7 +++++++ docs/architecture-overview.png | Bin 0 -> 60441 bytes 2 files changed, 7 insertions(+) create mode 100644 docs/architecture-overview.png diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65f1fb48..2c86c8c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,13 @@ documentation, we greatly value feedback and contributions from our community. Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. +## Architecture Overview + +![Architecture diagram](docs/architecture-overview.png?raw=true) + +* `pkg/controllers/serviceexport_controller` is watching changes on K8s `ServiceExport` resources (and corresponding services/endpoints). As soon as any change in configuration is detected, it registers all exported service endpoints to corresponding (same namespace/service names) AWS Cloud Map structures (namespace, service, instances). +* `pkg/controllers/cloudmap_controller` is periodically polling for changes in corresponding AWS Cloud Map namespaces (based on namespace "sameness" - a K8s namespace with the same name as a Cloud Map namespace). When new service or endpoints are discovered they are automatically created locally as a `ServiceImport`. + ## Getting Started ### Local Setup diff --git a/docs/architecture-overview.png b/docs/architecture-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..3e02f3647e96d52478aa7969808faeeb4ae06979 GIT binary patch literal 60441 zcmaI73p|r=_&>g(qEb#B9CM5g!fdmJu+27`VPi8(G}{<(BhhCS9&yG930ByfXr;s{~Vb? zthvIdXfs=k83e)>G2=po8(2b7#0DmxDG>^QV?OW&a1~$34h;_Fu>ZLdVhypev;kh$ zKESuOF4oq-o2~T*BofK^=Xz!cNAT~kB%vgf$77m7u&@o*z)%(>CM#4BEsWxt*`k2I z-2~Ab9`FgA2Hrirfj2tv-^M!71{P?y1~_zyh~RO2IYGqGX!$6%b{lNrz-bF2-iPFF z20;VIywGqC@PXs7!iDl%&|Hy_51hdwfk`(&;kJkka3pZvn;Fa$h5j{)d~uv;X2?Ga zvkXQ;xrBI5te42z-No9=3l0TjkzW(b5k-Xx1^--T1K9xGfcWQ&==ccEKWBqP!ni1o zNPgPZ6;Lrdlo=vo0t&Pa{a0@;UP3yA#vt(H0x8rOeyk^k^6w}DCLrxED|i8_v$XkV z5IdSUh>a9NUA>?-UZfy=RH#ibK7gK@=}9mY*+_N5L>XJy}#Z#EmOKNNgp(q2bnHBzzQ-<^ctCfZ`)1TvryLbu2l~hVF&; z@*~9|QPF%`cpP6qmeRfzSC4h5V5>xyByNhkz28#@*u$d>9B z!41MQV!gyHCKE~p1`v7kJzS!h{18u=H64xNGkh>K7%|MlMhtZqqr>40NH76yzdGrXlZ=^(Q zi$llT;Y83-vWG7#IM|vWN5s2E#^ZdDp+1nbXiv|9>+lJW5Lnj!|;d$BdZGoW0BAAhJQ8AcMCWa=m<`cab z1SHlv+=D>_Vg^Md1+!ee1$2){Oaw9p?L`ZXaSwr0d|+hYP8&8#?CEEXw86oMIK0T( z+XuxH12BZOfr-5$-53#kvNwbsiWONyAtCXA`LHaM4Vf9r@eBw2!``D<4_S|R=m4kFbXOl5=<$Jq1^b;NSrVpVa*gku_9KKfW{-+O0e!&sI3SS6)s{}Aki6I=YY2qQkMp>;;<-NYG@KX4juP$a>A~}sTRS$=&mH21jzkdf zuGSIpVjmGP+S*HK4R?bHa2^y7svyqICX9d~!+5+n4$hVe5i%&`kYJ*V4_82DgppWY z7;hV|m^dEC)zvyW-qlVVZ{y=Fj-?1}Jwl_ygAidMWSpxn!y8INMlcXDECG%r6#E8o z5fmV{u>y}srjNHvJer6Q5E*`6Bv%4}FQ^b2$<+qQ_ocZ&qPScMn(c~+<43bX5EwUp z2pq?ygfQ@-z7c$vFrp{PjV<)$x<-e`qGRGQ1Uem$;?NLu7SxwQXM1{5StKOg)7_os z=1L|pi2|NYG%+R)9*SX;XmKQ8Is*y=ntG);)w63X>2Z zLS87!j)aF(p>!_6nolNU#XMd#fXqxAv>z%KPGWIGX`$A(9z-!xqS){#fD8CCD7MTP0mFvt z3z3MzcxX2cB+NDrxIUhSLi1o`A3TW~%?PrVMDy4JfrQCNazjZ$F>$U$7oIPZ5koftmTZSWM3T6eP!9>2 zDip9hs7!tk1tIhx`>;q+LJ?a4V|%jbkvN8kh@!K240^bGB!L)@C)hhO-Oju|q^~ygZX~VF)41E?k}{H=QBfzk1fwfyOE}lc;Mq%(#u~=a=kr={5(J6LVw@5Js8^I%y5h3glwjTse z3bSDeFp^Lo4pvCy_`xx;@Yypitc7zzSLvjc7th$3+D)>KHi zI{`^2Q6jixJO?j{rpFS4V?q&<2y0uu$WxyFML?26`O&c+(RSg%K^%0nI2Ld1E(+p? zc~D_6V3~48A|KEP@Kdau*m?;JPRjM-TqC$LJqxfgbgf6;%7tfl|+g{C0sW-L&n0qJ;fpk3S;d@kMaSK2_+yw zZ9Jl4kyvUhgimBbgTr8gNIVqd^-Jz>esp{ko);Vo+$w?sUWASD3a7;4Bv>0sNVJ_t zFgXU}5lf-*(6Qt=m&jl&Jd90ZMX`f`E2*I*PB1`)umD*G-hcDuU*sA1``;49*46al zeLe_e2y(-?_=p20k1mMT>REB;hMP0$y`jEF+Jkv6YHIsN#@=T=89h#AE}IQjNm!E~m|s3$ zBNtQu=S1t|6D>>UHK{6SW!*JA`SE2Gs@x?Z%lLt6J8RkBs?AcKs_ik^R*z6rvM^BP zTABxxYacJYaJc9Wl)O&;Ec4}u;o8^}eutL*&k0Bh+XOM1T4nJ^rEi;A?XnfdFxAqe z-55mZ0pobI&wo$OOs5v6OUUn*ck;4`nVJc30@A(N&t5rqg zj-D+|Z`}TJsGWO3#hZ79SnlGlu2IjbSbb#2())F6HaIFd)MB%#(wIQ?rtMO;H>v(x z2W-+_Cu^={G;-oxQBJVyrVajZ<1G7eeDKBX0ocD>T8#^sREN7RxLEcW?-1~0!7I*w zEB=88C9|W?5&7!^F1uS4_S{XowwB}8W8C|;K1tP{CHn>*k98lOs9VbaS*2dvHqi0} z;r*BKlP)P=Dr_9@y|hyH@t)DvMY4X;r$gHqz3W4AO$7dq>jHR1s^2o3S&v56r5a$= zw^?l}v?O&G)H|x@EV^D>bmC~Fes!sTx7yfQ|I2g957)jasT@1FwEc!t)WPD0+`l4w zR#WcF55pqX-&N^HJ7hXNr#!sGxod6}0{3`dq>ro$O;xDE2dJVS@4o5KMLw^7VYuc7 zt7j~eS@waU#2q-U65>QqH`jG8hDJ`-U zaA)-GN4>}SyEf;gK-*`YRu)|Tz|iOpNvWDsbx4BfjkaVG>_IhSa?jp3zKW!G; za8Q(SctpJ&`(kE#<8Ga(h@;vTQq=4}%)R^R>=nHoMwEva zH(gF@{c$JsY1O*nxG#n0XPgUNXEt^Q92iQ9HZvG2H1gc-f?{+8)fn|TIE{-6aq+J< z>R7Hilz`T`XbhSJdnq*Tz&>3Lx(>3|X}s!lzIf4)g|za}MZ*fvq>|cb^Ywk8?2k6Z zpooEeZ>!e)u}doyh$ZT)T@$sJGi3E^QA`!771;kSRPiIK#l^~CbJh)OY+v40LS;(d zefQnA31cv-XVt~R=1(a2xgsjZ;TL@O#O(w>n*M^)iIvx)+3##7w6e0m z*%*5?$lY!9@x6MLQm{K{>gfb%5|(+A+^hR^A2pSB^8^T ze6ncBZ1jS@vB4sRtIM28pmnaR6thQ(FB%SkPVA^_RMG6(~t2@Vm6ivUB z%RWYy{#=#*qfTdTOyT0nM)7TvjS7uMAWy~F<-9HC7G#?ujX@p8J4M@0?=E$qtTWadvsfa-7IL_UFE+hER+tC4ha)>T=;mxzB8N3E z-hAL*xh?3P;{Aui3hG;giWZ!D^Br%+ic)`byOI+F2Pg^dD`x7nBVQsLt6wL7T}Cn3Z0lZQ0zwygr~P@sc`c2vE67|~?~#R&9AV?IbU zMBf~x8Q5=-mM0(s5;!HDgkny_ySG>UVFb>NYFyZ`0sy_^wU;5%kI9;@ya;uW3V$19 z>*5E^&bt_L1g|`G>dlBNv9h(O*;P?=u4{|g)yOB~7RF@AMMC5!!l{^}Vgvm>C(CZW zDvWLUmHi+--VTHVtH3F0>lB9qM++}Cm5TM=XverhJ?7t6ZuA7TX^(hKy3AnP#z*|L zWb~W;;qf<&3|t?j>Le;Rnu9)o=*&m&gD?F|AHG@67Tb1Q?0NM1ZV`UMvp#F&+=1{b z-VXCWTv(BL^moE*+yX=jY2=ggsc~97?cn{Mkc`&Gig%B9JM}IS$$sUak$J?*lsNJ& zntpSsa@*RIXz~?`#k+F_W;VOdmExnd4gd*l@Ex7^(|*a0@4BZoULXwK42xH@35wmv z8ZJd+Hpnba`MAsa2d5@e$KS5H{lhz8{@#x+%%e$M%^ zWtqfaPIg751@k+NWiu5l%|Fz_l#4fWbo+H~f1A~k-hl876Gcp~ee%41V*z)|gCKr# z#g9b8Vbet|_s(YawT)j$jHRaD26D}5x+mmPK;1z@!;@D>AxN6jdhItU^tTpdru}$+`(juG?Tr7u)-tR=zYe7`vcXoav;jO2Kp zQugv4NVYI|2-%qE@mu?9UP=toP*ixfO(d)&)oUZONTXL6lQg`&{-j4Mequ z6I>trPj$qk#6jY_(X4jMRR;q#9$sC0zB4{=WtfW9G83&~bGPspJJciIq@S=mHC|a$ z=}CEfFxz*aTCC(e$Sx_X8jsMBeoLP@v2=~`E6N84!bo8k7z%$)=s(MD&z&o{09qNG zLUg;jpW-OXCD7Imgdfsx+-kBWd);zl!4AdjzWx-wi*|+!{#&c_FV;5R`=};-UU~O| z3*uji&xan88&ou5aaY*u+7k4$QVczpV@95;eOz2n7)RM%dSvVs-E@+Ai*YhVjXF{l zZK2cGdK_})RP;>g`O@wWIkk3TE@`B~UUjH$Uu=`+nqne7Z090a3I2|uOkx1feBaut z)^yM`DT|$xySO6r?YovgeYH7BT^C~9-m>+nR=oMklh@<$ zuq9owBjnW^p5a<1PmkCf`5c6Lpf}KGq41y*52x9>$INxo$8va^12g$a3thJ-c8-}E zoz47aR9w8>ug7kmGBHZX5$Q&ngnCE|mhE->Gl%KWhNQeElj6-!%~~Zib>#pVV#3|e za>KqOp|5B;AN4vSym4K}SiO%l$D=o@JrhSAcH8$_Ec+_cedwwRPR? z%5V=i4BcIEDhByK8H=-m<(DTq(z4(#qXiY>yfL2Rk6SCZ+~&p|q}69VycXiiJej%k ze)m3$L$r>Dw9JyTBh7i~=Q2^=w}QU?d{5B0zzQ2Z*4KTM8l?Y+EGT=(P1qTuTH`nG zvN*3z_am*YZnypP8xCZ~=r^BEn7{v}js82>ZN}eKAeboa zssLY$uF)3X--k0vIAB_GcDVE0>J`~9Vdpg_TA$qweX21-*topKnr8Pky;svfKe*vv--k0(jmlrn%bmq$ zPGuY8M5a#f&@-4$Mm(%}H)6xaO{e$X>2)6yXF^-r7XB^VO*A0f&FX_U5t^G*aAt9!xnu)`Cl$L zW7;@QOE`b+M%QE72zmMKqZzF)1MYm%yUbkOl~Z1p8G^5agI2f7KI`rp{ZmsV-CGTi zr+We5nc^eTw@sz?x;Y)A$OoFqVac^^*IUO!0zEn&6d3OPZ1ks~&2}yX#4iLY?285R zd!}H5rnxm{_tmYAi&rJwyPi(;)zo-#f+|W5EqG?R_uTWJN41(wG+&&Ums`C5uVTVf zV<>7}#?4~oeu~3dFj5voBK7UKEV2QXMtD5A?~mPf?vkT4OMe%0dCjsc6@?Gmd`W%F z>6O{;*{pe-8J9mXa??$2irVu@@O7K_;r!Oe!HpBHpK;F~MWSo|>MP6ui>4)9xZ!l~ z+02NIi*rCf_*in*^4%(;ZAu3_{^MHN(x>vGTCnb2;K!5V`a{kkyFSl5k?4Fw-S<~t zU@m+Bsq@Fv4U?Qrhdk!@rg!Y?s?Z({zEEaikTHEw*Tyuw_x%0oZ2>l5lD+PZ(K`uR znE@qtv48wQ3h0xd+sX!N{6qDi$)Z0?|K}QoB%oztSgz56|FhJ8Jn)b^(1F;$Li+S? z#mdVYXG67P;>EkEBP;()cK^<2xL?J~A^3yB#e}~^|5dmsD**x|T_ot4{AKK424h3O zX)W#npjglUEkKepAb=Vzm1X)@?IS1WN$#tGHq6~cjqm;s0o0Ve9OmCz8=3xh6v>aX z?<$pRWetE|%!_zb@KG6bMX`1MkRu3kaM_)Zb>K_ZiGt40%Tz>oYvsTwZg~GQGmt%~ zKD3`{A-(E;J#fJBx7gBYC6VG8#bXN6D|&T-3WoP)KWEE+&_S&9;<2CI+X7{VdzG$D z&ELxeg(+MG4Xpze>%v#ph|Hie2?N!Zq$w6G*m}U`tWI)-16#RG9O!xQ?S84Q z(u+mMwWNa}e;Wazwp>?w4Lk{+Qjs$4)Q&;J=^vW#7h;>pYaEggXkzYf=HY0=Tm70(F=`||S%^;q&!JURp zDsa5Qa<$fl$BqtLt{7Y{tktUB8@=dj$cq2wx^6y|E^8TS1|3g-w{rW7gm;rSqZEQp zdxEY$OnLm|Mg0CwOEJKC>96;JUMut~Y)!VBe-G}~*ri;*WC$ufbSH4t98avb_O^JS56A0GOps(2f(Ji92f?$OR^p?Um=4WPdJHO3Py;(( zk4}sm)u9>vc7Um+3RPxf3s@E!$&xmw$iRDV72iwAea#I3pVW>QxK+<$ZjkiNXjq-`|!3`My`f zvuV@<6t{9%QdOJ-;5!g_PNNs>M^$$J4xC0SmHP1S-Jh4;(9>Vws&L)RdO>5>5~RbE zcb26iGBBtD%+iG1F`a zxLc&r_Vt3S{58BRJq>roeygj-MVln^43;A=UF!Y2SifOtn5zFxq2Nv_?0b+SBITaB zp}9)ymkHuS+q_RHxX^kiC^D)F2+$K*&Nd+2gE*D&{~}kMBo%9)Rzws8SnXT!HCWSB z%|ht~xA{$|YVQ`A#rckB*vBJFABN@0LDba9Yks2-sJL6xt^Ic?+^*uVeAF){>bp!RD?yJyVS&krRT-*&5rArmX%NpTC3ETWXcak3r z8GzlA*T&lyefDvv>(dX<0fcouwokq8YYb!Am;bu9(iWw6Bdd6h!L9QrUAJZruTOUi zeH$^hhB;9=*yD8|-HpIii7w93Lr;?p+#$#Eno9NB zb96zrWnY`WMyk2sht_~dAP3FeQKNZ(gR;WmMigjp|Eh-ty+tjJMG%~)!Vi)&W$%u6 z&(SYl{f5)6%B@iqN6NkjHl`OSnt^|s-M_EAE;R6sdQ>sZXMj{bXZ_vvxr|fG8fc|$%d6UoGApaS~Kz&xZar2#l7Y4AVeE^vH zYer~gm4G(Qx7Kf@DF8zQ*3>r9^|zPISq^u2{3F-S{m%11Szxgx;&h8&j`Jz?m5M{w z(w}#g^_p4UpbpTdyO&BTSsEA3laQbvO4fr#568>@2I}v5jn$xc3RC_fnZ<=D@YPqR z6~`~)4e2Tl3!PEEfhxT!qJaM-1P;JN=iHaA0VBKa)#zLdZ%z8os{vmJaaD)(x;#!< zW=p%2UnoZOc7L=uOCQVN7RTI8{(Dk<<2NO?RWAh;7=I(Q21s^8_{L8T*B5Nv6FHir zrjiZddYadiH42k9f-?^*DWpnMT4q3S5JCg5I8OgtAv{m#+%flgQ)T-2 zdb`+x>r-n_!VjMF;oU$!&mdlj8&0^+@;c8*wTZoG88>b*B4dFx!IROSje7yFm{Lj7 z02PA8gXR_09&NA$tyZLunGteflzN)zd<3@5WwciIG~IvlCX54m1gnWP z6=G*LKE9)U>mqe&xrJgr#i4F_R!!XmWUF2MZl!Wl!=)0O-n4SEaI`6<317rS~pB!UbH5Gw`o{Kk2p(ezE6?{CsSM=8!OZlsJNRJA)n3p~RX zf%b#)Am`)N8%x+9)#t~7mF+x~#l{2Y4$Xb(3Hlmlt(A4%_)XoG)%PMtj_(ua-7u6* z5ySa84LQyPV5cfnSB@fEJr(>w-&K!UOXnpQXe0$o^!6zr7H&Og)M{D}TPmFoN>(h` zs@&!azBm73zgjR7?5|`P(sS2I{ZJPeu{GuDXV79ur`zO`t-8``(5aOPDrP#XZ^(+) z804=Ec(CP;?Tbap{ojj?F!_qy1x}eTt{UF0HtgVFmfaA`mEnSNIQvhRtgFM z>8oZPQxoeqI)k<^zOW_KB+R`&OCyo7xUu@q_=jrK$Lcrx<~#F2mKnCY_bIOA00!R? zsG!D-A8_$X%bzkIolay~l_=BH_lh=%0nAN+nlVY{p!d2#wEFCIAl?GQTR!K{E*M%+ z>L?SWo&IEmzSkB)5hzZXg9e^kyuG`vYJn&andtajL zDBHAgxq9G35dKVvw~2$#rmls@&Y|&VO8axWnm=AqZqxco+X{SDi5Td|m< zDXy|dcag~>2eg=d_g$|H)yaO-K#(O-Ht4uJbv0 zpd0=QbfJRiS*iFF#>?$0_k-~O-Ru%|Mkioe?ZBr4D;jgyZ}GO=WA0yDL+y^GqCr{Kd=k-=vOH>q~7P}{TfK-Cq&d3yiuEKk`foDypdTZFh* zogg`P-qgyvNX(hscd@7cD&1YecQ{zVNb^4Jd3M5t-$eCZVN3rL>rMO?@Y+lyIJEMMly-Hqg3O+EUWvuLA(Btwo;-7DqlB>jPc;6-x z7VFxs>{yk!c-J@8kK%!*(l4V{gXD%VRrXTN@yI*v6Sk)#10P1Cyz{A;*CP_BdL2$mx-uGe%u zVbwHBye%2MKfG)9)7)69p=2sLtW$M}qEFxMNi^N+2oBzSxd@r?PhUdWU*YDg zUcy3WFw3opRfc)d|Ly{=r}kR!Vd}f{^xnhmW<@R6(URe!Ywx^=&&7m2-7~oMY36;k zNU6g9)1E%JBKxK4S4CqBzutJVVAYM!$am$`73w`iqmWMjSF&i&yfuFFx9`24awG9s znM1nqCpiCU%ID_+$8XW>w$;wte1(6rsqYSR=20yo?#GdZPEYG&b%SQ!y5x@+-|Ejd zKekOg&uru1m5G$=jve2Nce+^X*B=X zx9X5z`{c?S6mmely5700H9Fc(ALo?Ro(&le&_;KqSDUU0J1|MR8i{MJd+wdWjqaOx=gy(^T=(@rjWAp$XQ-&Y1FvQn$Q{EDq8F_tD2$m_%wX1Y){__A=jQeZ zw@d3X-bK8A)4i9m^<^HC;E;F)LQvVbX2j;+K}+4bn2DrT+&kA#y-q#h4?p&olf2#a zo%A~j+s>}HBbEk#4toEi!R?u5*30v6f11V%XRjK~e(5W|7<`Bq)8J>yp8ocBb9ZgS z5yWT%3Ls&WRNQiaF8(xXF)ip;)(|4)*>9z*EP%|kQxCe}MTQBM-$woiY_G5Kj^3RN;A=21o>pPnP7 zSoj8PUtQGxy$xI+75I|4ZH_>;7QCqkdni~M5!%hT%sAa3~-1!SpN4inE%J7_` z5oA{N{0(X8R~tSl2;awREe$Nb*>EvtF!;cC?C!w11f8Ge;x#d3be-MC&##Vka`Ubi zO*~RnOjw$`TE%dAARZbnEOAO#->=viMR5DWu_UB~Alx z2Y@`u4ifOocN5*D1GEai<(nN_AiN%(Qb+=j_D4E+Wxa)l2_{KUmHyX6Ym!d+Z(l#F zxwVGMNa+H)(Hl!RHPm81UES45>FUSC_g2~kxc(avE}qgNU?l}AQU3dI;l4=?@_rZ= zm??CXJY>}L6q1PHP122;~wCd5?glGe^CWDY{YAXFFyo`H_U4f0L+np zbb!W>Ul##9tN8Ccn}BxtD^?@+W6j9Di`4M+wtv^!Lkg+5TWR?+*3DqH)z zIS{7cUcj3B04NX8aiUp2unhpzXa)QOHNUo4Zn^-g2DV}5f|nZ`5D(VIM0d2d>2|pA z{p%Wt;sX?2B6V#|@%t~yF12B1cV;ux**}~C-F)nmn*452YAB-=(oHLa$^8BN9-SkT zdz2{VfWur;`%{EZZ33s^-rL6X7$Y~|KGcvcH8vc{vD42;0-7uUar@fEh%l8aZAb;M z{q*jlZ+#z-y}uGjkp^H-PZi(^lPL%|QAMI{armFUn{t>!OE2`)9icc9br!fnfJ(fgGzI*%S6pa~f!)fFG&H2<)wSt-cGZ5yb^bw411ALgn ziHzlol(~g9Mz~D9jP0T1K;tAuU9)>e0mG~*?|`4A9+@__gigPWg5 z_ZI2_0UZs*Z%?$z=)7OSYDY8_dmXBIs$~89U7?G&jQezZ}|52 z;PuXcFE@a4NLS@y*R*GV7GIq$Ee2x$Q}w*zh<~xFF;Ll>*Yo(mF?sn=v1Q&2@E~Wl z4&Xs2x63K z*dM;P8`--uk7f1gS2wA4qTCJ%NI$U}h>{8HokeoNfXxj+-LUqc!{hY8eJ!)IVm$rFR+dg*KA);O`{xIH^^>zA9sqck=)*{DiEyURA*HnkupDoecsGFI&mM1kkRwQvm5-btJhD zEtGnJ9)jv~{V`vGP6GFc(~eC632$LzG3l9^G$%X3la*bLpO3VxZ!Y~ZD7;ZZIZ04n zwCLj*N1z!cYhH2g=$31-Okm3XD$b&3JAQ7@v)tTUI$LHv?+OS?9aqlJL0!sSui!a%;SJhk@nhnkL+;?Yr&Y*L&}|c71ka-?^jP zzs)cKC=gHg_}YkG;L{Bpb>=P9@-h|z5#M^)?m}Hn%$weGKYRnX@A2I9MalQPKY1>8 z;=#O)U_lozd9$6Rr_Vq!r|HZA~ z>py=a)X&vmzDq6u5qR&{G6MfzhP#5XL5sple!Y=XmQCWppI;IcmU<$UOvYb>7-rWF z2=dnXzWXK{ZONah5$7es`Hy!CZ_F(R7J7TEJ)eBeXU&*pIla=@*ub+U=K~rTsU1r<%{E%fu0~H##yYevoU&Wg3=p=~8|m>6+xnNQh~*jIakQmO z4o`Ae00MBGdU{u-IamuiWhOB&+cy1StbJYdmKB2^7`0;AaYDL@(Q%EmI#$b)dxbFK zWv7_ekD4x0T9&v%Gj(8u)D|FM*9c3rWGkPE+ARst*jM*m-s1B5PKhUbD-LUo)2 zU!Of4F*S74^nEHWs-qAlNX4ntI>x=zOJ2I_9c;q`HK3640FvOjJ(V^ZQ%9Vwfu!fL z$MP364XLYLD4YmgT;IK*Gi8@0@K{~^5w>YZCA({smF1YcB0vl#9fp(qV^kauWK+Hb z?k-Nb_4smlSy$F7r}MjCR$O^@4l^{;ypLQL9XfqTyuqxT+Ei*KJA)+TWcNNi?OA4OpV{0~)Dqrze)z6=G3K<#QH{ar zgI_2P8*|t-RH62YK_jr09Bws$pT)`10eF((Mwe|oSm9Dadv!%x<6uL=6^)cCAz5Rr zwd=#rBgiPjyIjnWst!~cn7v5~R?KS!raZJ?oSu=(=YCJ=>;9A|iK+#< z(8urc%AP^3Ykt#HKy&i}!W8aqw6eOQ&UAd8j2pW;pxHL0^UU17D$nl9rsn41so~pU zLiX^jJ&pl>>c?{p%>l)w@+aytSuL5v0LZOnW~Ds5w?Qkba~+Up0KPE!-%mCSR{~Fq zFsjehj7$RcL51I`#{YQfih@(^;slaC0F?pKmU4NmXh_%L`!e(vt3ZTY( zcCRL^LZu2|u=}MI*OeZgxcTepDYdJ=687aqHN3L3lv44fJ7?|%0Q@xdS8sSHP+0u^ zJDwz|VdL|0aHV^aEk5vl{9-+g-i*8=dF!@r*;Y?L38CshD!!yFi;`0i<;C&`St>oi zwuCg?oBXtq?o=c|HH68NIls}gaX!$KtngzGvNta~vUu}l`Ro8MkCzuK@V|`Pa@(@5%!Mi!@&% zWdCtA%6_NQxaNClPv{l$xwmneMSZlL$(m0dMZ3?;6y zFxavW_r`apBfwlwU;qLB*+T&;5eox3Jr7$BtS1%;tmgm5yMWc~k&A>FtOBev&HX&zR%ME(3 zZb?Z(tN@W$e!VR6>cbOPUKh-rZZiM!?b+-6w{*00Bwox@l4{b29^Lx$;jhd!+m4%je}9zxEv@hTQIiB-C-Fnk{6~*9^{2zXyjDH@ zrtrG4OVFL=83VP`V)69Xtnk7aZK< zRqrubwR8DTCy-f2mlap%)cdnu(GHvDpc{^3cV&Sc&YGO~@$8lA;Re;~+cM?{zG+l^ z>7YMo6)xzO-)jY3)SZi#f)&?{;^-T?2jJfFe(L>kHpK)$VE6%97RfZhhXFak#$0 zSRJN=M{Hr)HJMaB1v({t=eP9T4u`Hh4*q~MCHL90Vlh2aGgteIIPxgec?47^41hs& zAKe)9n0x#vG?bDLR0SE+uX_f&K5x5i829dp$g0{@PNf(MDzn-Eqx)l!P zR)C8Y)rM)>jU|AdAjV5g!r+rjw(QVttOEY-1w<(a-}jj?Q)PXgOwrLKm)?ACykvCr zla5h|k-GHyqfaP8!&I2+xlp5(;4Mxos-qr8-Nk7Gm$}FguQ%SAe5wue0Jgq;u30&3 zGBw4%P7YRCqqjoW38!>&jRDd3n$?cp>}+3$@|*<$1xISOaZ2SCWH!%)t^xP^GkHnJluJl{ol4N}Q4G8GgUG7q)%PNMzQj z&}q>2?Y!lSky9T}nP_iz3>Y&5;`W_cHV@eLZWJn8$XbDFa>z-p@EyH7O=MDMamM_0 z&||fyCM;vNXu8k_v_jRXx_KyzAM+xN0uK`Bb=`UCFi)RYGRzz^dc+z;w~H1x$^eya zz8wy?xG>SrUmRk0%xZ7)dtuXr?$0^?$3rh}m(%U5y!?6JRpJAi#;-rOdmDXb|J5#= zL*tQAfQNnP`~LNTm2?N_NqCLA1lIRl(n<6ki*b+63p1&g>_QeMuYi&gAe*4EhnMS0 z8eie+PZ)2artrf?$VxR!9G2+s`oL*`e&q2w-_)JoaW0gh>+O)-$GwW*^^Dw9y(FdY za_!AiTtG9W&{wRCL{iBHs~st+nYpr-gp29*k8A-@E0T@&0D%1i2O!s+kc2S)h1fiW z1MjY05%adaVjgO~tY;_My`^rM$oaX2`4}7Wk0>hM3=8S zo;gBZ)N`IIlR2dw%dlTTVgLN0>T7b&?uY_umFZ~jhVj=&3&ogi+v2x83{bbBLw4Y= zp~pwozLq^+1^hn2xp*FTBxVlRcd4@Qyf8l^!1*vIt>ptu_fuM54sns{7yqn`GUntw zDoa#}!E(In5S=ac_J1s|Ua_;H@YUqu)LIq$ryK037{ty5v4>tZ& zbk-9ZmB){)E;)Dq)8~@#v5vW9<`?$pIcW<(06s6ZjOP((~z6XQq3!L{!Rk}R% z-HvEI|C6s(+nP}~hz~EB=d~&qc=YLnU zCWOvApJZ$hGc?f9O@6O4Cs^Qxdx+9`J5b5o<+~z3Ng-mj_6-71xk7~ zS?l2qPy2xl*jYK$eqC&-TazZO4eX}ws7uaFh{J4G3#euuRkGTQO0{2+>j=HD;oEfY zq4%kXukXi;t#lJiPlpy>7k6cntO|+`f79uh+nG6MmN>t@&V1wRvE2;^QxCa+e%UUX zsHv_*t~fkmv+sPk%qaUJb>OfcAr)s5pK)sRw&Q`c*v$Uz?(x=_iOAbClV4slg5XqA zf<|4y<^`sc+ZgWAj~W4dT)&{JshIXnJEOcRbM)FFKK1`2?yaM$`nq@VLr4fn2-1fR z5$X2O-6A0&AfO=K-O_mg2>}5G2}x;0x(+FzAl)h59d{i+@B6*)xWBk#+&{i|3}nbttbl9SXA0PzAbScu}v2{t^>E z#v<{~Pu~ z+TLvLY=jpnI&Iay()o=?1ZbKkhl)?DTPuWaf0q1J@Zba$$#n9FXdAtgs1x5DFWtDpq^H!m{X zhB`R@-3u`8tY^mPsFrX6Qf!^cL*rpn&q}EK$|z2mufLHh{4v#G9|j@ZsLzEB5&mj0 z+v4_e?D4H|EQ{lAL7sTV_Qe-````zx=Y#g?uW$3mE>5T z(}b`k(!Yb;fOei0YUXB>fG)v)0|4T1S~U_a3ssU5Jq6oBQj{PnfFBek+H9|DxB5I+ z359M-Z~vJ++W<8N-7wYtHv_#1mHZzjaIhI8^Fo^bO0N1!7Zr%E~=`Mf;8C&mr z?HKp1>7MG9Gh=%wA=7rI?oGv(KJ+uD_17P5Vv!jD&tE%3%S00L{YDBBAfIOg+~5^p z#B;J9V};&MeHqeuUtdwIfIqVMIen7~+-oAGggOy_z*S)(#<*#^(z zL<%)Bz(;RCFNZrW;1-EMZ}o6uJ3tHj>`h{+WdO$wg7)$@m+8(!=ym^W0~ZzW45%`K zY4gs-lySNt=ncpesS!P40PxgUd&)xwj1y`@gzsl}UJZFcFa5Q`7rEEjAAlrvAno5r zg@vSessDoyz6xupa$^}u(b?N_X&}GhozH0_hToW3%~D|dXL(040y2YWj}$34&PT0| zaRJCmiRE3!IMl$gOjvT9t+r@Fx?@r6zSlg9>_ceK-~sq$uPV$y^z{k2RrU8f4mU{U z2MynOksi1|dBB>;o+kqv(FS#?|A&nb5TNH(2@_Qn`H3gC7Zj}p_i?Dc{4z^oLCmXh%06n>xNy-M24Pvm5GVSg%ym zaav{+iGUYzy!^$o_Ku2Oj}P`}XE%-Jb>VFBBT~g{d(sb`KaNv)TnJY7UW0sRhCjIF zi#+g!`V-7kY@Pow2c0>*AP03d30w61>`}FYk?TZ{@Nu)y!?vg9EFZdPoCOdFV28(06ffY$1BliFlLlCKrz|HQw1~=DG0h524LY;i$sDai+f^c4p(JWuK_*Nn&v-6fGc)lcO@qwe`ZE5ZqL{vwlK5 zonB()_JQwRw$_KHhb_C$fAK$!M7S*6P3%RQv=Y!7WQ7SZ(N$Z(JU7%nt_@Gl;S<9Z zJ*GezT>p`oeXGPvo2W(+KD?n7alCqt*zOVIwMS_rgN;Tnv@cCpv9GGPl@^Z~Qc|=R z9T)oKwY+G>kd?uuGh>HC7S5tW11$=90v&RfO zA_djz`Nf>Ca-IkOIwZrDOLMvRKb?*31mKx}Rq%q7G+Vvp01v3~==XWFjD;E)Km`B8 zp26}!dZN(eY%%73_|1-KLXo_$hMFf9={@o6S@oOA@&m)i2dpM_`#F`peICybVY*Ll ziDS1LV-Xh*TWHo%n4W+6rGkpQB!l=^1*?UY&th?#HHDB!`u!p||-LIDI5HVl2o9ZoHfhDpZ)NoCc&#}kfo1VRhK-_7+SM*g|6P8w%>z8xyPSTs{ugfZn zw?thRPiv*lZlWrM_NKI}e7=PDPVv^BN;dE{t7Oxv&Kg^lAxa7*_Q?uDQxEC@X{o0P z{BdaIV`T4uksGin$47wwyu1ut9M+!&j2~)JOSk#=nG;yA5#r{sFHabZsqfws{k#!l zpe>0lepubWvRJMWhN{c8MKB3xjV&Tw+B47c-|t^eh{G*4Ck#w0Q_#_V?ss!Mb!d6A zt!r|%5lIaHL;rvv33+$sDF;x0{m)K^wbC99SiN%ZlVeusI|sm>@4hE7z?MQ~NV_f( z*kJLoWTF~fc+YdS+Da`^a(S4o-JCtXKjZm9IN2fA>}P_*%5dY@*Ud0cq;f1`W7`+< zUvKR=p#=W@X#^tebgh+I_2A{7=$!HW?K{(M?rOw1gQ3PvPvFi7yN|b5PFQ^;k(Ps7 z+Vb9?1)7$ZvcA?6b(-x>u!Ah6GOxxn7C4%&kKnOJEB!V_rF%e9-p;_p7i>Yv(YhqM z@eW$x0{*S#D+7MGJ1F*wG}}#=6UdxJ5KPw|Ryb0<>OGbfRns-ZzV!PZeJ*{E`M3Ra zxr6Bw3%DU7;0~cu^p8J{#-4QdD{@iI{Wry6A+AyC^`lv@V))Cl&PHvC;jJ$}E3W7#$km%g!0ba6@Dv0yH^n|pYtAyNHh}kEF_>1*T%|H7tijf4zD9`yo&N*ru3J#IBgomr}ew@<_?ukRqKt zw-^l|68@&;tmcE!Z9q1^4E{sOa7*@H%&0g61EmNCm%n`yi5hZuKH0#o{5tLF-*T>C z^fF}Kv+6e^ILbXF2|EC9yh9LZf`3g{THn#4%^Sdfew(zpBK0mdM1vj1J}^<@w@FsR zE}k1a_sDprv}5TOV^`2H_07-naH8m4T;YJmC!BQb77uWZbd6$6bjliqoY@~x%O}aK zczr9sD9Hiqzgw^Vuq1f(_e%cg?ZcQ~``3!rg->C5-}F(Q2QG5YC-*mFZdvcwy=o<5 z*V`jBbz^yzV0ZP@?1*Ncs5gmojf9KZz@_cJx@c^&t$|mrsO77T@v*(x>C?TQNw`L? z97$QVwSdh2dT`=|j-eiTXU^DeqZZ|VSZ~!I8til`>TE4)W97Cq77Su}kK+%8u^uEK z?70gY3KwZ~thm#BXsL}-FEBScFRZ9HUunp}Cep^V5`T|hP1BvPDfh z6w&jzPrWEh)}=}Gjd-MN@TbG|XKbFa#9t~mO3Z}nPYbO$bNAKrJaDxtRcxn`frt&% zf&&jZr~D-I5h6=7gR4tFzVCMIh`XEpE~<3cShnVqo8?-79y zzJEMl-0VBy?mj`CyKN-#JYc@Eo2s>-`R!AVA&PmVRhIjst#%f%qMR~(6|uU0{__xP zfG84Mn&sn*5T~=9BG&?~Z$vla=a>(CPD(G>IRx$vicR1(VjQWtJ?2XpH(*n7FNXC zhuw9J`#zmZj>A*p^@zDbxk!>!&iwhUi z@h?SOV~d0nd((*yOnh@Rmz+1_UTUK$r%ddl+V3q7_I_=Bt9ZG7Niyn(NIHytdB}t9 z>)L;sMquQ<7IbIGTmqYk)_D6su=EA7dVzD}L*wW99=H_Bsvofa?ca`bJz!r?A<%F6 zcv98HPy%heuBMS#M#CGfFjQ`M zVu5n^0T|-^)s;Qdz8%kEZwLFeNq+U!l5#fn_@3X{rT6lW^r@aSjAkwwPwwxhe?q>a2*ViUZ1n_dl^7luR9_$9y#QADjGOef~LsdP}WZWLn&t)c( zik>?dY&+OoSv~4aK7k2bXT6_U7d(=^YTB%okx_Tw3MCAZqf<@}c3`(Bi9|3m+8#2k zKfc6%>lTStfIg2NFc)*~f=W9Vkj+;p++QBwKDfDka`O>0d&T6+oO>r3Q-mbs>e76e z`>*dO8UD?R>zgcP)LN)R<;3`dvEgaL z<(8IGi&XguHa_D1&KNZ6qU%mRn$2IzC1aDS4lnAzu5CfOP=#>jApuE{pY8$Xt2b|v zRKgiR4=F;s=id0q9C~>TF^E_y=^rnIxv!EIln72XuCj?dmh#}5jf#!um1}0l$=XV= z2ahyx?^~!n&2qJ$qON^jRD7b`B;s}V#|M#^o=O(!6Ogwg4rvo+i|<}yn@qGjtk1Qd zxeVufd_52HIA~?rKZaXtWNTrs*T-UvQkae`$9v2VVE?S|(7;WSSyb7owvBu7Vq&iVswWp3Nw#0|G6VXS%$#5LDPqM`U zo;E9s!&mz$+XY6|7!_eXr?&t5{{G!oa{KAdB!F%|K81OW)?Za{J1(T}!-;T4hy&_+ z1yFQfM%Qr!yoE6C{M0B9Z{do{iL2XQ$X-R}y6gCU=Tgk3pw?LtmO7%sZAJfRcc;PLail0` zsHX3o;s(d%8*;fL%7}#5N`_&DquyB;$2MOKcAmdlAwQ9FneLPu{QVIU%fZ%@C35 z&9@xXnHtC9hBw@+oIaE(9LCSCPyU<}RRs6C9}FCHHH_^quta@5-?UY1h&C=WK-*V1 z*_cHX2}tk1M`rW$+fZM$Vx)sqBjsYj_G82{k^{V*&HY3d(knj9y`K7{Pq%)d?p76sv5`w3k)m6-$ zIk|N~&Ah<%f8x9+3!2ZlZ;tmPXz^)a(U_Zjoh-)wZzI~73!nZbjlPDj{m4Z0+-K3q z^;$vSAKxHMQq=A}cV5#d`zcyj(!;=F)|Zv8J(ahn(GaOFeuL`hW=b@yU`~LIj#ZYY`N1PSsylaA)hk*u4BR9q_ITd>u21zcKGEr|wr77I!Hh4s zpT>>s`~{Hu`R^5+eb)ci36Qwpn8yF!36M*c)*)*N{LvHZ`xYMCtI8#Gk@Cu1mGAnQ zXftKb-))^jl3(vUMXf!(iSbwLuPHSDght5kybxkZvUhwbU+%UL-(i!tx|WT)$@e(G zEvT2QZJ*|Ym>acdzIFiCAbR;3yija=>PbRF@=3beLx6YF(QYLBW1}L<{I4$feo(0j z)2Tc4p4&IxafPx6!1JIjaO=+Dt&eX}RBlTjbmpw(P23HD@hhFE`qvGzLPd==-SvU{ zb!W}hSUbc|G1a=&u$p_B;k`jCAn@SOVgRxA#W~S^e@1xMbRe?&{b}n9rY$*qsvMY) z=9;Q25&Wn2oW;fB6lZh0=Ac;aiJ}sB=TZ(_yzqooiRVMIElqnOx2D5!FF!5?FTtsH zDLgO-BirmW|EufLsmG@!oHNrUr^GTwc}6B;Y-d~j+K0&{29c3TkC>b0zFX5UJjB_7Q2fV>vZ(d#D| z!PnuC8xY|ior+z9`PH29GD!^7Fjnzro1!;enBrlaki8#QYfib$_NKw+tJ8Z`mb>*$ z1$t7y_$!I*w>MeM^&y(m%8!*Alw6&7FrpS~wjBMO7E_u5c;Pk951?0CSS!3BINR=W z-#a>;-9RAV-_t)*r|Z&>xtx2cn4rxDps7wt#jBP>l;FI(F?{wSAH0MDnqF-g z`DH0TM^AHt&8d>l%8m16XD)qNbpOrHWai#;_exwEL3Huki~GOX--)lzuoG2WT%VMz zPd61kPe}!X?SDJ%Twg1hX`(0yZQNyOmCwVaCwsCzw$75DBhow8A7k|E5&g#DlCA)P zjvO!GB&aWYNBYjbaX(2v%{R;hy6WqB9##!8OurZ2cU{vSXkC)Y!jwFH4Q)S&c~MV| z&8h@wHi4(a(>P!3c0mc@hLMlBt(x3YoZAsmA77U}^KjR~^_yZpH%>?<-`oX!I%%Hw z@m(_Sh};^+EG3V{>KbHIV#7mdW4E4^8vd;}8Iw|q0<5Cl#)xSA%6(n@MYErDNVL_# z7!N)eUwrPkaCIl&nWpST(_86VsMN5jY)03-q&s$H(C} zGxR3B<%zE!i9e^y^}itwn%a$Sxe~^4+?_cdwUCd!3X2;eNMzRleT}X~0(99g-u4;6 zQm(^}Tb&KF5S&l^yQX;ORF4#t#y95UulM)R2KH3)gPTYGH$zOH73pVIega~ zw>p{r^uD3u!HJAj~!ns<5?aQ>UU@UIwm(Uo7(O8!!WkD_lqeJ3{n3y9zWc?lVVa4ap+u^ZqTet zfk(rc<}hX0;u}L&(jYJ?4PV{*RBhuTtZNYJCQN|GOJ} zaoBP~dk5vlNx7?&baD9q#=KQ8mX6TWrVHEd+32QnlZ^#^?`WH2j;j{gOJp^h6& zxaDw;i5_}-RpGBP_-Ffm1!UqosL;$_D8K*Ev3zu|;S=WX@FWix4X?olyhS=7v zB$Z+c`IF9w%U&H+9KzomQC~?VaE(1jG=;jU!;bw=O3{=eO{0w7CZP8y5fPwBO2}TZ zB@o7cAnplYj{b<=Bg`fVep$p7zmmjd{rE8;Ad*($N2 z6LMr^KTkhT{Va5F_*!Ml56d!BCD<+JurPXQi%!1WyQD#lx7erVm_*y&TiqZDn@Hl)CBmk^fioN$gsQn3)qcUib+?zQ%sx+(R*9?#*-RlQ@k7R3yFK!e?YIo=;Pxf zavGRcOfJKg_X956&%ITcqS=k4&gJyz;}G4u5k6-`@p~ksE3+-=2}TOgoUjb5BD=h|8Oz*^ z7DiO~st4T~gfFpRc(fGlbob$$B7IcwHvcu;x_N zEMG-B>=&5*MPnlaxBJ}m?sT0Se$M1|y+cMd;^H^WhGUc!n zLAZZd;7|J9l7~@8-dLTgDRlGsuH5DK?>FJp&(Gbq$=NBKsRYYrW2)zt+#A#m3j7r! z-p=?!lx>|GU7}E@s3_&fs`Bs+n|cD*D9P`M!zxSiM*Vg-PvQqDlt|dxi?BB7zI^Gzkb25j z%LT;`Pe_=n?ms_YOs8jrI|dSaWNF4dRqrpTDa2fhJCZsed%UG(z~p&!t1rpCQ#yFJrlM84{jIbIV=#A=!eO*rCSwwh0y?d7j?jB$_i_>!_9%eG^pgq zmCs`7A|w2UI!YDNaOb0VK;r|~kmX@|T1#eJlmz)}hQMWRid8Lx*Q*-0GSM6H%fq+a zFY!%fXdecYLY7|_nCuRl>{Ftul6d0&;E7ls#tz&@mtgevMy<1!3{!wUh71Q|7nYivt!Sx{pi#A=Pt)_anU*ou`XBRNDM!Dzy_M+&sP_587kKNc9EO*S=P? zDRDzoQATRp$ZXGICCupHlKOdZVE(8g+uU;)p{8LwDi2W=z)$sM@5$*QG%?{%K)iYR z%yx9;W0}=eTnsK5o7bPnk#paMaeZE+Kpub7M*?1KQbY$r3l0BfCnx zhwC3CITm&AKd!q48Z644J6Q<*ApH}k$E@nBrfm|ClQl{YLr>1FJe@ZI78ak^c@gID zYaK&QuV_U-elD@BlWR&w_z#NU83+vRV>$lJB0Nwm0FgIL_!5Y^fEKyOWBIlPDA+V5 zEm8+HZMApzoYzVS17sjgd;ZVIgTk@+*sHkT@WOxjvq~SVH!O&C5%h_yzXtxJ5j}Qb znqi(E-XYJ4x>kB^P-36kV{qiAC6(U&JK_&8>4v@z zg??;26j}-sRD@FV{f;gTpFD7-sh2%iNArO4;XSO@#&&qnu9n?IG==TlAwk zqUim78+17}8%$z0a_Ge}0aisi*@^}-EvV>!wOaYUHl%p3;&HmA1Z^}7!vg4}%L}HU z@w*bL_})Fo)6y-v8CV1;2nGVhx*z+EuQ4MeuxKT!wQteCJqU1uP+TBH8;MZp=N-_U zP=&TkXcVEM=+UDQ5JCqSDlieB@Nr3c33L})F4PuoNYRxmm$BiyTviU%0; z{AW^nf+yg!D_C@>;BOFW6n35+enNckweWx^=n0Uz1r$C;I5c2IT6=K|&BI0s9VQF5 zkxAcmAp@S$Qcz>X*;J*59u6Vgdl1+I`HiY;`V%s$ql&s>#-AWhgYwFmejWw67)+f? z^MSWIiLF7cpT1kofhn~}9>W4sKkU#8tog(HU>92GUZ@J@4fX9RG{6ionC3QvqbE07 z72{|a1upNAKlTWh=z*!0ceV?(>Vwe59gm1-kjJndqCR~AX2lWpuA2(bc6w>xl!M(c zxhPw*E)0J$5YnQe1WOP?s!^&0CM-(Vm;+7uow2YyRjxcTUr$8#D_NG_u|AJ%*{43{ zoNaLVO3`B`lI}S-D^^Bsmn#s`lY3rZ4T z8{|(38vzzV0Jve-^=aC3xm?@#e&VtQ&F>>(yg*dQGsw844Y(IADlXiH&OB*^2)MTIddKxf_|;!<7A5g;_`eJ=hY51fyDVTnF(@tY8!J@hXzQA;4N zF!su^m$VG{|9;t=M)}}P>zn-{bg<}_f|?CG+<2ImP^C|?8u)Sx@LWfZ<6ni5D{i5| z8bN;wZSiLQplLgbuJAw$8(8q&Rb4Jq7?$p{~ASF6?*)C^qK*u9WAC?K`>iVBu+h4ewXnc@FP z2_tY3Qj=Bywt&irQy7fCK(bL)>I)oA^;I$#-uN_ftmGKA!MJ*yNLHWmzB0F~7&d&*of z4_mWX3LRn&J%*zu$e%82_%l-N#HY$PqR@Lr2MYpUc2C@H=V(#YUWPW2VE%$gY54bE zF(aHpB6jiZKQbK6@NDqE6;sO+i($$1Kn3%OW8yee>)@t=lGqs>j?Sba<*VC2+Jp%rGqnMor^6jm7xv zXQ_mpSC4vXzIQRglckk{Kn#GqtUli;Rz#W|cv@ByPapv5*m3LyL_9)M8J<@@lTy*? z=rsO_8IRQRuNdqa-$^9A89M`l1nP$Ko7%^hUpy z!pr52jipoQ^025UZmOb%+viD1Xb7KLD$gATx#*{Qh(_slepGyru8R(`cUB%!Im!fowDYu zhQSj<3;^vV6|1$~bNcQb2szA4Acq0LP#m87`F8ExBADo8GxJ#cL9LP@LIRSLF68JW zn3BwK`SX+D*?6`rj#i02$>G|d{3of95l%MS$tn%ESn4OCtwES(>qDQ`Cn{OIPIqEP zzGyy5sP#E_g58`g71|ze9Ed!52}V?O>m2Cprt3ak3D}GiU=uNhqX$KSSzw0b!m%Nb1hbzb@-}v#SuA&atMj@R*%)PoXHuupnzW(6x!!-8l<}g;4kvRk9Y=gi zG(a_~>1ozXqc{F*t7@q%wSjVnQMp9UV-dCn4fIQVsAOMhQ2Fk>i59CN`t|p!4&I51 z`8nRTp{6k`n~8G!*W$opnSOx6 zdu#yyo+82DdPNXSX|>${_J&pW>qn7N{f`nT%U}{4RxJh=C`%Q>+DkYh8H8a16u!`? zCxMAhUfz1NFgNydWCk9kKmqPjG&%3y_32y@dtO}YH|I6e!AwyyLAww131GOo6-C zQ$Nil2li> z!I(M;n6qv9zS~q(k;ri{US=##%<)3#199)ZW)GRtD$ncQ#D&!M(W2*%DugY5P-!kZ z{`r~9mi|yr^%211jC=r-tjtGFB~Rc=O(*LHAtNgRIZ7!f6=5>{OBT<>*qe(>4V8>$ zsn?Y1A4Gzd;Y^+5frdl#8Fve&zvj?;yc7TPCDU@_Dv33cy zI9ohQapzFHXh+#?>+sWvD-!wHhMgZz%)G$&MP)NtYYpayz~PCH6KjCPUZQmIWf?Pd2eZ0p2&3Rtz+`c zFtI%dNugW-bq`oQ>0Sc6P2!5BB23CD5Wl_R6}#NxUGY2HTUbCSF`URNn3Wkd5_^8& zw@&VTpcEgyfSgEHFaz^+Bfc`{-=kmxmOo_(Wh_a{M@kGx{co?`Kl{InUIjpm{@OqV zp|jGxovWNO)adQ5Oe&oque|V?dyrs$^#^Z*ZaNU(A&4s6@FY(DFEtxDUUE}KW2Fox zZNUm3SXE54!(TPWzax%^HQEQ&<9n{J8^^=2za>Uz$078!XSGrl9y&Et-xl-;T@CQO z@>qDRLNkiFqC|3!1zu}E*O`Cc{LvzLIlKNbiIu=E#kOS^{-4kXcvdnOrv(U|5D6W> zT{KxL@$Fy3s4{ITz!`-l00b7ALcLtBl*IOCw2-71JZNi2BQ~=){@HmmfjC_VXX8=A z#Z`YQZ{!=X^W}Gu=sz+=Db+=f6E9SFQk;slyK}-P!ttKVJ>!Xx4#m|-?XR8lGZ-c1 zc`xSc<%)X}LTdE%lq$D7{x|TfkK!}JJE=Vzz`(T?7+MCy%3)L0R^|yX{Sd}&k3e9f zjgT1CaEWjRfb}i#0^;cYxS2yYYqxWG+%?6vNuV=J{#i07&)Js3Tq}q2F1d*Pt6!d7 zBXCkUio8QPNxE~o&9)0)CnXdTpR>DWKI_#9Gh%Q!87tdgXMhL4L)fJ9W{*61?66ig zvthK{^Xt~6XG66+gW*x{Z@VD>Q@f{n>JPK8()gV*J}bXt(>Y+JaUeArJj38oq z=FTt^FHcwejB)C$lhk;A`kV86m}iBEn)Pe$ZnG?qLY!3pv#(o|77 z!tdrZcrpnc5nT3HAKxr%&=$;|F|ZqaC%2vRe?IdraZ#j(&7JL+*mDZ$@3qqHX(Yp6 zb`_@UJaHJjTqWv#>|c5ep1c|R{6N5NCMFV3WIkm8XaB`ceoj0>a%S?QO8N7V^3f#9 z8q+Cn z_XEJX^6P={xR3t)!rTWBF_43<9BSmqN$$|r2RqX@t!ZrGV3Sm^`6uIkrh1)}Qwbz_ zLAx?xk)^$?X+*sk3#-3TwF&QrzoIsrQ13Hekp>hIg6k;~jP`QN82rU2F<9>n^S{&- zvao6yR72LhPj_+*2N%c$eKqR%ztxVPwy$oul96&6OU5u~0KY*@Sf~fHd|$uGnu-t^ zot0Xe)NY%qkNxOZ>BM%Il#M_QYPjq9fzKk9{v(HhQx(5`X@2?36^s$9eaE^C$r=3| zgDJV;%%K@?Xh`D8=Cssn;S!)-!*QPhVs3qn>FIJFU%VKCLppLT`z@cWeEPdP{_WKv zyq+?Z!%!p((MX^{*zL*K#+cBD$XCzm+McRSdI^uy^e2TH#1NxYrW5&xJ=VR_wB^^O z?^&Zoy&(8FStwTg=OHR8FegL1q$UxTn_ndk%9QI2ju3?o=`j$ z?!K$7s>CR9H6T#`(NYj=y8SxA_mS$Xl*3glWQ?>qjJ8hXV-O63)U-()52RnX!2Q|j z#=|beCSc`&bi>v!w&q5ip4P*CBjYRu{l)0*r1z(x`q|`qXdVOY-f`eL>7e0dEk#cb zL+G6`Qc)AItACM3M}RaM9nhln9QiH=IFQctS$aC)LMSoOdvaptyI!U(Yv2GHZWOiB zl7&wCXR0TkC^LJJ!ha<=3I9rP3QOeL;;o8)<37@Ngn)$gA_$~RcbqFPnF~v-u;tgW z2IBuKdHshgr|Zd??dj<^V{90(Q#!(km<H`Yy)%58UDf;hUia!0MdO7{u=HOusWCbAT>5$=Q@02C8nnXh)Et+%Wh{t zdTa=r=NZJo(n5PHB0-vn@!BR7u&{T`f2m1@e&#)V#ruy>2#*g8_cm zkr-f8xjjXwExI7aFGjdNkVN5w_kS4KP+>KN{vi4RZN6YwaDW3b;1K{#Za`}CS?U1o z`nGnQ2ZQGbjRN3s-y;^|H=}ae2+uO;z<)`DWxfX#Wvcx3eeOc}AlON&kfW*b?P;he z7B_m`w@U7LWWk}YKgZ8O^tAi9)49LF5^T}_sTrjv& z@xr{A4t_tN`G}F^mjo^9J?WOMybhBYbN6rw7xXP=_+j2adxPcSKH`&j;^8y-WAE|4+YCj)g`KwYRz&=Qul&o4vp>3ZekhsVH^O~e^|+mq(06EWFy`;~ zQHLc<{?+{B=%_digHbe=MuI?H!jF?|m{^1kWT@tXO!qe3THL~b;)niFJE*fbg^rZ{ zglLkgOMZHevoO>ueP*+6A>@7ndr+Ki-=j%A?N4C&D_^#r<)@zM?oP&c)(>m0uQz8f z_=mqdXHnjg<0C@zB>s8E4RQ3qmmvDPfWnecord!&;NxL367lj60K4}g2*8ZvuD7pd z1GFPhz`lV(eUlWD3X z-UefHecXh+8cd}&E}IFS_HP?8>FvJz$#H=B&9#Pgv2po5**VtGMSJ4* zPeEbBWy<1IA`S}IjkYN@9&DGILf)(F7^h@!cisHLEVBAyVns#h)o);S8Z%9nPsHrT zsJFjYBi37U%2(C~dh?T*?s~e!ci~4{vq@2XM)bxS~q!*3!ELLSi?#M%_-Hlp5b)WI4g^%A?9%^|3M;Htm46gacz z@*|&Q*e8E>DpI%U*3E2dww!Wz#2-`JN=L3y>6qJlg}vddnRNfUlxEMU`gCt5e&&r) z{vE54FR=%sy-P7Ey&q*#7Yp=@*z)|u@GO7k$<3)>Wb1sJ1j2M`n7i=x`R*&o>@Pq* zchiK^Lqj_I;G2uxsO)R*^cw58V~4~VdcU4WHY}1CIF39x@HiS@5Xqt7WW{Kar7%p9 z97IvD{vu5gg$K6k)g=$8!;QdqM&S}$K--fTg0Tn!pUMGBRzPk}?SOSg?BJ4+J)>3y zWFc81F|XiE(x;L$By=`2A7kh}uEn`;<+_ za1}kC1(1NSCN~Rt;r)r=A?cHmI_b@QG6+x`sMwoOy?8o zv(<5H2sFaflRUt;t68=Io_{&`TlA|xJ0MHM>k!LHj5B)sFYVt_0S+F+GvM3{&hH)lpv5~%FNfX-vg#MU5%O$szKj*QVe8;E)oZpA6%mN>vo9YT zGV-W38BlOd888loQhU)m|L)(IQ@`?*il4DbQ%ZScJ3-)dmVtPwLBs!D4YTi1UqH;32e`-dfLuNL zc?=+ZCtU)RFhEgiE`GJcv~dS?zuEd0BV2ivL$)loaip;|1*bXsyZf$buZ1R;+u0i1 zccJ6MFS@l(qD}94jR?Qb(wu0JH!T?B!sVcM-k$8%BA#0}$<1`Rj*2Qims z*XVF4Y%;jheu3LQKv}z|k%h0pXs_Wjz9~Iotxx)-c1NnntK_TcrW5SlWvaugmG{;Y zw@IB9s><`V9j?Jtd<$UYO#cr}yGOI2D!!Tu@|SVJi#pDL(f0-W_KEpS++I&!8h)U7xXu(`ZG<@*+(2U&22MR7ev$*pRgO^L&pF5 zGZNm)_dt4m<(@lIOJ-DB2^e{thd3OpdkrovPDGKQxK1SGln6Vmv`mt0?wK6ow{N?g z)rqFnSW{qYf3tq*&XEK4HvIO6q`rDPbSA91)2xT4l99+~Fo%gt|8?+Yl(xdpdw`lz zlLTs`7J+^#36^b)%#-EDfoCqY>x=*8e{+GEn%PTy(|d2sg8rrRL5JdWldn8Ac?{`b zZLQ`^gX8_@9L5nRvk1yD`}q!ImOYtJmh9tJIggpN*dW-(j+&N#YXT^j16Wk4aIOQl zDr#Cl{FZ59K!O><7sV>*;SzCwu_9Vw5NvY%I)G30pn%EdUtp2(Pu?d}S^9Zospyzm zTAvnGhaLb+WW;W!2RMbsFYKNCb-?)2FuUHT$wJXH`D!a<{w2*RLt6(Mw{S;gRA3m{$@sPLRPtqhs+Ghb|04|yrI|8L!|7@BWLB;yY#nxGLVPqgWfE@0l-b4DY z7v19v??FSrh`0ci-UyoB8}!;-nvxDMj0(sv`n8kzk4RXgnEfJRksIM7&bz zpNykAwzyCJ7r4)x0m!O(3FKL~eEh2XDdunN&qBujzqpXAC&y3XY7^i)|NVS-fDVe~ zfiC_wQ;I?cRsxWWOpviB$WylBsT`15A%wm)t*0Ud9mLy6fD!^OCQJZ=F4k*s11qy5 z9akOtRBA?0xsVg{AaV>n*eoaYFB zxF~cWfuyixGRR-6$b9Kjz910%TS}x5-F?noP{JY2=xAJTwko{QgJH}d!9td$PaT?H z%_+UBT|c(!HTX(p@Q)`;yIV^02t*V3{4dvTpx2{uIKnjJf3V)qBnn*%qvra9BT7^# z&uR^j_!{>R+@Sr~fQQ*kDQ{JJR2N=0zC{hIM zAAP9g83@>yL6Y@@!kqEXk4Q}K`^!l5+Ktwaz!5@T-GR<53|aBvM(;-oWrLjT|3@Uq zYv&PQCHLq2)srzThihE`u>H`pN~MunSdVjFUt82miWY@|KC9pGqGFC69~2`V(wYB8 znwqtPi}lJpqrz%P*~VAGB{G$i{$IpC+F`zR0C|hqSWcj$PUojVda~K3^X1S0WdVw> zL^P*g-;58ds9@QU^Fcl&4i5p#pM5A@VaG5l1-&nxX`xp`U-_@J)xk&pI!cg}x_I>| z(4g|g3nDARv7-hl=r{Fh8k-%>8B>*N?F9Kt78=m(z?HWqL^!|7p%&gy1`=XXx7ivO z;3}%kKD;n)pzhAB#t}w-cn~AJwptN7!ypeTdvy-ayq*letPzTT$c>6^alkQF-}%jz z^S}Cwyo_Kf#Wk_fqw&2j^rv6ALFqKz#W99at+$S#V%1qqsrGMaZ$b9aJqh5X{0V^h z9iy4T<^LwX|5Gz58XU3;QC>l&^XofP0fqB&;q&t6k)Qwr&P*TdJQN3-mOJ7nDWwq- z(a3HD8Y;j{wmR92Zi#I79Xk06iu+2b*(P0T9R{_hqya|l^b7c7O6Nezd!d@gEV-FO zU^}YNm2NEy$ZK4jIY?Y|E^C;+#w%6i%Og3XnNS==L*4jM*bmzBSX2v9#xdvd-^Bpy zBVx5L1eM&NG<#c+cDE0{vVJnE^Q{R%g|0s4{%!%c!D>2xBZ3^cad}ZfELzSFiC%9z z75_X~s~gdIhgxSIhj4^ELwkcKG1-$<(k*3<8Hd$kZxoIG{B~K{&pz?QBD%@j(PqO2#`%aIjsx8=xJ|`*Z*9!9gCiLc-A*v3SJ0!0cp!QFuY$C8-4r( z*$5Do^zPtdApLA}+evmzefJuyR z#dp8})&qziCT&nXWxy%~dWQ=Wyj8FkWsB$H^wSG<$k5#u?4@s@0{JyR{mFf1y-q+A zX_$$%OD57yo~s>-)p8(oBeG}4_a2uP=dbVzsCqNI^-q`NysK%_wdX;8XB zy2V9FcZ26%zxUmHpYPjyobQY?&VLLTto1ziGw*rNdClv(9^=M#pKtA8(}Wj?-A-kE zhlxxJ^MacWjwW0`{;@G+{mO}*$T4M-)wFonw%XJz->5&~C1sdWX4GmNWnUe_y>G?| zPPUR>E3)za4H1S2Db>KrRPoTbP3P+~>lKCjtrvU_%i?9TMlvVna>uI|_GxF|)#-Cm z^5{P8Jm*H!6a+rRc8D|xt`-#ekR!IhF$fDUX9^R9fD0|%d?Q2!`w=@)ryV31fdR>h z@UXV~QV?;JH{|O)sbbpW@@xz(6217(J7yd%L}tf zj(9fV)J;vq^9bU|{Wk0t=KyN+*S|lH2_KvCFaF5wc^4SY*4p=SW8#5^iSZLoN?YsU z(`P6A7Jt<0sMuxC*M5}|uEovRt;X%_Kc#)Nv_v0CeIJ_qlXcT_N-LCx7osxOj}}#s z2w}LL!G8T5qKuHx#M-UUuZIGOjUC$<6CY;Kzs3$s$4q*bITQo&CMA_0Ku4#=H)Faa zF$__YEyX0ezICP#W(?8%;%^(fx276w&`fobmx!0z5xp88i?iOp&jLxZCkTi{SmOMW z96;Lsw2(1G`%9i$7RkEC9flVH-HU82CZU<4fERsW=B^sR{(s9gj#FUUhTmUp58OEL zQeDKLi0W*JXyR=DXTCXh19oW6ID_2wVSaDYX|x+E)?Q-YITIwOvxxm|us@n#@>*Lz z{s!$EcR0emb3LMYXEhYImMgecPn_5H&F9kes_u^tr@RtAVvGMuG>6Nk$;y#w0<-AO{}_G`0}$}GsCl&Pevh_`&S5q#H>4~!y84e7^Vis zLE=^Gvu_8$o4a7e;-7AiKAS~s*EvIVL~uIpbQ)5a(ckbajb2ChN+~QIBX@@(;r_TL3&eu#AUWON z&}J$LB;7=aQEg;mlkJ1Nn3p+2O5;_B{pM@hzp5)6gWWo2gFF$o*~^+_mb|J%K$K~D zO>*9-%!5jDSqjqgKYO@(UG+=geS1b=M@N!y_3iILA*JnQ46kwxsdlS8vGm?UC~VY_ z9eTr@p>CMCO6F;Xck>ZSF7J)v1_kv^9TfOVVfg z6rZtMGcHCRujO60W(3q=y%q@~P@PNBxd=%k$&?ZJ@j_B-@GA=KFoH>xGxDVijElBZ zJcOBSokJPfnOb=O)T?dzKb#vFykhX_hL{tC^sq}`1-qA4zVkz^Fj%d}L-eLDPX0Ev(nv1@n+NAi(?NH@_T3$}Oj-=Q9FfL^v zxlfjA^>F@8}fh zwt9xNkKR4_UTvMdo%*x%$Pt9Bv}XB>6Gj>Pzv-f)>#`0jgPI(rj*bLOj0Rn|;wRfg zE#QC*EsE@n#bw9)f~WS}T-Dz+!PqWz0;4whRS2C9J(5IRSd?tsT%0`B(>F|)^f*%e z>|iC$xi?lm)Fu0bh$~3}!B5|V)f~}JrfN*QC(jtpGsLK#sxwv(N9afnZ#hdimBfkR zN)gEPuV(kwPM<&;$f~KlX1Bkc86&$hykeS{zt+@ymcBHf0xicIC`3C{;O-6{##H)B zipaF2`F?d^f*ulg_g5tj|JdZ9N?aS)zzvI`xftP zbNZ~v+n`SV$KZzTEW5>kTB~+3`ZE@zinvO2gj?Sg-@Cf2HKxv_Lq6$qK6!fW8N0-O zSQ)GS#C5W==B6szX2?pLrZ5w|)u>vZK_U8Pf2}5Mys=_t2zNDcPdooryw0Yf8ArZd-gP)tz?or{FTl}FegQv+k`(9iO{Dx8_`hGpZd@6A5OUt(3`1l#sbS)Pf&4P0 z;_uSh`ottOX9lTg_9Nde2FM)hPxC53<~VmUaLKp9W(QR*41g(ojaTwimz07bf2Yd| zBXTI7x}^#jSM>%y#y{STvLNBLZ>Eb^w9$~Mzo0xioMDy46>T5fD`wN8DDREyDK6KZ zDDucZf>pyd**dLUHbIOcf9)lA{Gdp=_DChFZ0n^#Orp?1)+_S!W><;P3guF0?e24> z&SGZm9|JHMdXb)k`&$m5ej2y%FAnKd{ikP>;8?h1)26fSjaj=VpK2G*YG-{;(grHQ z6{yt`x$go$?~6XELLyaYYGLJxUTn6A7kWz}V+Tb}l$lJ3>$*cxg0k@{88e_b-l~Lu%5wzRZV|z) zc>4tfe?(?5Ds4bj&eid(v1+}c&zZx#8lK2(?X;ZVEe&))ZAWCyN3;CbuA@hx(QFle zy~WojOmKGeF<7xJ}*Eq_N&fW{CRM`Y$ucxoe(YzgC|s+@R-zTFkZw~ zWPY}h>9RHPH7ks?v7$N1(fXB7^Ej*+Z{8voK>5q5;tA62Gxgr{utQ(Yx^{+g%6em9^gno-^jXTC5`aR?fRjt(|Z@t?4 z8l@pG+RXd*qurGr;Ih(@yfY3$L4P+Z6H&OEO5Tpm*5Cm_1Ta43>V3>d&Fo*hp)l3G z>7cp4i+E=gyi`^|VC`YMu36BYp6`E6sr>0pFs5?R`FsZGWP)2hnq_`$blb&FtC|nd zS`IVmi(>M;w@B}=8FRpek4YG1hBkOaKAN%g4c)5-{Rhl4J6mgZc>@TeY;@gzHcKCz z^LPC7<>6adM~uqQa?H3nBw#F0qCtT)k-OI5_?_`fqo$w$rF^lA;mnWIulx=u)QV-_ z%TOnlo8d$!3>60CzYcT#^W%`ISD4VO3PhZJ&dnjq7t7&5IIn##6r<69m*`z;b;RUi z!>^O+a@p(`sdOD$1WNLk$v-_EKN8|jN-x3D0f(5@D%0m)IsZwI6KU0dSWJZJG`<5o zgp~5;z$sDz=a*GR9dX_~gN0Ur9h||Apcf#dVeve@dNFA0bn^#}501&?N46;|kHZy- zOufqS5_K0AgL=G;H*Im~js)Kyf@Q|P?3(Po@+2XYCl>qrlJ!`jf#vFQSfK8Z2T|Ys zAp(3#_c+!5k(r2Bo6S1iy7Oj+|L2=DhQbdE7M=wxGh^h@e9;{{JiV&q(zNj}e=-i* zocOVIs~#Z7Hw+h8g$0Ew%ojK8kY-g-enE4}2W3Asswki&y?5LwI)gf~ITazqAC2~RhuFZ(DDbF9>tvp?QzHb(N z1E=G%(S1@WrFs?{isBn0B=EIRHC$vT<1_BiPab`<$+*Lb3L`h}3SES1Z3l`xgr#O% z9Qi5hu%YmV0)yoSYl9MW8*M)fUE`{VC`STS8J+O>?BW9*C9(;mY7Og%6sDv?m))={ z?^_QrArX9e*Pwf^-%Yo!incezw%&8T%UA2VZ<)jIMRvXF9x7{!Ag{chIMHBJ2R1K z*Jo~xWb4it5LLnjyE#~XS0D|xw^M}Yb0!V;OShdY2S-3XQ@Z; zIMgZYhpA`gb=M`fO}^D-E>c*x8tpUegAl-;%_i+Jc0FU^0`uzcnMP(( zG*eSr7N-v?hzpzF!RBc7-WPoGOd+B>@<)=5IKyphPEo}AA3yRbzkUgszfor);x9oX zwqC21%BT5EEsDZ%U(|*ZJK!rOsgWnfq|CWklUiuk0BnEJa=e3Yj&OSG0>Xx2!J zw1o3rf`j!0%@;DaDQLMk4Rgokd%C#%3+Fc7zRSSwM6Z-t>Z0T2H~soG`u05Q2p714 z2}4Kk@^{wIj=uzFW9PnF`^^o`*C6U`gF|mvTJmwEm@*Op~tZ=4!RKNpVy32tqjWc=7ifP zXWo$70vKKm*D1fn3Oe_nDPz3qC!xkaa$7t&T=y`~s5rzdh{I#E3%(@STE?q(SIx^H zB{DeE4U>y9l6eu(l=D5H4b3=6FJ7){UNRIn_7^UdD%={p#oS-P#QnIca;pCcIc^n5;L~=*IVgcRSFSm#y2SiN0e{wjcJv!CvLl9P`fH&LEaP$JN)j z+pA4a7x*6g7*`YatO0e9&_{ebz`+WZqUnE}(Ki#Gpsl|l!ei$Nz($L*7vCvfI;hrw z24lmGgst6BeOj>D0gLmKa`l7n!EY(^7f(yHKlDREFRFTI*82CC$ft|L@V!jVwee%o ziB@A#K%k;tv{X<>YMHuKta13^Nx-C|(&k-wcy)Yrc;|I47#ksVFxG7`mTgMc@x_W! z7rk$$8tZwT<(}{9r1Fko>WK}`tj~)Q914?9rHg_L@eZaOElkH`9=tIjuRoVoq9|^v z&P=%rheManq}C8peWLwD?W&BzosUllr>uYKK8DHS6=d}!hn2R(C3#h-@3V4O89r^T z+(p~rV=npVRDig%O5eWa{4>WQDzMpZU}C1VDX%vum0YDfwDl2v1?J8S+42H!6z|AV z+m@>A+m2CImNw1EM#oM!>ca4k1J6B^lQWSn%!|k zFI2S~G^g%KU=}vCKzzV3)$cu?Aft`E(N_aFqT&rKqzOC6H9XUr>!^+Y#cr#JZcq6i z=xwUEzoJDQ7kb60+BgPySLktVcB3z4Yl~(6bx6w~^)3to93e)px2+E(eU!MriIQyBE2;W2U3`2y*XTmrJDMGi#GD2j&kv9u5>g@` zrS(=*SCVz-@sv1d%Ao5)s%F(5grQ!9=q=p`D9`%Trb`GbQLp1-dS zLm^3e8{IiMBjQ|(=9_6JfB=!OJAd2>PWW+--~ReI9Yd@rW|ELKtnCdkyCA~d z&O}|`GTo*@f}ql4yWP=xAAtLQvvr(mCy$-QJ0md3#2poj&y!9kaU>f%_!d?mKymOCiVr( z#25SvR`?_<1Du{(!kEBjYkQ%2)KCAoYkh+nD>qL_zWYHciBBbJ6|npOcuyl?nT;N$AT?{5^=6K^#R)Oj#P!O#pzph2$5Y1x}) zp$1_f`I$uIUqUqxbVGmwI{Yoyl%YheR6AKKpW5_oB13wnpdnf1TrDDR^M|uKqLWvu!Tt2{X4kts5GE(P}a$vZ_m;>)HVl zaMHUD=e|7Q9sYw3{ckj4cULcxGum~WE683ET^_6g*6*%n#pk%rhk9ouDlX1cI`$ME z2^6>V$)`0EC)VDA;Y;Zs-kR%h{K5r<$$)-Ys(KSRNsy)>&534xTFdZ99DL*!%d*$+ zSGJK1*p_)oeIOIi_{_g6oByuKd}O{*2gW$a)(jZmwmQXzDTL6jT{gPoJ08l=G!D~P zCq60JyB*d@bv(%1*U!B%JvpKOH1_&0+cQbQ7dt!J1{b}fx#8Egvt~01-}_TU+6D!+ zSPh@VmTSi+3W=oBBIU6dJppa9#t#uN?~(%Z0@62WKj65f(e#U}o5khNv-QkM_VY2z z;5~fw@%g@I1!_5zO9jGuvsJ}A7}o@g#4nBMyv}?_vzAjl9Wq&${^2b2J^F70@9vP- zc5>JU>h#Gn2kNZ(GCYlFX8*wj0N}0!peBzAeDz~iV|ari(Cc-&$+`zH6fj(glB62T zc@>{_k7{z>V88Y$RAa!0BAG!x#OLy`5LUC6QFBz#dKWz=5}n7IUFbXUaiSSNU07AH za;^;Pf?A|vfynzMTUsntsJUEn^V9gMOVvTHy&EsKxQgp1V=2`na~34C z2lDc1b)eVY;i1=D2+QS)zsi#t=$<*eQ7LOT&YqE<=P~oMq*7Ei0^;|u!KA1H=@~D( z=_FH_0XI8U0s}sxBj){7i{rq&svd z{dj9Yy+4aX;)?l|*(rXuFl|xa-)osRW|mWY<&3n#bdg3Q;wRIWJKNj8FyFKSbZ}v@kHmD~U7mgY5@TvYmwb!V`_lXdxgQEeHM-0ptY9PBY};s!0Nd zI+81r;m{nwS=V7b|KV3w>~u3dhXlA1gaJc2I4I+fJD)?EbuClF?%u;WEuuF>F(Rc6 zWHG86TzCjRaPCdp)lcsu+D<>4qi&5vnY2#n*Zw2dIIrm--ylWM3yu#+0I6wP!{GT+ zbv{?NqVkg*&xr!DprM`rB?Nee;DTq1XIfK%4+sL7@_$K_j9ml~JqbGPA^DuQ+Z`t3 zE}gWUBZi%!F_g~`5rP3@U$_7r@C}4(^fp6pHHJR8hS8Q)3esb*T+scl7I;ZMEaDyg zZ1Ep@8CK8d)6B>~M>sEs@2QGLgo7DpXWk`Zk@Cuk#U|JX;Nl|_W@G_`36eWe8B3!@%V>^ zydNzlIRvNfd@TXH@!!tF^dqUR3+veY(b%zCfF^SKv5G4P#b1+aghRMCAZ`o(`-xtaUQ)lT^SKHNp& z;&=J9;PYawMthfD{soju{Voucc#i+xx?KOo)$hms_3(@zQNDiqS45#Vy3*h;xO2vN zFUQm-R($0SrSU>rqL59v5N|j<&e46{d!5moMEGAa7`a8*lpB5D9G1>W8I8g4*|0{Lv2v%wR{6j*C+dKjM<8=I6&Lf~{TUX%p*@&nO*@a;I5>D@>A zyRI(%3L$_P9&}Om@(Dt$2SIxmp^`FlyJBoPFx!6$4c@;qPR;@&8{-vC4iGLHxRV_6jbUhd!*_wMJeR?ar zTJXAk3jirk^7nE(oT#J*+k?nbSEaWumJ=A>s_iZR#oys85AHqB{;0kg+b9-CfbX9)Te>A716adh?38!otB z?M16*@C2Z!mB`25)RTQQ&ArsTml`5A8H|Z+bidC!S*+SXmCc|rWytCC<#^&kJo+OJ ztA*Zzl+R8NbwIYkVPW)VBbb7tmsGlrPHal~Ym)&YH4|<21B&D`r z(=TREUJojU2)4&6CjBKygwA_2rKmuY|2~fTTUXGZH!7ch{Y@x*O~ml#f+UO2CbA;) z>qoy?-mCq-!WUhPl8(2dmEghV2{trajPT7h-I8NcOyEGnDpXYty_4HA}xCih+B>KD33a;_CQT?A$JXVRw+8hnyV zChEEwcxz~~_S1_*)PjY#`r+9V%=4@nwe>H`-yZa&FB&XbsxPv~_iMWXs0jk}At~Gx zLwNF$qJIskv4iRjaLnNDBezo~q)32XnkJhhgrdrD%_xY6#&cp#++c-J#1XTY=hf=F z!I|_-u`DiPP<3i~^-#Qnwk`FR6o2zqz*FOxiyXq$vY3f35$*NkUCl*;f@C_f)!$vF0SuB;j@%T~Ew(}n-Kg{Z^vo)RG% z$R(%gXedp>l7AwInGhplzgJP$uZMCmdr+a0e-zL=5F%p9q81j}iQP$E#in)|(qdJW zk;i#BLOmfEe9x3W*120vq?w;D==S6ozguQgL6^X5>1@M&z5gKwE|sQish9YAsb@0Q zPR-$bHgyzJM4`XG^p$=vwKVW@mrwJ2)~_Cgl*?Sa`u*~;O~LGVyXG9n^qcgTYaL&$ z68;Lv{SaZ{+ESDlL!gHB+21DcDmEYCgw%OaDgzo>wE9 zV**YY&HJeGYz_TbjDpVVoMcWXGeUi3P*zenyY8sZiBZ*;S2i2@&?0l#rrF6e%Jk3m zydkbJzmV6cRL9HYG*|21JIf4E?Rd*tXFHF_Q$8CdO?9}^7K|8rT2@$X9FtWM@o|^G zJfW}pn}4W9IT7^7XQoDfnBbrWQ^A4wOWh{4^4ZGnGq%lNwdU~!wukWvKrH9W>+4%I z_asY~RAJVkyEO^;+p-Z)rPMbB-$;OD|A01lov_>;xj&HRvhCk)9Q++Bp zv6bo_s6Nu)1hs#f*q;E2oRo+7_k_YG6qTaM;D*nvGl&=>lRv{?3W0xO(iVi z(a5X3KHy^@6TIm*^@^-#_VzscS671}jf_?HH^&MfFTVC=V?1Hf9r$dqTJ1B@?ouw> zXApit`CiX)wq8gH?%%RA`eEK}A&%O%PRJxspkTqq1bKL`^>Iwe<7Fc?d!7ihF*#C- z@8%TnT4qYC-obL;TQl4?LRRZ|uj%W~!^b7{7@dQamT1wxf^kK1$+TVgq`p*|;*yh} zl*;3>ng@eXagN3Y@EiSZdB0fskO5_MMJ%1$U*S_W;5Ac#<}ju0eYGUk1+X4X*Q}s9 zt+gB??p9Rxd)}Q{psCZ?u6xjdeSSW>ND8ug&!WELUHdtaKzKbkZ@UnGOL*`z`9b0p zC5YgM8Hx9Q@6sCoa$uzzGjl5Whpq~4fzk&1O@RLIqB9TVies=p=&&BbVm4JdT4jABRI#Nj3ul(!cD2IQ8N(zn&{J*TE{wdReob;FX zfhxGRJP$T&JuvcS7aurW{BLDC;j0}Qc;O9Y|NmOu7x3VV1FJ9+;B`neeC<~b?~qAw zCRMKM>0s;HSFA3exfuFz)>w_$L+&`pnl7HiBGdIxKwC5h5@Igb! z^-|Tx!hHp~cQV9*4P+L{+Aa8^aK?GVfIb8+M*$KP5`|=-_yybc{}hFKT(rO^4D~Bq zehcsVCG6oy!Cnju_m{7efo$szE+Ewfjbz61Gv$&JUkuKpIjzi*P{YFsm@JAm=J`y4 zYuj4J0G`bHSTQOmx4i7X39R53_FNJ{{e3d0iFi+(Alg4sh)Em&n2KS}t`H;3J-GAV zJ!1gI_gXCYpHV8c!-k1NxM(S` zBs)Ejul#mjg=y)T2vtZA>Lk_&JJ=+^De^%0GhlVVaRU=&1w^jAuf~Lv4 zSLLXSKTJI3ItH#b;Y0*w{)0~8b1-TEU23(_$I)yzxZQA`DL-l;J;WPAVttowel7K3 zhY1qUH~wB5%j`;#g_7XaB*gsqgx1&ouvlM)_cR+QY-7%0-}llp&-30+$Rm>gfpN?t z(Xi6J>QabUIhJxPh`BFA)>WCpjRj2pS0Ek6$}L$RTLySJZ)|VbB3B>_wAfpg#>|dySxfgYQWRPHp#v><^PS;@M=O{H;oyfqhuYVPlJU?=A z2hxnxh+x7Ls10ew_@xTDhqOXX*eN=F3-hNTbx21~0yiKFkVXpPBcgxhOXZR%vv^*QJrR9Vz0$$jZ#>1)%A7qpENY zN^G2dyeEM?!s5)v>STe4M}*!PAvvMF)`##Q$hvEODT*hR)Q#BJg?#j&{hRmW>MO2{ z+%pEA{@Sxs;=&jkat(FdGh1Bi07(s}s*Wrya7G~_p|2#* z+eQ~ysH|foN+BWU`<@K+bUfUY1M`PUlhfN7El*+IQBP0IOk=Xf=wx$@4KRqJ3dNMy zQuLonwade8ijRSoCAmJK&AmW*{K^!UsfnC4QD~4uQy}R@W_yH`Y>_#4KKp~}f;}pY z5Tn`gr?v#W0Vg5|&HW%nsL&>NNMLuo2~9UJ!jVUm`x31dMMffo@r#;oB9XrBU6Ce# zuuwI@cLmPRNv6}fcV*zYK=%XJewVuJ%xy(G*Wu{v&A_KDmYo|E_ydL5<0Dm*dI1#^ z$G@=80Z0^%@=X5Q(g=CeSH?b-r}}&={k$R$<_ioimu+Ho>s9$n7%a%YtYGbClzc#d zDLaU#Mq8w2#M{4=$_N)A$tW*AM|N6j3}Ce1_`LJ9tAqLUs?OPw3viil)81lzVfp*= zl4XP69tEO}pn*cuS6dt*USUqZ@wsg2Q7UGM!QWlmyQZqmqgzFL&AjA)KZLKwmRWry z%Roa8+om-U(E@OKJfwFf=YKi?J#NVb^1K&4M}fl0z9M6QXHUb*p_fx4C+%Ip|JlfW^gBcb>33w#$I7d z5Bz>CkOciCJ{lr9z@grB$S4(&{<#wUdlEG7MB+#Xrc=&X*1SU5G}wReSeILZN=mYC z@?seifqK9J-L%!wTrxlAy%0Kr2;>(Oc24&PD~PereZwnMXE0Gi;NgXor2l$_ORe z`-`n!QyFOh0Y6N~4ykum?$28VoqRN8Bsjjr5d{~Ly#Wik6ix-k1-Jk)V)4`5U-#Lw$CUWL#l_g1}=I~#yrmYovAxiboA zKr}SbGYqi#|E=;MAoa&PP1FwSZ=u-8+eGP*?rU}FD67mmv^<)wgLVSEs*m540yNOR z=ny#D8v%~*|9e}VtdF_^R*qHgQ=97H(dVFA<@UNUW82cFe~d0RD_-h#-~4DlOa z&+q|YWVHO^J01X^44fcU1_PHE{u<)? zPPC^Tj!5)M&KI02Q(7=zTrCx~jwi}+bBbZO@ztr0mqGIS1hn7Rp>w!!3^$7d{F00D zPkGmUyhw%LaX0^UP+sr?keG^nKdNQUNfANZP5|yUZn88%b>#mPP&YDwCjZ~o`2UaU z4zo7k=$BZkR5X#B=T@$VG;p(WlwucBM-xRN)^7U_(nhT3q9WDdYDfHJi83tmdEV>0 zt7CiLq)raFu38J!Hi%rg$QVFYiSmYF+65H430>E}z6Xav*3(Dq@bX}dclxcK$Ck(^ zUqhW%;Sl8&nEvhehqxlL<8`KfF8CTPv;qu~V14m@y|LuZYd;EM4Ll$MTxS4Uo&*F<^83GAG|J6cbJvQ+01VIr9s<&Q^^FbIOM~?%=@Z^fX;9M4#!lce_){U0x z2G~f|H!h`W#p4yaT>1I3oLS(&mN{5zN(8-e{$5i*EQ-)HTpXK%Kaj2C?#Zg}DVF7E zPNdyD-NVIJGV?UJUJp#xI+MP59wK)(YXzY86)RX50nt!X$_%#bW1aF5>5Lf4s-?#K zcbjqL8MQEU@H6#DYC^wFXi|Y#Vqb!goG)!$AtdHmWS!uPI$~t_;O3HMFkYALjA2VU zG$!6PQMRQY*7+XEd1s0#hVm6}>*cEdE$d6;_6p80U|#ke*kIOromvWA%t1dJw(!cQ zzr>;qW!0;DJzb{B5%Azk@IphD(!+l8Q|uR)y*Um{Dj^byaBL~y>^RZl;aH@UtC!fi zF_aEF?xWVWFW0WF+Nm~dAp+*9-Ld3+DXYM`kWLJ7Z2~9ye>Ksh;H?2BntI_DENTBX z9=f9c@b>dwIhgDF%?XZxq9>Oqk@4Q^=-2x@7S=v>7(=Wo^;55;D{CDF0lA_sC`GXk z=nN`6AL%*n4~FC*B`iQ-4bhS_pH-o^r(>d`CF&(g@63LDq08)hxW6ql`5v^>X!9QN zQ3Ab8+>uW_{xb!YZqg~vu&CS=C1`o22W^f>_ z2K;RsG(#uNdFAo)VsAd!8P>~0=e03Nh)Kc@!60Jk%nx{o9L;@I=y!Y3pT$#7mz(-R z^M}vjKtV#|z<+sKGCudOWM+t;F(r+DnncLTp6&j_Gb3#9CTgifFTS*k6(oXE1kPtr zM}fQ-GHOpJ8~4I(9k^nhg@dCdwOCU$;u_?iy%Pv*KmLtBRxt;GPz}gQifQyD7gn;2 ziU%48jYi3^Ur3`Br9?>noE4>pBs@pg@ko05=E6SxrQwObB2~{Ac(=pxw6?M7<-s!W zsucWUKJf8NXw$n5abQ+AB`iT3sK%hqL@GPLj_$QLQ%OFU%3KmrJVnZDM`Gl6uE}jZ z4moVUtr}DVa;6UddzD6~H77O3KINxS#>^qGFT$*_wX2OP`!{n0Th9^o3_^6(ju4d_g>UD-;$$M1Y_p}H@%wk?zmLbPnG@f@Wr7Iy44U3Jitr`sZg4N z>Vj^^{LU9U)XnMGU&lj(4;5vjuO37hX(ke72)(8}k{scaZJ{h@Bp1Lqn465PX+4RZD)Dc8OdoS~D3YM09u9b3JvDRlfoL_FB33EvL^;gamu$b*-F1C#@hI91*CDC|Gh+lsi;g)R2{K z&(RyxFsm-1TxeTZdoTEkosJa0Nr+A8g%D0@<;}7$>2B<)ewG^UE$#!t2L$)*4Ef0>@?hn)c#kd6mC-Ab> z@dRH@sS{){lh+*3EV!*R=`pJ|iNK#>XCh@u=4%^Q>;W(6grDbVw8zxXUx%8qP$D zni%hF`SU6J=%D5BcNngM^onwNHXUr@>4?&8(n1;Wa4qVDMCaV+-`Z0GJR;>~(Qsw!Dhl*HP8mtib!Sx5Kf8`sb?ryQK*`K8}_i>uOefn$pUyU+c|KtQ->AmVj|6M#e~g z`u*!twP3CPEBbw6|sFnl{{=NiO1U( z&(8$*AGz#_6*Z8NYu-f(mh@U_)@FgH)Eh<)4MXsJ`=gEgXmU2WhtJ)TyTMq-T=J#P z1ubJ|a#0L_)$>i`<$Oh@qzI)a@050SRDL5v_muwmq?0I4s$?$Py3m+D>Bqe`Z1vMR z85*UZkzJKocSgN3zfEbinKf@Sz)(%8E0NrZ9bW`-usO6>E?sC?GyTw>6SQi z?HHf+b3MWU5Kvb`M?sMsarRrZuDd#8}|4 z?15k`lNkB?M(wjwc{&vZL{2KMB|i8Ur5}G z$+$mDgKlHRsC$N{Xv?*{uu?J(zx=k3@W^pf^c&%9?0&3mA62t*vJ~@z)3VU!NtLnP z5^RSCEw0j7pGWdT4Y;eUwV46WVce=1nZHh=RH`=RBSjs5=e*IjJw;uW+%W6Rm*woI zPr;YPAsnH^@{_qZN41GBSZH1ldp)NwtZO^&=kHJaRvpR3u)Uune9S(xU!Fo)o6uCo zP_L=YFs(fKxZm@v<56?NPH0mMAzKC%=i_}uY?;4QuKF%lnI{&=c>V|^P*1Eg1Ttr5 zH5>N7U>~|`TJvL5PC=_n618=zw(wm4nb{rL*z>q;XHTE?Yn z4-UdT-dUG!p(hOuYyxTBT>^aL4}<(kBlbQjNt^Oo)#>WbE;fGP-P^SFnyYLO=x{}< zt%rZ8v_a0#iSx)KE7~mZT;t1b0t)}G9A%e7hK7x zW=^X}sfQfcEFNyIjLv1zxh%ZY<#uKblE}1C$j6h zQ`QE4NSWkNBOAku?=_9rlwNKr+Oov0M+hs9i@_2;Zb-2`7tRT1W^tLd*x^)DSoqSE zL9|FdDo2(b8&4yBa(LURp^Sfp(Sj!01vEW{{(VA_o}?XT&@L%8ryXzSH%Sw`=foQq zs8?vo9MA*L^zn?6AW!cZLQP~Zb?kTGTaX(p;Yb%dVx?7Q`lB@5WTba_uGt>vS)9$| zP}l89^HJHTjqwO`o4B~vIEwf`f{a!8B?|oYkCMj@gK5x6$jj*vEf{EE99@Q~;9Ctq zEy~IB7q49J}3GJm9PPik!&zGhX zPGI4tvVCLKOTXJq8PNaXz`D&JlAVKx{M{`6Gj|Hw6&*NQFy)5+$7r;MM=GwX<-uLiFA(~J-{MK5<%2Hc!F zki?E%azXrBte#UW}!cT3(lL8r;D>@oCIzwXvty{)T6xI_g?g zH$&Ae4H=rD<=81Lzk_NH%j1uP_9cBI^(2uSDwL^4y~WjzT`8xxVZFcWLbJPU%kBhV z6!bV2b$OAO_MPRn9%c`k_hL9^_K`6Y{I(@Ocl%*J4>V{&7_cer+_1;n@g`kp(_%(w z@EwBBN!7+ayHy8Ivxy5fb{cN6GGMpYvN-dp>7O?>fjm1&>~J6fV^u0}mYvADK%s?E zNvvhpwn<1N^;*~q&m6MM(g&nVI;S?#Rg*9JxZooD!G?9+$ew+Wkn2LjcLS>{*_fmW z;wC0NdV!P5P^Drqd`fGVPA!z~2?(BumQBA=@Li!xBwU!CZ(ZhG#W){cU^ zcDPBrgn86d4O`w4SLI$hT)iUCZp<$b*tiq%Sv*ZA*eNhkUJ@6hBG>j(!$lr~HK?~u z!P5(wVObdGjJj#`Y(y^I} zmn&tAPf>VMXjui!vUiA0_tk**pur)L9@-RWG~$_R)U_lxvrAiZhN2Eg=Kv3@)$+UHS$Y^dlig{ zKb^?xZBJ8he!gv`L3^pZpmi=qvjxA;Q>>q7k~HD)AFCB7BwnN57PyolMG*cG2SjMz z1(MU=y^?Pf_&s#bDAV$!qGnw!grunkoxje2d9K8TsxBqp?mjy&r(&kB5w!xz^cO+2 zAn0VBcD5w^6VcLNLHC}K4N`*+hXi9p7T36)C)Wu4@Qx5#G# z%iL#_*evzaD3=mjQz!^ps_dHTu>s4WiE|rAikzAKWhyrdgeOy5_31|`&M}Ky= zRg&@yv(nV`6(tnyB|j{H3)S1dXItKJQ(|3CvuvUPHIy9R&LW9~$J)+}1@+k{;+n*t zsewwj{*&nIb%%YM-b+uA#rhn^q=||e_8)`CaA+&u+ibi-ricu`G8!vzSdYwyt@{Xq zu&fvD%_xO9c&z!I@mR%manqi;BW4gFe`m>u+Opssy?)(Wh}usk4*yS12AXhx7X}U+R?#g0FD4K6B<6`QK8+BB7w)3obx`Ox3J#RJ$j_fuh zGGpW@8BjQ0FK`JXyVg+)mIxy1o;ZLyz&UuoKClPm8lglKhDj@KHO3EIjfV)SIw zCad8BCq88x?>|Xy37+IGAIL!Qqtj|FrSE8Ds;EzL%kQdk$$m+XCG+W|(S4;Q!_x+X zZex00*->uMFXR6z>dM2RT-*5jj=?AzOJp75NTuu}N*enT5hClTDa&XODkMe=#n_Il zF(gZtN!iNUNXC*aYqreEc0?Jwv3~D-=Um@6*YoFGzxRIc<#+$?=X$T_5xN0s@8QO` zQV83E-^M(N{5_J5xu0#YEO+eiWtT8FK&L4m;~1MrR_ZtCLk~ifXE>V>Dn+*Oq-tKw z{LXCE^{vS{6)d+%zl1xP!KGFHqlqEEZ-HB_c~>u{jv}}ip8o)G7o@tgsd!@6+wF?Yo;0zepxA^wo zbilxuFG3`-^V>Yp{TMWacoV-+ZyXp^iY-gf4bK=fVf*$3?k5TsRvAb$03s70pYF{? z+p+4MH@u!zMPH+JX8TE0p*qJxiu!G~SJWx=i@TW`-ZNVf>;ai;D>)Fe4W?*KVA?EFR!?2gB*y(k(&&Y zdxLiYnz^Y|0pC;a5f-T#tx3Zm9DCkWIQP-Apk${9@{-jWC zQdi--%U53J^xLkKeW9h!gB@n_*@<^0_*+dbDwN-a^rYVCwU#UnhAe>HI9FmQ0P-C_ z4hD8(`xNOzw?&?RuT;hc{)t$0@^c17b zHHM^P0j;F&IZs>YEuiJh3n_?#l*q0^GG;bJ_`rL^NiPi<^_7yH`*s5=BI)rBR;%wj*KY+_$w z<}B4vK{vgeF3sAtd5Yi+RTsyg_MKYgt2*dCGEHjAZs5G$&U~OW^Vnr!C07cOd~V)J zqFPFcovL<3q7eDt!~qN7C>X@=lTT!*r zxQ=Ncm8wLWK@`I&(lLEx;!Hy564zPU0+)X#@dW!Em)JkUJq zt48!`3k>+A;wfjxCd^G2GZ8ClfPiHYAwtyK>mXKEsKg)&a~_K4po~q;#0H#kUNr6` zo0A+Wh2&Ym_-8F47#Uk=k+>z-!ej`O$<+Zey452c znRdm3kh4F_Qp@=#f7&dY(7k~baOYH3UtfuSwiA3a+}OhC3}Erx%hMqS>Qsn7q>|*$03SZ1Y;<3}r3~bCjzmaEG5L0E}6-3VQTPTcTiL z9P}VSG1=Y3;Ta3(a5}*5cl^5HHJzjo{%l<)2T;DOdA`#WZmLID1rFseX>d>lfrhK; zd7ejEgHzpI==OnW@mt^bl)FzNqw6^G{#$jbK;0pF=*!mA;TA9rfU4+!PS4y7-mXK; ziuB4!H98rrT@)ZO{mq_|6A$Nrxps^%9`ReBPFJ&@s)M<@RVU{SK(_q5FnHJs5aRey zvyfT%5}_fB;o6)8wSHCD-ARt1N?sip_yXLr=nEhD=kI}XR5jRE$9k+Yo9$!jOW@c_ z7O5p_;Y0EXm~{M8z`y3SNxv?u9klDy<0!!e`hHRGimeycuIJU9q=kAWvmfIXiQ^o` zOq$fxIMB!Y`=d>X??h{(&jximvMSHikS;DC?=MS4UaYDrQ;4K#H)}Om4YK2%rQ?sl zD#=Esmp^jReU?=;JN4ivb%_tJM4IHvrqrJQ>LTm%;>)n76%_e5T;edg4zLf$xSSda zD4+KMh>Osf0(N%gk4nh$7a?}H12^vjqxcvI4(pm^Cx21y)f4N+C7#g@AGW}@(0xb1 zwx3ijNW~34{l-lv07x)3{%c)7K^w8dMr>AZyRGe(Yp)OY27RCJ_8b<3I2q5YAwn&~ zpQ^Me)`C&DtSGG$*8?uCwbVS7QKrcJN}1PSt}DkGIWFS(jwH`&PH@vp7vRjb`StrV zLp#--^Bdem0WgD|uK7X%L&K5p4Rnr0fFDdWKaWv6ULTn`wEZM^OEst1`yYZzH-~zM z+aji+rJ()i^_kW>NnVP)i})!^8wCGk4)3YX(-9N@%5MDn9J z7(Bp(TceVoh9K1mm^l6W?v(e8jl)fpGL*%ZW_axlgK~Z_1@6tsKoN%(&RxNN=jwSx zQypCSZlthIght2v`(@w;I)!rql{>d*$U=WJabN1EA#`}A+29E06n@FL`de4 zug_SBe)mQ^>}9=nTOd-jCJbMnTX(jWR#lLo#|L&{4d6wF^fTZNc0h=-7KLf!vb=Fq zgXjYbq)d68-d^4WGTg;x6z!fFL0%2au=&FOcWdB{*m6Z+Q}42$E^){vNP;NowyLgB zrZXO9h0_{5q(4bA9R1=Ph+;neyKiu`bMUTX8B3cyF=*XZ z;HL%jwT2A1v`jj?efM-b-<9?=OU(?jF-Z*-=J9d%tc|D~Mkd3>Dd{?|YwW9Y;7+^t ztrN5o^hGGIg5iNp5kEBctxI!4u?X14X!$|*0Gpoz(1@e`s%UW2#waT~b2=zDWx;v- z@S7s4Q^_{Qt7l0Rzx~N8XO*EclK^e|!OjdI=9L(=yExR)dvQ+l zf+77;4|D!eS@n(A9sFR7%WNh7l#w) zZreoZ5f|-)HhSy+0!JvfI}? zQzWiV^-`rW?Nec{oDxkmfC6mBJ$V@AW zfzj`QI|cQAdPwQ>Egx}$0rrtJ8QIngZ`l|d6Xg?O1Hni4VN}YXb$j*FHv3??Nw*BL zGeozzR~VehscE8B8=kC-S$jRG&BdrUvDi)2<{Y>RaYPq_K{cZgnZFKqXYypn>Ss%_ zI)1Am`da$EVc*)_u3gAl^Hki4$f=~yijGaOLnihfJxlk4SMorKIkT%Y{X8fy*-qO-DgF6_ z*J%iwaKPY=u;QoHvtiCp*u7$;Eq!7AY#KsTHg;@eJ~j*J%UX&MlI&S=OOC>__VuTK z5h^rjGWE8r-g8`;?+ITqva*;(hM$;^L$9o@pZ(<@W3v31qNzQvuS_|m{-~zuu>ACw zUa+ANF|3(R@_Km9yk_AAzE*jVrm5nkmx6TsAsk0~hciPpU(FG&u2XPW;SaClr}tR| zKi;o9_FJvgf1s|HFI#G0yXwUG4Y9sEuULXA7NN8zj0RF8^uEg(Sswhx!_F=<{tR{r zYb^x&z<;wdl4H0Q19~7!#pb1^2oq|STwIKw+(gi-nkk40-ZoF+@D?zma)KK?6%sJFQCjr)fDoc!;K|miM_{^0eX)xas7faA?wdM;i;&;^;nEmzWeY1&F zE*hn2NbpuYHg;11!)kSL%U*kmL`g&hB0!g@d*j8gMK{2;9T^%C%1Mn|=vIVN9Z_{# zrqktr&vs0U3hwU6@c*#XK18z9f)1H|VLO?ZzH#y1wr{NG=hqyi(lAdhao6UB!!O9m z*&~7pK%WwsAbEYI$}p$*>C)A6yh(pBR~Ps6Ulo@jo<~yc(UIHBC8qLACPcCT?XO#J zAN4-!oDZynot}IhLQ6owLMjR}Z)e~0;3Z;*R}JSMG>D*WZz7a3h*5PN#~GCN5{H9= z;T(t$DkzJ}p?6!b4+2z0vDC!$C`P&Qe*L25q~!Cl;HZ& z@mE)#WZy|i2I}V%0TSSgBxb4ASlQ8VYRDO@;Ksdzl$fQ<7{6Dwx$8kPmiNtHEPa#* zw8-yG@Y+yl{#k_xCxXP$?tDDOP&!@`HUY=Xe*Xvy>}{@a5{s+!&v|mOv;zf%taRQ{ zzP5H0W-d(~QZ3E5N{jNLwxp{edHyGE<0U{;MNnv7Rf{)ZXVP=(+_#^@B4Dua3F3+f zAm*jd#G{2WxRdjigv}Yyq0^hX+mE6_M!LMf`XFsd$v3g>#4G+2`y}u8R#1GTe#G-c zS}&k&g1fLF$c}GW6Gr!8X2UB+W2?rV?t@ zV#|6kPz$ Date: Thu, 11 Nov 2021 00:57:19 -0800 Subject: [PATCH 031/163] Minor README update (#73) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e77aed12..8374c01a 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ metadata: *See the `samples` directory for a set of example yaml files to set up a service and export it. To apply the sample files run* ```sh -kubectl create namespace demo +kubectl create namespace example kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-deployment.yaml kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-service.yaml kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-serviceexport.yaml From c0a67bcb94c26d3d015e3211ed52732163cd827e Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 11 Nov 2021 14:52:38 -0800 Subject: [PATCH 032/163] Add integration test workflow (#71) --- .github/workflows/integration-test.yaml | 35 +++++++++++++++++++++++++ Makefile | 19 +++++++++++--- README.md | 9 ++++++- integration/scripts/cleanup-cloudmap.sh | 2 +- integration/scripts/cleanup-kind.sh | 2 +- integration/scripts/common.sh | 4 +-- integration/scripts/poll-endpoints.sh | 4 +-- integration/scripts/run-tests.sh | 3 +-- 8 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/integration-test.yaml diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml new file mode 100644 index 00000000..9000bbe2 --- /dev/null +++ b/.github/workflows/integration-test.yaml @@ -0,0 +1,35 @@ +name: integration +on: + push: + branches: + - main +jobs: + integration-test: + name: Run Integration Test + runs-on: ubuntu-latest + environment: Integration Test + permissions: + id-token: write + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-session-name: IntegrationTestSession + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + - name: Set up env + run: source ~/.bashrc + - name: Start clean + run: make integration-cleanup + - name: Set up cluster + run: make integration-setup + - name: Run tests + run: make integration-run + - name: Clean up clusters + run: make integration-cleanup diff --git a/Makefile b/Makefile index fdf15b6f..403440cb 100644 --- a/Makefile +++ b/Makefile @@ -55,18 +55,26 @@ vet: ## Run go vet against code. go vet ./... ENVTEST_ASSETS_DIR=$(shell pwd)/testbin -test: manifests generate generate-mocks fmt vet ## Run tests. +test: manifests generate generate-mocks fmt vet test-setup ## Run tests. + source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -covermode=atomic + +test-setup: # setup test environment mkdir -p ${ENVTEST_ASSETS_DIR} test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh - source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -covermode=atomic + source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR) + +integration-suite: ## Provision and run integration tests with cleanup + make integration-setup && \ + make integration-run && \ + make integration-cleanup -integration-setup: ## Setup the integration test using kind clusters +integration-setup: build kind test-setup ## Setup the integration test using kind clusters @./integration/scripts/setup-kind.sh integration-run: ## Run the integration test controller @./integration/scripts/run-tests.sh -integration-cleanup: ## Cleanup integration test resources in Cloud Map and local kind cluster +integration-cleanup: kind ## Cleanup integration test resources in Cloud Map and local kind cluster @./integration/scripts/cleanup-cloudmap.sh @./integration/scripts/cleanup-kind.sh @@ -126,6 +134,9 @@ MOCKGEN = $(shell pwd)/bin/mockgen mockgen: ## Download mockgen $(call go-get-tool,$(MOCKGEN),github.com/golang/mock/mockgen@v1.6.0) +KIND = $(shell pwd)/bin/kind +kind: ## Download kind + $(call go-get-tool,$(KIND),sigs.k8s.io/kind@v0.11.1) # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/README.md b/README.md index 8374c01a..5234c46d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![CodeQL](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/codeql-analysis.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/codeql-analysis.yml) [![Build status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/build.yml) [![Deploy status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/deploy.yml) +[![Integration status](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/integration-test.yml/badge.svg?branch=main)](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/actions/workflows/integration-test.yml) [![codecov](https://codecov.io/gh/aws/aws-cloud-map-mcs-controller-for-k8s/branch/main/graph/badge.svg)](https://codecov.io/gh/aws/aws-cloud-map-mcs-controller-for-k8s) [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg?color=success)](http://www.apache.org/licenses/LICENSE-2.0) @@ -60,7 +61,7 @@ kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-control ### Import services -In your other cluster, the controller will automatically sync services registered in AWS CloudMap by applying the appropriate `ServiceImport`. To list them all, run +In your other cluster, the controller will automatically sync services registered in AWS Cloud Map by applying the appropriate `ServiceImport`. To list them all, run ```sh kubectl get ServiceImport -A ``` @@ -93,6 +94,12 @@ To install from `latest` tag run kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" ``` +## Integration testing +The end-to-end integration test suite can be run locally to validate controller core functionality. This will provision a local Kind cluster and build and run the AWS Cloud Map MCS Controller for K8s. The test will verify service endpoints sync with AWS Cloud Map. If successful, the suite will then de-provision the local test cluster and delete AWS Cloud Map namespace `aws-cloud-map-mcs-e2e` along with test service and service instance resources. +```sh +make integration-suite +``` + ## Contributing `aws-cloud-map-mcs-controller-for-k8s` is an open source project. See [CONTRIBUTING](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/CONTRIBUTING.md) for details. diff --git a/integration/scripts/cleanup-cloudmap.sh b/integration/scripts/cleanup-cloudmap.sh index 00d25575..bb3d4173 100755 --- a/integration/scripts/cleanup-cloudmap.sh +++ b/integration/scripts/cleanup-cloudmap.sh @@ -5,4 +5,4 @@ set -eo pipefail source ./integration/scripts/common.sh -go run ./integration/janitor/runner/main.go "$NAMESPACE" \ No newline at end of file +go run ./integration/janitor/runner/main.go "$NAMESPACE" diff --git a/integration/scripts/cleanup-kind.sh b/integration/scripts/cleanup-kind.sh index d1a9d446..4166a580 100755 --- a/integration/scripts/cleanup-kind.sh +++ b/integration/scripts/cleanup-kind.sh @@ -5,4 +5,4 @@ set -eo pipefail source ./integration/scripts/common.sh -$KIND_BIN delete cluster --name "$KIND_SHORT" \ No newline at end of file +$KIND_BIN delete cluster --name "$KIND_SHORT" diff --git a/integration/scripts/common.sh b/integration/scripts/common.sh index f3cfbab6..211da54a 100755 --- a/integration/scripts/common.sh +++ b/integration/scripts/common.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -KIND_BIN='kind' +KIND_BIN='./bin/kind' KUBECTL_BIN='./testbin/bin/kubectl' LOGS='./integration/testlog' CONFIGS='./integration/configs' @@ -11,4 +11,4 @@ ENDPT_PORT=80 KIND_SHORT='cloud-map-e2e' CLUSTER='kind-cloud-map-e2e' IMAGE='kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729' -EXPECTED_ENDPOINT_COUNT=5 \ No newline at end of file +EXPECTED_ENDPOINT_COUNT=5 diff --git a/integration/scripts/poll-endpoints.sh b/integration/scripts/poll-endpoints.sh index 4fa8338e..1fd4db2a 100755 --- a/integration/scripts/poll-endpoints.sh +++ b/integration/scripts/poll-endpoints.sh @@ -21,10 +21,10 @@ do continue fi - endpts=$(echo "$addresses" | tr -s " " "$addresses" | cut -f 3 -d " ") + endpts=$(echo "$addresses" | tr -s " " | cut -f 3 -d " ") endpt_count=$(echo "$endpts" | tr ',' '\n' | wc -l | xargs) done echo "$endpts" -exit 0 \ No newline at end of file +exit 0 diff --git a/integration/scripts/run-tests.sh b/integration/scripts/run-tests.sh index 7cd432a1..4e016a2f 100755 --- a/integration/scripts/run-tests.sh +++ b/integration/scripts/run-tests.sh @@ -13,10 +13,9 @@ $KUBECTL_BIN apply -f "$CONFIGS/e2e-export.yaml" endpts=$(./integration/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") mkdir -p "$LOGS" -make ./bin/manager &> "$LOGS/ctl.log" & CTL_PID=$! go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT "$endpts" -kill $CTL_PID \ No newline at end of file +kill $CTL_PID From 45da1e9ea4a8a5b0e9e413da4d46e1925399e7aa Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 11 Nov 2021 17:27:55 -0800 Subject: [PATCH 033/163] Rename workflow extension (#74) --- .github/workflows/{integration-test.yaml => integration-test.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{integration-test.yaml => integration-test.yml} (100%) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yml similarity index 100% rename from .github/workflows/integration-test.yaml rename to .github/workflows/integration-test.yml From 1c097de2d04ef5d5741e1361d3dba99e0cd99db2 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Fri, 12 Nov 2021 17:30:12 -0800 Subject: [PATCH 034/163] Extend integration test to support service import (#76) --- integration/scripts/ensure-jq.sh | 13 +++++++++ integration/scripts/poll-endpoints.sh | 9 +++---- integration/scripts/run-tests.sh | 15 ++++++++--- integration/scripts/setup-kind.sh | 2 ++ integration/scripts/test-import.sh | 39 +++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 8 deletions(-) create mode 100755 integration/scripts/ensure-jq.sh create mode 100755 integration/scripts/test-import.sh diff --git a/integration/scripts/ensure-jq.sh b/integration/scripts/ensure-jq.sh new file mode 100755 index 00000000..8d354068 --- /dev/null +++ b/integration/scripts/ensure-jq.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Ensure jq is available to parse json output. Installs jq on debian/ubuntu + +if ! which -s jq ; then + echo "jq not found, attempting to install" + if ! sudo apt-get install -y jq ; then + echo "failed to install jq, ensure it is available before running tests" + exit 1 + fi +fi + +exit 0 diff --git a/integration/scripts/poll-endpoints.sh b/integration/scripts/poll-endpoints.sh index 1fd4db2a..5b47af76 100755 --- a/integration/scripts/poll-endpoints.sh +++ b/integration/scripts/poll-endpoints.sh @@ -15,16 +15,15 @@ do fi sleep 2s - if ! addresses=$($KUBECTL_BIN describe endpoints --namespace "$NAMESPACE" | grep " Addresses: ") + if ! addresses=$($KUBECTL_BIN get endpointslices -o json --namespace "$NAMESPACE" | \ + jq '.items[] | select(.metadata.ownerReferences[].name=="e2e-service") | .endpoints[].addresses[0]' 2> /dev/null) then # no endpoints ready continue fi - endpts=$(echo "$addresses" | tr -s " " | cut -f 3 -d " ") - - endpt_count=$(echo "$endpts" | tr ',' '\n' | wc -l | xargs) + endpt_count=$(echo "$addresses" | wc -l | xargs) done -echo "$endpts" +echo "$addresses" | tr -d '"' | paste -sd "," - exit 0 diff --git a/integration/scripts/run-tests.sh b/integration/scripts/run-tests.sh index 4e016a2f..ee2d80c8 100755 --- a/integration/scripts/run-tests.sh +++ b/integration/scripts/run-tests.sh @@ -2,20 +2,29 @@ # Runs the AWS Cloud Map MCS Controller for K8s as a background process and tests services have been exported -set -eo pipefail - source ./integration/scripts/common.sh $KUBECTL_BIN apply -f "$CONFIGS/e2e-deployment.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-service.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-export.yaml" -endpts=$(./integration/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") +if ! endpts=$(./integration/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") ; then + exit $? +fi mkdir -p "$LOGS" ./bin/manager &> "$LOGS/ctl.log" & CTL_PID=$! +echo "controller PID:$CTL_PID" go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT "$endpts" +exit_code=$? + +if [ "$exit_code" -eq 0 ] ; then + ./integration/scripts/test-import.sh "$endpts" + exit_code=$? +fi +echo "killing controller PID:$CTL_PID" kill $CTL_PID +exit $exit_code diff --git a/integration/scripts/setup-kind.sh b/integration/scripts/setup-kind.sh index 37fb6358..f37ee54d 100755 --- a/integration/scripts/setup-kind.sh +++ b/integration/scripts/setup-kind.sh @@ -7,6 +7,8 @@ set -e source ./integration/scripts/common.sh +./integration/scripts/ensure-jq.sh + $KIND_BIN create cluster --name "$KIND_SHORT" --image "$IMAGE" $KUBECTL_BIN config use-context "$CLUSTER" $KUBECTL_BIN create namespace "$NAMESPACE" diff --git a/integration/scripts/test-import.sh b/integration/scripts/test-import.sh new file mode 100755 index 00000000..099bb5a2 --- /dev/null +++ b/integration/scripts/test-import.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Test service imports were created during e2e test + +set -e + +source ./integration/scripts/common.sh + +if [ "$#" -ne 1 ]; then + echo "test script expects endpoint IP list as single argument" + exit 1 +fi + +endpts=$1 +echo "checking service imports..." + +imports=$($KUBECTL_BIN get endpointslices -o json --namespace $NAMESPACE | \ + jq '.items[] | select(.metadata.ownerReferences[].name | startswith("imported")) | .endpoints[].addresses[0]') +import_count=$(echo "$imports" | wc -l | xargs) + +if ((import_count != EXPECTED_ENDPOINT_COUNT)) ; then + echo "expected $EXPECTED_ENDPOINT_COUNT imports but found $import_count" + exit 1 +fi + +echo "$imports" | tr -d '"' | while read -r import; do + echo "checking import: $import" + if ! echo "$endpts" | grep -q "$import" ; then + echo "exported endpoint not found: $import" + exit 1 + fi +done + +if [ $? -ne 0 ]; then + exit $? +fi + +echo "matched all imports to exported endpoints" +exit 0 From 1a831be975ab0f1267c9acff93a06edd806f21e2 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Fri, 12 Nov 2021 21:06:00 -0800 Subject: [PATCH 035/163] Add polling to import integration test (#77) --- integration/scripts/test-import.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/integration/scripts/test-import.sh b/integration/scripts/test-import.sh index 099bb5a2..47050429 100755 --- a/integration/scripts/test-import.sh +++ b/integration/scripts/test-import.sh @@ -14,14 +14,22 @@ fi endpts=$1 echo "checking service imports..." -imports=$($KUBECTL_BIN get endpointslices -o json --namespace $NAMESPACE | \ - jq '.items[] | select(.metadata.ownerReferences[].name | startswith("imported")) | .endpoints[].addresses[0]') -import_count=$(echo "$imports" | wc -l | xargs) +import_count=0 +poll_count=0 +while ((import_count < EXPECTED_ENDPOINT_COUNT)) +do + if ((poll_count++ > 30)) ; then + echo "timed out polling for import endpoints" + exit 1 + fi -if ((import_count != EXPECTED_ENDPOINT_COUNT)) ; then - echo "expected $EXPECTED_ENDPOINT_COUNT imports but found $import_count" - exit 1 -fi + imports=$($KUBECTL_BIN get endpointslices -o json --namespace $NAMESPACE | \ + jq '.items[] | select(.metadata.ownerReferences[].name | startswith("imported")) | .endpoints[].addresses[0]') + echo "import endpoint list from kubectl:" + echo "$imports" + + import_count=$(echo "$imports" | wc -l | xargs) +done echo "$imports" | tr -d '"' | while read -r import; do echo "checking import: $import" From 28cb1b591677431f0239ff7ac2779daa7dbbc86b Mon Sep 17 00:00:00 2001 From: Ben Du <5668844+bendu@users.noreply.github.com> Date: Mon, 15 Nov 2021 09:14:48 -0800 Subject: [PATCH 036/163] Add video from Kubecon to readme (#79) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5234c46d..aec2f68f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ ## Introduction AWS Cloud Map multi-cluster service discovery for Kubernetes (K8s) is a controller that implements existing multi-cluster services API that allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. +See the demo from AWS Container Day x KubeCon! + +[![Watch the video](https://img.youtube.com/vi/3f0Tv7IiQQw/0.jpg)](https://youtu.be/3f0Tv7IiQQw?t=24458) + ## Usage > ⚠ **There must exist network connectivity (i.e. VPC peering, security group rules, ACLs, etc.) between clusters**: Undefined behavior may occur if controller is set up without network connectivity between clusters. From 122e3aae1d9e25ec36f8cf1acf65488d44b5f26b Mon Sep 17 00:00:00 2001 From: Ben Du <5668844+bendu@users.noreply.github.com> Date: Tue, 16 Nov 2021 09:22:02 -0800 Subject: [PATCH 037/163] Infer aws region from eks cluster (#82) --- README.md | 4 ++-- main.go | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index aec2f68f..419e69c5 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ See the demo from AWS Container Day x KubeCon! First, install the controller with latest release on at least 2 AWS EKS clusters. Nodes must have sufficient IAM permissions to perform CloudMap operations. -> **_NOTE:_** AWS region environment variable should be set like `export AWS_REGION=us-west-2` +> **_NOTE:_** AWS region environment variable can be _optionaly_ set like `export AWS_REGION=us-west-2` Otherwise controller will infer region in the order `AWS_REGION` environment variable, ~/.aws/config file, then EC2 metadata (for EKS environment) ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release" @@ -74,7 +74,7 @@ kubectl get ServiceImport -A AWS Cloud Map MCS Controller for K8s adheres to the [SemVer](https://semver.org/) specification. Each release updates the major version tag (eg. `vX`), a major/minor version tag (eg. `vX.Y`) and a major/minor/patch version tag (eg. `vX.Y.Z`). To see a full list of all releases, refer to our [Github releases page](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/releases). -> **_NOTE:_** AWS region environment variable should be set like `export AWS_REGION=us-west-2` +> **_NOTE:_** AWS region environment variable can be _optionally_ set like `export AWS_REGION=us-west-2` Otherwise controller will infer region in the order `AWS_REGION` environment variable, ~/.aws/config file, then EC2 metadata (for EKS environment) To install from a release run ```sh diff --git a/main.go b/main.go index a12c492b..bd76d197 100644 --- a/main.go +++ b/main.go @@ -19,10 +19,11 @@ package main import ( "context" "flag" + "os" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" "github.com/aws/aws-sdk-go-v2/config" - "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -84,18 +85,17 @@ func main() { setupLog.Error(err, "unable to start manager") os.Exit(1) } + setupLog.Info("configuring AWS session") + // GO sdk will look for region in order 1) AWS_REGION env var, 2) ~/.aws/config file, 3) EC2 IMDS + awsCfg, err := config.LoadDefaultConfig(context.TODO(), config.WithEC2IMDSRegion()) - // TODO: configure session - awsRegion := os.Getenv("AWS_REGION") - setupLog.Info("configuring AWS session", "AWS_REGION", awsRegion) - awsCfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(awsRegion), - ) - if err != nil { - setupLog.Error(err, "unable to configure AWS session", "AWS_REGION", awsRegion) + if err != nil || awsCfg.Region == "" { + setupLog.Error(err, "unable to configure AWS session", "AWS_REGION", awsCfg.Region) os.Exit(1) } + setupLog.Info("Running with AWS region", "AWS_REGION", awsCfg.Region) + serviceDiscoveryClient := cloudmap.NewDefaultServiceDiscoveryClient(&awsCfg) if err = (&controllers.ServiceExportReconciler{ Client: mgr.GetClient(), From 239ad38a9e753b914ff9430155e07e807bfa51c5 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Tue, 16 Nov 2021 09:43:57 -0800 Subject: [PATCH 038/163] Exporting service and endpoints port attributes (#78) * Exporting service's Port, TargetPort, Protocol; and endpoint's Port to the Cloumap as an Attributes. Also, create the derived service and endpoint slices using these ports information. Improve reconcile service logic. Also, add the new tests. * Include the protocol to ensure the uniqueness of the Port. Move test literals into constants. Minor refactorings. * Revert the golint refactor --- go.sum | 174 +++++++++++++++++ integration/scenarios/export_service.go | 23 ++- integration/scenarios/runner/main.go | 13 +- integration/scripts/common.sh | 1 + integration/scripts/run-tests.sh | 2 +- pkg/cloudmap/cache_test.go | 6 +- pkg/cloudmap/client.go | 2 +- pkg/cloudmap/client_test.go | 83 +++++--- pkg/controllers/cloudmap_controller.go | 124 +++++------- pkg/controllers/cloudmap_controller_test.go | 89 +++++++++ pkg/controllers/serviceexport_controller.go | 24 ++- .../serviceexport_controller_test.go | 38 ++-- pkg/controllers/utils.go | 69 +++++++ pkg/controllers/utils_test.go | 183 ++++++++++++++++++ pkg/model/types.go | 143 ++++++++++---- pkg/model/types_test.go | 138 +++++++++---- test/test-constants.go | 86 +++++--- 17 files changed, 962 insertions(+), 236 deletions(-) create mode 100644 pkg/controllers/cloudmap_controller_test.go create mode 100644 pkg/controllers/utils.go create mode 100644 pkg/controllers/utils_test.go diff --git a/go.sum b/go.sum index 0e7d1fae..b8793a89 100644 --- a/go.sum +++ b/go.sum @@ -12,17 +12,24 @@ cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -42,18 +49,29 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag= github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= @@ -79,47 +97,74 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -127,13 +172,18 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -142,17 +192,23 @@ github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -182,6 +238,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -195,103 +252,159 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -300,12 +413,17 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -319,8 +437,11 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -328,7 +449,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 h1:0XM1XL/OFFJjXsYXlG30spTkV/E9+gmd5GD1w2HE8xM= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -351,35 +474,54 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -387,20 +529,28 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -436,8 +586,10 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -451,6 +603,7 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -502,6 +655,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -543,6 +697,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -616,6 +771,7 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -643,6 +799,7 @@ google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -650,6 +807,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -662,19 +820,27 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -691,6 +857,7 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclp gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -707,15 +874,18 @@ k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apiserver v0.20.1 h1:yEqdkxlnQbxi/3e74cp0X16h140fpvPrNnNRAJBDuBk= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/code-generator v0.20.1 h1:kre3GNich5gbO3d1FyTT8fHI4ZJezZV217yFdWlQaRQ= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded h1:JApXBKYyB7l9xx+DK7/+mFjC7A9Bt5A93FPvFD0HIFE= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= @@ -726,9 +896,13 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAG k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14 h1:TihvEz9MPj2u0KWds6E2OBUXfwaL4qRJ33c7HGiJpqk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= diff --git a/integration/scenarios/export_service.go b/integration/scenarios/export_service.go index 46b2e386..75527cdd 100644 --- a/integration/scenarios/export_service.go +++ b/integration/scenarios/export_service.go @@ -30,20 +30,33 @@ type exportServiceScenario struct { expectedSvc model.Service } -func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, portStr string, ips string) (ExportServiceScenario, error) { +func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, portStr string, servicePortStr string, ips string) (ExportServiceScenario, error) { endpts := make([]*model.Endpoint, 0) port, parseError := strconv.ParseUint(portStr, 10, 16) if parseError != nil { return nil, parseError } + servicePort, parseError := strconv.ParseUint(servicePortStr, 10, 16) + if parseError != nil { + return nil, parseError + } for _, ip := range strings.Split(ips, ",") { + endpointPort := model.Port{ + Port: int32(port), + Protocol: model.TCPProtocol, + } endpts = append(endpts, &model.Endpoint{ - Id: model.EndpointIdFromIPAddress(ip), - IP: ip, - Port: int32(port), - Attributes: make(map[string]string, 0), + Id: model.EndpointIdFromIPAddressAndPort(ip, endpointPort), + IP: ip, + ServicePort: model.Port{ + Port: int32(servicePort), + TargetPort: portStr, + Protocol: model.TCPProtocol, + }, + EndpointPort: endpointPort, + Attributes: make(map[string]string), }) } diff --git a/integration/scenarios/runner/main.go b/integration/scenarios/runner/main.go index a82cefd6..6fe27428 100644 --- a/integration/scenarios/runner/main.go +++ b/integration/scenarios/runner/main.go @@ -10,23 +10,24 @@ import ( ) func main() { - if len(os.Args) != 5 { - fmt.Println("Expected namespace, service, endpoint port, and endpoint IP list arguments") + if len(os.Args) != 6 { + fmt.Println("Expected namespace, service, endpoint port, service port and endpoint IP list arguments") os.Exit(1) } nsName := os.Args[1] svcName := os.Args[2] port := os.Args[3] - ips := os.Args[4] + servicePort := os.Args[4] + ips := os.Args[5] - testServiceExport(nsName, svcName, port, ips) + testServiceExport(nsName, svcName, port, servicePort, ips) } -func testServiceExport(nsName string, svcName string, port string, ips string) { +func testServiceExport(nsName string, svcName string, port string, servicePort string, ips string) { fmt.Printf("Testing service export integration for namespace %s and service %s\n", nsName, svcName) - export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, port, ips) + export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, port, servicePort, ips) if err != nil { fmt.Printf("Failed to setup service export integration test scenario: %s", err.Error()) os.Exit(1) diff --git a/integration/scripts/common.sh b/integration/scripts/common.sh index 211da54a..0067ae21 100755 --- a/integration/scripts/common.sh +++ b/integration/scripts/common.sh @@ -8,6 +8,7 @@ SCENARIOS='./integration/scenarios' NAMESPACE='aws-cloud-map-mcs-e2e' SERVICE='e2e-service' ENDPT_PORT=80 +SERVICE_PORT=8080 KIND_SHORT='cloud-map-e2e' CLUSTER='kind-cloud-map-e2e' IMAGE='kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729' diff --git a/integration/scripts/run-tests.sh b/integration/scripts/run-tests.sh index ee2d80c8..4b2894b2 100755 --- a/integration/scripts/run-tests.sh +++ b/integration/scripts/run-tests.sh @@ -17,7 +17,7 @@ mkdir -p "$LOGS" CTL_PID=$! echo "controller PID:$CTL_PID" -go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT "$endpts" +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$endpts" exit_code=$? if [ "$exit_code" -eq 0 ] ; then diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index e8377bcd..03473544 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -91,11 +91,11 @@ func TestServiceDiscoveryClientCacheGetServiceId_Corrupt(t *testing.T) { func TestServiceDiscoveryClientCacheGetEndpoints_Found(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) assert.True(t, found) - assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, endpts) + assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, endpts) } func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { @@ -117,7 +117,7 @@ func TestServiceDiscoveryClientCacheGetEndpoints_Corrupt(t *testing.T) { func TestServiceDiscoveryClientEvictEndpoints(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) sdc.EvictEndpoints(test.NsName, test.SvcName) endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 0f32bdcc..6c78675d 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -233,7 +233,7 @@ func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, nsName str for _, inst := range insts { endpt, endptErr := model.NewEndpointFromInstance(&inst) if endptErr != nil { - sdc.log.Info(fmt.Sprintf("skipping instance %s to endpoint conversion: %s", *inst.InstanceId, endptErr.Error())) + sdc.log.Error(endptErr, "skipping instance to endpoint conversion", "instanceId", *inst.InstanceId) continue } endpts = append(endpts, endpt) diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 4aaf77c9..5ab71be0 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -45,20 +45,32 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { { InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ - model.Ipv4Attr: test.EndptIp1, - model.PortAttr: test.EndptPortStr1, + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, }, }, { InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ - model.Ipv4Attr: test.EndptIp2, - model.PortAttr: test.EndptPortStr2, + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, }, }, }, nil) tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, - []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) @@ -76,7 +88,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName). - Return([]*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, true) + Return([]*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, true) svcs, err := tc.client.ListServices(context.TODO(), test.NsName) assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) @@ -283,20 +295,32 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { { InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ - model.Ipv4Attr: test.EndptIp1, - model.PortAttr: test.EndptPortStr1, + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, }, }, { InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ - model.Ipv4Attr: test.EndptIp2, - model.PortAttr: test.EndptPortStr2, + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, }, }, }, nil) tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, - []*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}) + []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err) @@ -308,7 +332,7 @@ func TestServiceDiscoveryClient_GetService_CachedValues(t *testing.T) { defer tc.close() tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName). - Return([]*model.Endpoint{test.GetTestEndpoint(), test.GetTestEndpoint2()}, true) + Return([]*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, true) svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) assert.Nil(t, err) @@ -321,8 +345,26 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName).Return(test.SvcId, true) - attrs1 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp1, "AWS_INSTANCE_PORT": test.EndptPortStr1} - attrs2 := map[string]string{"AWS_INSTANCE_IPV4": test.EndptIp2, "AWS_INSTANCE_PORT": test.EndptPortStr2} + attrs1 := map[string]string{ + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, + } + attrs2 := map[string]string{ + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, + } tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). Return(test.OpId1, nil) @@ -336,18 +378,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc.mockCache.EXPECT().EvictEndpoints(test.NsName, test.SvcName) err := tc.client.RegisterEndpoints(context.TODO(), test.NsName, test.SvcName, - []*model.Endpoint{ - { - Id: test.EndptId1, - IP: test.EndptIp1, - Port: test.EndptPort1, - }, - { - Id: test.EndptId2, - IP: test.EndptIp2, - Port: test.EndptPort2, - }, - }) + []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) assert.Nil(t, err) } diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index c5469b00..917a5eb6 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -9,7 +9,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" - "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -70,6 +70,8 @@ func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { return err } + //TODO: Fetch list of namespaces from Cloudmap and only reconcile the intersection + for _, ns := range namespaces.Items { if err := r.reconcileNamespace(ctx, ns.Name); err != nil { return err @@ -89,6 +91,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa serviceImports := v1alpha1.ServiceImportList{} if err := r.Client.List(ctx, &serviceImports, client.InNamespace(namespaceName)); err != nil { + r.Logger.Error(err, "failed to reconcile namespace", "namespace", namespaceName) return nil } @@ -111,7 +114,10 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa // delete remaining imports that have not been matched for _, i := range existingImportsMap { - r.Client.Delete(ctx, &i) + if err := r.Client.Delete(ctx, &i); err != nil { + r.Logger.Error(err, "error deleting ServiceImport", "namespace", i.Namespace, "name", i.Name) + continue + } r.Logger.Info("delete ServiceImport", "namespace", i.Namespace, "name", i.Name) } @@ -121,38 +127,36 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Service) error { r.Logger.Info("syncing service", "namespace", svc.Namespace, "service", svc.Name) - // create ServiceImport if doesn't exist - svcImport, err := r.getExistingServiceImport(ctx, svc.Namespace, svc.Name) + svcImport, err := r.getServiceImport(ctx, svc.Namespace, svc.Name) if err != nil { if !errors.IsNotFound(err) { return err } - if err2 := r.createServiceImport(ctx, svc.Namespace, svc.Name); err2 != nil { - return err2 + // create ServiceImport if it doesn't exist + if svcImport, err = r.createAndGetServiceImport(ctx, svc.Namespace, svc.Name); err != nil { + return err } - return nil } - // create derived Service if it doesn't exist - existingService, err := r.getExistingDerivedService(ctx, svc.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) + derivedService, err := r.getDerivedService(ctx, svc.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) if err != nil { if !errors.IsNotFound(err) { return err } - if err2 := r.createDerivedService(ctx, svc, svcImport); err2 != nil { - return err2 + // create derived Service if it doesn't exist + if derivedService, err = r.createAndGetDerivedService(ctx, svc, svcImport); err != nil { + return err } - return nil } // update ServiceImport to match IP and port of previously created service - if err = r.updateServiceImport(ctx, svcImport, existingService); err != nil { + if err = r.updateServiceImport(ctx, svcImport, derivedService); err != nil { return err } - err = r.updateEndpointSlices(ctx, svcImport, svc, existingService) + err = r.updateEndpointSlices(ctx, svcImport, svc, derivedService) if err != nil { return err } @@ -160,14 +164,13 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se return nil } -func (r *CloudMapReconciler) getExistingServiceImport(ctx context.Context, namespace string, name string) (*v1alpha1.ServiceImport, error) { +func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace string, name string) (*v1alpha1.ServiceImport, error) { existingServiceImport := &v1alpha1.ServiceImport{} err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existingServiceImport) - return existingServiceImport, err } -func (r *CloudMapReconciler) createServiceImport(ctx context.Context, namespace string, name string) error { +func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, namespace string, name string) (*v1alpha1.ServiceImport, error) { imp := &v1alpha1.ServiceImport{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -182,36 +185,34 @@ func (r *CloudMapReconciler) createServiceImport(ctx context.Context, namespace } if err := r.Client.Create(ctx, imp); err != nil { - return err + return nil, err } r.Logger.Info("created ServiceImport", "namespace", imp.Namespace, "name", imp.Name) - return nil + return r.getServiceImport(ctx, namespace, name) } -func (r *CloudMapReconciler) getExistingDerivedService(ctx context.Context, namespace string, name string) (*v1.Service, error) { +func (r *CloudMapReconciler) getDerivedService(ctx context.Context, namespace string, name string) (*v1.Service, error) { existingService := &v1.Service{} err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existingService) - return existingService, err } -func (r *CloudMapReconciler) createDerivedService(ctx context.Context, svc *model.Service, svcImport *v1alpha1.ServiceImport) error { +func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svc *model.Service, svcImport *v1alpha1.ServiceImport) (*v1.Service, error) { toCreate := createDerivedServiceStruct(svc, svcImport) if err := r.Client.Create(ctx, toCreate); err != nil { - return err + return nil, err } - r.Logger.Info("created derived Service", - "namespace", toCreate.Namespace, "name", toCreate.Name) + r.Logger.Info("created derived Service", "namespace", toCreate.Namespace, "name", toCreate.Name) - return nil + return r.getDerivedService(ctx, svc.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) } func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *v1alpha1.ServiceImport, cloudMapService *model.Service, svc *v1.Service) error { - existingSlicesList := v1beta1.EndpointSliceList{} - var existingSlices []*v1beta1.EndpointSlice + existingSlicesList := discovery.EndpointSliceList{} + var existingSlices []*discovery.EndpointSlice if err := r.Client.List(ctx, &existingSlicesList, - client.InNamespace(svc.Namespace), client.MatchingLabels{v1beta1.LabelServiceName: svc.Name}); err != nil { + client.InNamespace(svc.Namespace), client.MatchingLabels{discovery.LabelServiceName: svc.Name}); err != nil { return err } if len(existingSlicesList.Items) == 0 { @@ -256,16 +257,16 @@ func createDerivedServiceStruct(svc *model.Service, svcImport *v1alpha1.ServiceI } } -func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc *model.Service, svc *v1.Service) []*v1beta1.EndpointSlice { - slices := make([]*v1beta1.EndpointSlice, 0) +func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc *model.Service, svc *v1.Service) []*discovery.EndpointSlice { + slices := make([]*discovery.EndpointSlice, 0) t := true - endpoints := make([]v1beta1.Endpoint, 0) + endpoints := make([]discovery.Endpoint, 0) for _, ep := range cloudMapSvc.Endpoints { - endpoints = append(endpoints, v1beta1.Endpoint{ + endpoints = append(endpoints, discovery.Endpoint{ Addresses: []string{ep.IP}, - Conditions: v1beta1.EndpointConditions{ + Conditions: discovery.EndpointConditions{ Ready: &t, }, TargetRef: &v1.ObjectReference{ @@ -281,11 +282,11 @@ func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc * // TODO split slices in case there are more than 1000 endpoints // see https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0752-endpointslices/README.md - slices = append(slices, &v1beta1.EndpointSlice{ + slices = append(slices, &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1beta1.LabelServiceName: svc.Name, // derived Service name - LabelServiceImportName: svcImport.Name, // original ServiceImport name + discovery.LabelServiceName: svc.Name, // derived Service name + LabelServiceImportName: svcImport.Name, // original ServiceImport name }, GenerateName: svc.Name + "-", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(svc, schema.GroupVersionKind{ @@ -294,7 +295,7 @@ func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc * })}, Namespace: svc.Namespace, }, - AddressType: v1beta1.AddressTypeIPv4, + AddressType: discovery.AddressTypeIPv4, Endpoints: endpoints, Ports: extractEndpointPorts(cloudMapSvc), }) @@ -303,49 +304,30 @@ func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc * } func extractServicePorts(svc *model.Service) []v1.ServicePort { - ports := extractPorts(svc) - - servicePorts := make([]v1.ServicePort, 0, len(ports)) - for _, p := range ports { - servicePorts = append(servicePorts, v1.ServicePort{ - Protocol: v1.ProtocolTCP, - Port: p, - }) + uniquePorts := make(map[string]model.Port) + for _, ep := range svc.Endpoints { + uniquePorts[ep.ServicePort.GetID()] = ep.ServicePort } - return servicePorts -} - -func extractEndpointPorts(svc *model.Service) []v1beta1.EndpointPort { - ports := extractPorts(svc) - - protocol := v1.ProtocolTCP - - endpointPorts := make([]v1beta1.EndpointPort, 0, len(ports)) - for _, p := range ports { - endpointPorts = append(endpointPorts, v1beta1.EndpointPort{ - Protocol: &protocol, - Port: &p, - }) + servicePorts := make([]v1.ServicePort, 0, len(uniquePorts)) + for _, servicePort := range uniquePorts { + servicePorts = append(servicePorts, PortToServicePort(servicePort)) } - return endpointPorts + return servicePorts } -func extractPorts(svc *model.Service) []int32 { - ports := make([]int32, 0) - - portMap := make(map[int32]bool, 0) - +func extractEndpointPorts(svc *model.Service) []discovery.EndpointPort { + uniquePorts := make(map[string]model.Port) for _, ep := range svc.Endpoints { - portMap[ep.Port] = true + uniquePorts[ep.EndpointPort.GetID()] = ep.EndpointPort } - for p, _ := range portMap { - ports = append(ports, p) + endpointPorts := make([]discovery.EndpointPort, 0, len(uniquePorts)) + for _, endpointPort := range uniquePorts { + endpointPorts = append(endpointPorts, PortToEndpointPort(endpointPort)) } - - return ports + return endpointPorts } func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport *v1alpha1.ServiceImport, svc *v1.Service) error { diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go new file mode 100644 index 00000000..c926b431 --- /dev/null +++ b/pkg/controllers/cloudmap_controller_test.go @@ -0,0 +1,89 @@ +package controllers + +import ( + "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + testingLogger "github.com/go-logr/logr/testing" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/api/discovery/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "strings" + "testing" +) + +func TestCloudMapReconciler_Reconcile(t *testing.T) { + // create a fake controller client and add some objects + objs := []runtime.Object{testNamespace()} + + s := scheme.Scheme + s.AddKnownTypes(v1alpha1.GroupVersion, &v1alpha1.ServiceImportList{}, &v1alpha1.ServiceImport{}) + + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + + // create a mock cloudmap service discovery client + mockController := gomock.NewController(t) + defer mockController.Finish() + + mockSDClient := cloudmap.NewMockServiceDiscoveryClient(mockController) + // The service model in the Cloudmap + mockSDClient.EXPECT().ListServices(context.TODO(), test.NsName). + Return([]*model.Service{test.GetTestServiceWithEndpoint1()}, nil) + + reconciler := getReconciler(t, mockSDClient, fakeClient) + + err := reconciler.Reconcile(context.TODO()) + if err != nil { + t.Fatalf("reconcile failed: (%v)", err) + } + + // assert service import object + serviceImport := &v1alpha1.ServiceImport{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceImport) + assert.NoError(t, err) + assert.Equal(t, test.SvcName, serviceImport.Name, "Service imported") + + // assert derived service is successfully created + derivedServiceList := &v1.ServiceList{} + err = fakeClient.List(context.TODO(), derivedServiceList, client.InNamespace(test.NsName)) + assert.NoError(t, err) + derivedService := derivedServiceList.Items[0] + assert.True(t, strings.Contains(derivedService.Name, "imported-"), "Derived service created", "service", derivedService.Name) + assert.Equal(t, int32(test.ServicePort1), derivedService.Spec.Ports[0].Port) + assert.Equal(t, int32(test.Port1), derivedService.Spec.Ports[0].TargetPort.IntVal) + + // assert endpoint slices are created + endpointSliceList := &v1beta1.EndpointSliceList{} + err = fakeClient.List(context.TODO(), endpointSliceList, client.InNamespace(test.NsName)) + assert.NoError(t, err) + endpointSlice := endpointSliceList.Items[0] + assert.Equal(t, test.SvcName, endpointSlice.Labels["multicluster.kubernetes.io/service-name"], "Endpoint slice is created") + assert.Equal(t, int32(test.Port1), *endpointSlice.Ports[0].Port) + assert.Equal(t, test.EndptIp1, endpointSlice.Endpoints[0].Addresses[0]) +} + +func testNamespace() *v1.Namespace { + return &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.NsName, + Namespace: test.NsName, + }, + } +} + +func getReconciler(t *testing.T, mockSDClient *cloudmap.MockServiceDiscoveryClient, client client.Client) *CloudMapReconciler { + return &CloudMapReconciler{ + Client: client, + Cloudmap: mockSDClient, + Logger: testingLogger.TestLogger{T: t}, + } +} diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index bdef8bcc..e6fe4d9c 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -186,25 +186,31 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. return nil, err } + servicePortMap := make(map[string]model.Port) + for _, svcPort := range svc.Spec.Ports { + servicePortMap[svcPort.Name] = ServicePortToPort(svcPort) + } + for _, slice := range endpointSlices.Items { if slice.AddressType != discovery.AddressTypeIPv4 { return nil, fmt.Errorf("unsupported address type %s for service %s", slice.AddressType, svc.Name) } - - for _, port := range slice.Ports { - for _, ep := range slice.Endpoints { - for _, IP := range ep.Addresses { - attributes := make(map[string]string, 0) + for _, endpointPort := range slice.Ports { + for _, endpoint := range slice.Endpoints { + for _, IP := range endpoint.Addresses { + attributes := make(map[string]string) if version.GetVersion() != "" { attributes[K8sVersionAttr] = version.PackageName + " " + version.GetVersion() } // TODO extract attributes - pod, node and other useful details if possible + port := EndpointPortToPort(endpointPort) result = append(result, &model.Endpoint{ - Id: model.EndpointIdFromIPAddress(IP), - IP: IP, - Port: *port.Port, - Attributes: attributes, + Id: model.EndpointIdFromIPAddressAndPort(IP, port), + IP: IP, + EndpointPort: port, + ServicePort: servicePortMap[*endpointPort.Name], + Attributes: attributes, }) } } diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index cb351ecc..badbec31 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -6,6 +6,8 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/aws/aws-sdk-go-v2/aws" testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "gotest.tools/assert" @@ -14,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -27,12 +30,7 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { expectedService := model.Service{ Namespace: "my-namespace", Name: "exported-service", - Endpoints: []*model.Endpoint{{ - Id: "1_1_1_1", - IP: "1.1.1.1", - Port: 80, - Attributes: map[string]string{}, - }}, + Endpoints: []*model.Endpoint{test.GetTestEndpoint1()}, } cloudmapMock := cloudmapmock.NewMockServiceDiscoveryClient(mockController) @@ -71,12 +69,7 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceNewEndpoint(t *testing expectedService := model.Service{ Namespace: "my-namespace", Name: "exported-service", - Endpoints: []*model.Endpoint{{ - Id: "1_1_1_1", - IP: "1.1.1.1", - Port: 80, - Attributes: map[string]string{}, - }}, + Endpoints: []*model.Endpoint{test.GetTestEndpoint1()}, } cloudmapMock := cloudmapmock.NewMockServiceDiscoveryClient(mockController) @@ -125,14 +118,25 @@ func setupK8sClient() client.Client { // Service object service := &v1.Service{ + TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Name: "exported-service", Namespace: "my-namespace", }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Name: "http", + Protocol: test.Protocol1, + Port: test.ServicePort1, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: test.Port1}, + }}, + }, + Status: v1.ServiceStatus{}, } // EndpointSlice object - port := int32(80) + port := int32(test.Port1) + protocol := v1.ProtocolTCP endpointSlice := &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-namespace", @@ -141,9 +145,13 @@ func setupK8sClient() client.Client { }, AddressType: discovery.AddressTypeIPv4, Endpoints: []discovery.Endpoint{{ - Addresses: []string{"1.1.1.1"}, + Addresses: []string{test.EndptIp1}, + }}, + Ports: []discovery.EndpointPort{{ + Name: aws.String("http"), + Protocol: &protocol, + Port: &port, }}, - Ports: []discovery.EndpointPort{{Port: &port}}, } endpointSliceList := &discovery.EndpointSliceList{ Items: []discovery.EndpointSlice{*endpointSlice}, diff --git a/pkg/controllers/utils.go b/pkg/controllers/utils.go new file mode 100644 index 00000000..00341972 --- /dev/null +++ b/pkg/controllers/utils.go @@ -0,0 +1,69 @@ +package controllers + +import ( + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func ServicePortToPort(svcPort v1.ServicePort) model.Port { + return model.Port{ + Name: svcPort.Name, + Port: svcPort.Port, + TargetPort: svcPort.TargetPort.String(), + Protocol: protocolToString(svcPort.Protocol), + } +} + +func EndpointPortToPort(port discovery.EndpointPort) model.Port { + return model.Port{ + Name: *port.Name, + Port: *port.Port, + Protocol: protocolToString(*port.Protocol), + } +} + +func PortToServicePort(port model.Port) v1.ServicePort { + return v1.ServicePort{ + Name: port.Name, + Protocol: stringToProtocol(port.Protocol), + Port: port.Port, + TargetPort: intstr.Parse(port.TargetPort), + } +} + +func PortToEndpointPort(port model.Port) discovery.EndpointPort { + protocol := stringToProtocol(port.Protocol) + return discovery.EndpointPort{ + Name: &port.Name, + Protocol: &protocol, + Port: &port.Port, + } +} + +func protocolToString(protocol v1.Protocol) string { + switch protocol { + case v1.ProtocolTCP: + return model.TCPProtocol + case v1.ProtocolUDP: + return model.UDPProtocol + case v1.ProtocolSCTP: + return model.SCTPProtocol + default: + return "" + } +} + +func stringToProtocol(protocol string) v1.Protocol { + switch protocol { + case model.TCPProtocol: + return v1.ProtocolTCP + case model.UDPProtocol: + return v1.ProtocolUDP + case model.SCTPProtocol: + return v1.ProtocolSCTP + default: + return "" + } +} diff --git a/pkg/controllers/utils_test.go b/pkg/controllers/utils_test.go new file mode 100644 index 00000000..ec970b31 --- /dev/null +++ b/pkg/controllers/utils_test.go @@ -0,0 +1,183 @@ +package controllers + +import ( + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + v1 "k8s.io/api/core/v1" + "k8s.io/api/discovery/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" + "reflect" + "testing" +) + +func TestServicePortToPort(t *testing.T) { + type args struct { + svcPort v1.ServicePort + } + tests := []struct { + name string + args args + want model.Port + }{ + { + name: "happy case", + args: args{ + svcPort: v1.ServicePort{ + Name: "http", + Protocol: v1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8080, + }, + }, + }, + want: model.Port{ + Name: "http", + Port: 80, + TargetPort: "8080", + Protocol: "TCP", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ServicePortToPort(tt.args.svcPort); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ServicePortToPort() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEndpointPortToPort(t *testing.T) { + type args struct { + port v1beta1.EndpointPort + } + name := "http" + protocolTCP := v1.ProtocolTCP + port := int32(80) + tests := []struct { + name string + args args + want model.Port + }{ + { + name: "happy case", + args: args{ + port: v1beta1.EndpointPort{ + Name: &name, + Protocol: &protocolTCP, + Port: &port, + }, + }, + want: model.Port{ + Name: "http", + Port: 80, + TargetPort: "", + Protocol: "TCP", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EndpointPortToPort(tt.args.port); !reflect.DeepEqual(got, tt.want) { + t.Errorf("EndpointPortToPort() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPortToServicePort(t *testing.T) { + type args struct { + port model.Port + } + tests := []struct { + name string + args args + want v1.ServicePort + }{ + { + name: "happy case", + args: args{ + port: model.Port{ + Name: "http", + Port: 80, + TargetPort: "8080", + Protocol: "TCP", + }, + }, + want: v1.ServicePort{ + Name: "http", + Protocol: v1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8080, + }, + }, + }, + { + name: "happy case for string targertPort", + args: args{ + port: model.Port{ + Name: "http", + Port: 80, + TargetPort: "https", + Protocol: "TCP", + }, + }, + want: v1.ServicePort{ + Name: "http", + Protocol: v1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{ + Type: intstr.String, + StrVal: "https", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PortToServicePort(tt.args.port); !reflect.DeepEqual(got, tt.want) { + t.Errorf("PortToServicePort() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPortToEndpointPort(t *testing.T) { + name := "http" + protocolTCP := v1.ProtocolTCP + port := int32(80) + type args struct { + port model.Port + } + tests := []struct { + name string + args args + want v1beta1.EndpointPort + }{ + { + name: "happy case", + args: args{ + port: model.Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + }, + want: v1beta1.EndpointPort{ + Name: &name, + Protocol: &protocolTCP, + Port: &port, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PortToEndpointPort(tt.args.port); !reflect.DeepEqual(got, tt.want) { + t.Errorf("PortToEndpointPort() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/model/types.go b/pkg/model/types.go index 1320f27a..a91eb8e7 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -2,12 +2,10 @@ package model import ( "encoding/json" - "errors" "fmt" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "reflect" "strconv" - "strings" ) // Resource encapsulates a ID/name pair. @@ -41,61 +39,130 @@ type Service struct { // Endpoint holds basic values and attributes for an endpoint. type Endpoint struct { - Id string - IP string + Id string + IP string + EndpointPort Port + ServicePort Port + Attributes map[string]string +} + +type Port struct { + Name string Port int32 - Attributes map[string]string + TargetPort string + Protocol string // TCP, UDP, SCTP } +// Cloudmap Instances IP and Port is supposed to be AWS_INSTANCE_IPV4 and AWS_INSTANCE_PORT +// Rest are custom attributes const ( - Ipv4Attr = "AWS_INSTANCE_IPV4" - PortAttr = "AWS_INSTANCE_PORT" + EndpointIpv4Attr = "AWS_INSTANCE_IPV4" + EndpointPortAttr = "AWS_INSTANCE_PORT" + EndpointPortNameAttr = "ENDPOINT_PORT_NAME" + EndpointProtocolAttr = "ENDPOINT_PROTOCOL" + ServicePortNameAttr = "SERVICE_PORT_NAME" + ServicePortAttr = "SERVICE_PORT" + ServiceTargetPortAttr = "SERVICE_TARGET_PORT" + ServiceProtocolAttr = "SERVICE_PROTOCOL" + TCPProtocol = "TCP" + UDPProtocol = "UDP" + SCTPProtocol = "SCTP" ) // NewEndpointFromInstance converts a Cloud Map HttpInstanceSummary to an endpoint. -func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) { +func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (endpointPtr *Endpoint, err error) { endpoint := Endpoint{ Id: *inst.InstanceId, - Attributes: make(map[string]string, 0), + Attributes: make(map[string]string), + } + attributes := make(map[string]string) + for key, value := range inst.Attributes { + attributes[key] = value } - if ipv4, hasIp := inst.Attributes[Ipv4Attr]; hasIp { - endpoint.IP = ipv4 - } else { - return nil, errors.New(fmt.Sprintf("cannot convert service instance %s to endpoint without IP address", - *inst.InstanceId)) + // Remove and set the IP, Port, Port + if endpoint.IP, err = removeStringAttr(attributes, EndpointIpv4Attr); err != nil { + return nil, err } - if portStr, hasPort := inst.Attributes[PortAttr]; hasPort { - port, parseError := strconv.ParseUint(portStr, 10, 16) + if endpoint.EndpointPort, err = endpointPortFromAttr(attributes); err != nil { + return nil, err + } - if parseError != nil { - return nil, parseError - } + if endpoint.ServicePort, err = servicePortFromAttr(attributes); err != nil { + return nil, err + } - endpoint.Port = int32(port) - } else { - return nil, errors.New(fmt.Sprintf("cannot convert service instance %s to endpoint without port", - *inst.InstanceId)) + // Add the remaining attributes + endpoint.Attributes = attributes + + return &endpoint, err +} + +func endpointPortFromAttr(attributes map[string]string) (port Port, err error) { + port = Port{} + if port.Name, err = removeStringAttr(attributes, EndpointPortNameAttr); err != nil { + return port, err + } + if port.Port, err = removeIntAttr(attributes, EndpointPortAttr); err != nil { + return port, err } + if port.Protocol, err = removeStringAttr(attributes, EndpointProtocolAttr); err != nil { + return port, err + } + return port, err +} - for key, val := range inst.Attributes { - if key != Ipv4Attr && key != PortAttr { - endpoint.Attributes[key] = val - } +func servicePortFromAttr(attributes map[string]string) (port Port, err error) { + port = Port{} + if port.TargetPort, err = removeStringAttr(attributes, ServiceTargetPortAttr); err != nil { + return port, err } + if port.Name, err = removeStringAttr(attributes, ServicePortNameAttr); err != nil { + return port, err + } + if port.Port, err = removeIntAttr(attributes, ServicePortAttr); err != nil { + return port, err + } + if port.Protocol, err = removeStringAttr(attributes, ServiceProtocolAttr); err != nil { + return port, err + } + return port, err +} - return &endpoint, nil +func removeStringAttr(attributes map[string]string, attr string) (string, error) { + if value, hasValue := attributes[attr]; hasValue { + delete(attributes, attr) + return value, nil + } + return "", fmt.Errorf("cannot find the attribute %s", attr) +} + +func removeIntAttr(attributes map[string]string, attr string) (int32, error) { + if value, hasValue := attributes[attr]; hasValue { + parsedValue, parseError := strconv.ParseUint(value, 10, 16) + if parseError != nil { + return 0, fmt.Errorf("failed to parse the %s as int with error %s", + attr, parseError.Error()) + } + delete(attributes, attr) + return int32(parsedValue), nil + } + return 0, fmt.Errorf("cannot find the attribute %s", attr) } // GetCloudMapAttributes extracts endpoint attributes for Cloud Map service instance registration. func (e *Endpoint) GetCloudMapAttributes() map[string]string { - attrs := make(map[string]string, 0) + attrs := make(map[string]string) - attrs[Ipv4Attr] = e.IP - - port := strconv.FormatInt(int64(e.Port), 10) - attrs[PortAttr] = port + attrs[EndpointIpv4Attr] = e.IP + attrs[EndpointPortAttr] = strconv.Itoa(int(e.EndpointPort.Port)) + attrs[EndpointProtocolAttr] = e.EndpointPort.Protocol + attrs[EndpointPortNameAttr] = e.EndpointPort.Name + attrs[ServicePortNameAttr] = e.ServicePort.Name + attrs[ServicePortAttr] = strconv.Itoa(int(e.ServicePort.Port)) + attrs[ServiceTargetPortAttr] = e.ServicePort.TargetPort + attrs[ServiceProtocolAttr] = e.ServicePort.Protocol for key, val := range e.Attributes { attrs[key] = val @@ -119,9 +186,9 @@ func (e *Endpoint) String() string { return string(bytes) } -// EndpointIdFromIPAddress converts an IP address to human readable identifier. -func EndpointIdFromIPAddress(address string) string { - return strings.Replace(address, ".", "_", -1) +// EndpointIdFromIPAddressAndPort converts an IP address to human-readable identifier. +func EndpointIdFromIPAddressAndPort(address string, port Port) string { + return fmt.Sprintf("%s:%s:%d", port.Protocol, address, port.Port) } func ConvertNamespaceType(nsType types.NamespaceType) (namespaceType NamespaceType) { @@ -138,3 +205,7 @@ func ConvertNamespaceType(nsType types.NamespaceType) (namespaceType NamespaceTy func (namespaceType *NamespaceType) IsUnsupported() bool { return *namespaceType == UnsupportedNamespaceType } + +func (port *Port) GetID() string { + return fmt.Sprintf("%s:%d", port.Protocol, port.Port) +} diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index 0e8f45b6..6fc91f2d 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -21,15 +21,31 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - Ipv4Attr: ip, - PortAttr: "65535", - "custom-attr": "custom-val", + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + "custom-attr": "custom-val", }, }, want: &Endpoint{ - Id: instId, - IP: ip, - Port: 65535, + Id: instId, + IP: ip, + EndpointPort: Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + ServicePort: Port{ + Name: "http", + Port: 65535, + TargetPort: "80", + Protocol: "TCP", + }, Attributes: map[string]string{ "custom-attr": "custom-val", }, @@ -40,9 +56,15 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - Ipv4Attr: ip, - PortAttr: "99999", - "custom-attr": "custom-val", + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "99999", + ServiceTargetPortAttr: "80", + "custom-attr": "custom-val", }, }, wantErr: true, @@ -52,8 +74,8 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - PortAttr: "80", - "custom-attr": "custom-val", + EndpointPortAttr: "80", + "custom-attr": "custom-val", }, }, wantErr: true, @@ -63,8 +85,8 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - Ipv4Attr: ip, - "custom-attr": "custom-val", + EndpointIpv4Attr: ip, + "custom-attr": "custom-val", }, }, wantErr: true, @@ -86,10 +108,11 @@ func TestNewEndpointFromInstance(t *testing.T) { func TestEndpoint_GetAttributes(t *testing.T) { type fields struct { - Id string - IP string - Port int32 - Attributes map[string]string + Id string + IP string + EndpointPort Port + ServicePort Port + Attributes map[string]string } tests := []struct { name string @@ -99,26 +122,43 @@ func TestEndpoint_GetAttributes(t *testing.T) { { name: "happy case", fields: fields{ - IP: ip, - Port: 30, + IP: ip, + EndpointPort: Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + ServicePort: Port{ + Name: "http", + Port: 30, + TargetPort: "80", + Protocol: "TCP", + }, Attributes: map[string]string{ "custom-attr": "custom-val", }, }, want: map[string]string{ - Ipv4Attr: ip, - PortAttr: "30", - "custom-attr": "custom-val", + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "30", + ServiceTargetPortAttr: "80", + "custom-attr": "custom-val", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { e := &Endpoint{ - Id: tt.fields.Id, - IP: tt.fields.IP, - Port: tt.fields.Port, - Attributes: tt.fields.Attributes, + Id: tt.fields.Id, + IP: tt.fields.IP, + EndpointPort: tt.fields.EndpointPort, + ServicePort: tt.fields.ServicePort, + Attributes: tt.fields.Attributes, } if got := e.GetCloudMapAttributes(); !reflect.DeepEqual(got, tt.want) { t.Errorf("GetAttributes() = %v, want %v", got, tt.want) @@ -127,22 +167,28 @@ func TestEndpoint_GetAttributes(t *testing.T) { } } -func TestEndpointIdFromIPAddress(t *testing.T) { +func TestEndpointIdFromIPAddressAndPort(t *testing.T) { tests := []struct { name string address string + port Port want string }{ { name: "happy case", address: "192.168.0.1", - want: "192_168_0_1", + port: Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + want: "TCP:192.168.0.1:80", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := EndpointIdFromIPAddress(tt.address); got != tt.want { - t.Errorf("EndpointIdFromIPAddress() = %v, want %v", got, tt.want) + if got := EndpointIdFromIPAddressAndPort(tt.address, tt.port); got != tt.want { + t.Errorf("EndpointIdFromIPAddressAndPort() = %v, want %v", got, tt.want) } }) } @@ -150,18 +196,34 @@ func TestEndpointIdFromIPAddress(t *testing.T) { func TestEndpoint_Equals(t *testing.T) { firstEndpoint := Endpoint{ - Id: instId, - IP: ip, - Port: 80, + Id: instId, + IP: ip, + ServicePort: Port{ + Port: 80, + }, Attributes: map[string]string{ "custom-key": "custom-val", }, } secondEndpoint := Endpoint{ - Id: instId, - IP: ip, - Port: 80, + Id: instId, + IP: ip, + ServicePort: Port{ + Port: 80, + Name: "", + }, + Attributes: map[string]string{ + "custom-key": "custom-val", + }, + } + + thirdEndpoint := Endpoint{ + Id: instId, + IP: ip, + ServicePort: Port{ + Port: 80, + }, Attributes: map[string]string{ "custom-key": "different-val", }, @@ -176,13 +238,13 @@ func TestEndpoint_Equals(t *testing.T) { { name: "identical", x: firstEndpoint, - y: firstEndpoint, + y: secondEndpoint, want: true, }, { name: "different", x: firstEndpoint, - y: secondEndpoint, + y: thirdEndpoint, want: false, }, } diff --git a/test/test-constants.go b/test/test-constants.go index 6954f4e6..e9da5f22 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -5,21 +5,29 @@ import ( ) const ( - NsName = "ns-name" - NsId = "ns-id" - SvcName = "svc-name" - SvcId = "svc-id" - EndptId1 = "endpoint-id-1" - EndptId2 = "endpoint-id-2" - EndptIp1 = "192.168.0.1" - EndptIp2 = "192.168.0.2" - EndptPort1 = 1 - EndptPortStr1 = "1" - EndptPort2 = 2 - EndptPortStr2 = "2" - OpId1 = "operation-id-1" - OpId2 = "operation-id-2" - OpStart = 1 + NsName = "ns-name" + NsId = "ns-id" + SvcName = "svc-name" + SvcId = "svc-id" + EndptId1 = "TCP:192.168.0.1:1" + EndptId2 = "TCP:192.168.0.2:2" + EndptIp1 = "192.168.0.1" + EndptIp2 = "192.168.0.2" + Port1 = 1 + PortStr1 = "1" + PortName1 = "http" + Protocol1 = "TCP" + ServicePort1 = 11 + ServicePortStr1 = "11" + Port2 = 2 + PortStr2 = "2" + PortName2 = "http" + Protocol2 = "TCP" + ServicePort2 = 22 + ServicePortStr2 = "22" + OpId1 = "operation-id-1" + OpId2 = "operation-id-2" + OpStart = 1 ) func GetTestHttpNamespace() *model.Namespace { @@ -42,24 +50,52 @@ func GetTestService() *model.Service { return &model.Service{ Namespace: NsName, Name: SvcName, - Endpoints: []*model.Endpoint{GetTestEndpoint(), GetTestEndpoint2()}, + Endpoints: []*model.Endpoint{GetTestEndpoint1(), GetTestEndpoint2()}, } } -func GetTestEndpoint() *model.Endpoint { +func GetTestServiceWithEndpoint1() *model.Service { + return &model.Service{ + Namespace: NsName, + Name: SvcName, + Endpoints: []*model.Endpoint{GetTestEndpoint1()}, + } +} + +func GetTestEndpoint1() *model.Endpoint { return &model.Endpoint{ - Id: EndptId1, - IP: EndptIp1, - Port: EndptPort1, - Attributes: make(map[string]string, 0), + Id: EndptId1, + IP: EndptIp1, + EndpointPort: model.Port{ + Name: PortName1, + Port: Port1, + Protocol: Protocol1, + }, + ServicePort: model.Port{ + Name: PortName1, + Port: ServicePort1, + TargetPort: PortStr1, + Protocol: Protocol1, + }, + Attributes: make(map[string]string), } } func GetTestEndpoint2() *model.Endpoint { return &model.Endpoint{ - Id: EndptId2, - IP: EndptIp2, - Port: EndptPort2, - Attributes: make(map[string]string, 0), + Id: EndptId2, + IP: EndptIp2, + EndpointPort: model.Port{ + Name: PortName2, + Port: Port2, + Protocol: Protocol2, + }, + ServicePort: model.Port{ + Name: PortName2, + Port: ServicePort2, + TargetPort: PortStr2, + Protocol: Protocol2, + }, + Attributes: make(map[string]string), } } From 7dec8da1fddd74df22ff90ee673c4e5ec2f01fa8 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Tue, 16 Nov 2021 11:07:25 -0800 Subject: [PATCH 039/163] Fix go.sum file (#85) --- go.sum | 174 --------------------------------------------------------- 1 file changed, 174 deletions(-) diff --git a/go.sum b/go.sum index b8793a89..0e7d1fae 100644 --- a/go.sum +++ b/go.sum @@ -12,24 +12,17 @@ cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -49,29 +42,18 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag= github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= @@ -97,74 +79,47 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -172,18 +127,13 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -192,23 +142,17 @@ github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -238,7 +182,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -252,159 +195,103 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -413,17 +300,12 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -437,11 +319,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -449,9 +328,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 h1:0XM1XL/OFFJjXsYXlG30spTkV/E9+gmd5GD1w2HE8xM= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -474,54 +351,35 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -529,28 +387,20 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -586,10 +436,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -603,7 +451,6 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -655,7 +502,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -697,7 +543,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -771,7 +616,6 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -799,7 +643,6 @@ google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -807,7 +650,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -820,27 +662,19 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -857,7 +691,6 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclp gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -874,18 +707,15 @@ k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apiserver v0.20.1 h1:yEqdkxlnQbxi/3e74cp0X16h140fpvPrNnNRAJBDuBk= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= -k8s.io/code-generator v0.20.1 h1:kre3GNich5gbO3d1FyTT8fHI4ZJezZV217yFdWlQaRQ= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded h1:JApXBKYyB7l9xx+DK7/+mFjC7A9Bt5A93FPvFD0HIFE= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= @@ -896,13 +726,9 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAG k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14 h1:TihvEz9MPj2u0KWds6E2OBUXfwaL4qRJ33c7HGiJpqk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= From 15874d0b4f2edf0fbcd8a2e335fdf76b2235a54d Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:34:30 -0800 Subject: [PATCH 040/163] Reformat instance id to the format {protocol}://{ip}:{port} (#86) --- pkg/model/types.go | 3 ++- pkg/model/types_test.go | 2 +- test/test-constants.go | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/model/types.go b/pkg/model/types.go index a91eb8e7..fc781e73 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -6,6 +6,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "reflect" "strconv" + "strings" ) // Resource encapsulates a ID/name pair. @@ -188,7 +189,7 @@ func (e *Endpoint) String() string { // EndpointIdFromIPAddressAndPort converts an IP address to human-readable identifier. func EndpointIdFromIPAddressAndPort(address string, port Port) string { - return fmt.Sprintf("%s:%s:%d", port.Protocol, address, port.Port) + return fmt.Sprintf("%s://%s:%d", strings.ToLower(port.Protocol), address, port.Port) } func ConvertNamespaceType(nsType types.NamespaceType) (namespaceType NamespaceType) { diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index 6fc91f2d..db6eb5ed 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -182,7 +182,7 @@ func TestEndpointIdFromIPAddressAndPort(t *testing.T) { Port: 80, Protocol: "TCP", }, - want: "TCP:192.168.0.1:80", + want: "tcp://192.168.0.1:80", }, } for _, tt := range tests { diff --git a/test/test-constants.go b/test/test-constants.go index e9da5f22..8cd87441 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -9,8 +9,8 @@ const ( NsId = "ns-id" SvcName = "svc-name" SvcId = "svc-id" - EndptId1 = "TCP:192.168.0.1:1" - EndptId2 = "TCP:192.168.0.2:2" + EndptId1 = "tcp://192.168.0.1:1" + EndptId2 = "tcp://192.168.0.2:2" EndptIp1 = "192.168.0.1" EndptIp2 = "192.168.0.2" Port1 = 1 From 94c448895b92839e7727b1d9c914dec41668d739 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:40:34 -0800 Subject: [PATCH 041/163] Add slack channel link (#87) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 419e69c5..f0c53689 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ The end-to-end integration test suite can be run locally to validate controller make integration-suite ``` +## Slack community +We have an open Slack community where users may get support with integration, discuss controller functionality and provide input on our feature roadmap. https://awsappmesh.slack.com/#k8s-mcs-controller +Join the channel with this [invite](https://join.slack.com/t/awsappmesh/shared_invite/zt-dwgbt85c-Sj_md92__quV8YADKfsQSA). + ## Contributing `aws-cloud-map-mcs-controller-for-k8s` is an open source project. See [CONTRIBUTING](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/CONTRIBUTING.md) for details. From a23d0c7a1fab58047860a63d16d3355b1807e917 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Thu, 18 Nov 2021 17:25:50 -0800 Subject: [PATCH 042/163] Sync Service/Deployment changes (#88) --- integration/scripts/common.sh | 1 + integration/scripts/run-tests.sh | 27 ++- integration/scripts/test-import.sh | 11 +- main.go | 2 +- pkg/controllers/cloudmap_controller_test.go | 2 +- pkg/controllers/serviceexport_controller.go | 229 ++++++++++++------ .../serviceexport_controller_test.go | 219 ++++++++++------- pkg/model/plan.go | 12 + test/test-constants.go | 4 +- 9 files changed, 344 insertions(+), 163 deletions(-) diff --git a/integration/scripts/common.sh b/integration/scripts/common.sh index 0067ae21..3c94c44d 100755 --- a/integration/scripts/common.sh +++ b/integration/scripts/common.sh @@ -13,3 +13,4 @@ KIND_SHORT='cloud-map-e2e' CLUSTER='kind-cloud-map-e2e' IMAGE='kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729' EXPECTED_ENDPOINT_COUNT=5 +UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/scripts/run-tests.sh b/integration/scripts/run-tests.sh index 4b2894b2..39919ba3 100755 --- a/integration/scripts/run-tests.sh +++ b/integration/scripts/run-tests.sh @@ -21,10 +21,35 @@ go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT " exit_code=$? if [ "$exit_code" -eq 0 ] ; then - ./integration/scripts/test-import.sh "$endpts" + ./integration/scripts/test-import.sh "$EXPECTED_ENDPOINT_COUNT" "$endpts" exit_code=$? fi +echo "sleeping..." +sleep 2s + +deployment=$($KUBECTL_BIN get deployment --namespace "$NAMESPACE" -o json | jq -r '.items[0].metadata.name') + +echo "scaling the deployment $deployment to $UPDATED_ENDPOINT_COUNT" +$KUBECTL_BIN scale deployment/"$deployment" --replicas="$UPDATED_ENDPOINT_COUNT" --namespace "$NAMESPACE" +exit_code=$? + +if [ "$exit_code" -eq 0 ] ; then + if ! updated_endpoints=$(./integration/scripts/poll-endpoints.sh "$UPDATED_ENDPOINT_COUNT") ; then + exit $? + fi + + go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" + exit_code=$? + + # TODO: The reconciliation during the import is not implemented yet, uncomment below once done + # + #if [ "$exit_code" -eq 0 ] ; then + # ./integration/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" + # exit_code=$? + #fi +fi + echo "killing controller PID:$CTL_PID" kill $CTL_PID exit $exit_code diff --git a/integration/scripts/test-import.sh b/integration/scripts/test-import.sh index 47050429..8c111949 100755 --- a/integration/scripts/test-import.sh +++ b/integration/scripts/test-import.sh @@ -6,17 +6,18 @@ set -e source ./integration/scripts/common.sh -if [ "$#" -ne 1 ]; then - echo "test script expects endpoint IP list as single argument" +if [ "$#" -ne 2 ]; then + echo "test script expects \"expected number of endpoints\" and endpoints IP list as arguments" exit 1 fi -endpts=$1 +expected_endpoint_count=$1 +endpoints=$2 echo "checking service imports..." import_count=0 poll_count=0 -while ((import_count < EXPECTED_ENDPOINT_COUNT)) +while ((import_count < expected_endpoint_count)) do if ((poll_count++ > 30)) ; then echo "timed out polling for import endpoints" @@ -33,7 +34,7 @@ done echo "$imports" | tr -d '"' | while read -r import; do echo "checking import: $import" - if ! echo "$endpts" | grep -q "$import" ; then + if ! echo "$endpoints" | grep -q "$import" ; then echo "exported endpoint not found: $import" exit 1 fi diff --git a/main.go b/main.go index bd76d197..cafc203c 100644 --- a/main.go +++ b/main.go @@ -101,7 +101,7 @@ func main() { Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("ServiceExport"), Scheme: mgr.GetScheme(), - Cloudmap: serviceDiscoveryClient, + CloudMap: serviceDiscoveryClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ServiceExport") os.Exit(1) diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index c926b431..3eca59c0 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -37,7 +37,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { mockSDClient := cloudmap.NewMockServiceDiscoveryClient(mockController) // The service model in the Cloudmap mockSDClient.EXPECT().ListServices(context.TODO(), test.NsName). - Return([]*model.Service{test.GetTestServiceWithEndpoint1()}, nil) + Return([]*model.Service{test.GetTestServiceWithEndpoint([]*model.Endpoint{test.GetTestEndpoint1()})}, nil) reconciler := getReconciler(t, mockSDClient, fakeClient) diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index e6fe4d9c..74db7571 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -24,8 +24,15 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -36,8 +43,9 @@ import ( ) const ( - K8sVersionAttr = "K8S_CONTROLLER" - serviceExportFinalizer = "multicluster.k8s.aws/service-export-finalizer" + K8sVersionAttr = "K8S_CONTROLLER" + ServiceExportFinalizer = "multicluster.k8s.aws/service-export-finalizer" + EndpointSliceServiceLabel = "kubernetes.io/service-name" ) // ServiceExportReconciler reconciles a ServiceExport object @@ -45,7 +53,7 @@ type ServiceExportReconciler struct { client.Client Log logr.Logger Scheme *runtime.Scheme - Cloudmap cloudmap.ServiceDiscoveryClient + CloudMap cloudmap.ServiceDiscoveryClient } // +kubebuilder:rbac:groups="",resources=services,verbs=get @@ -57,119 +65,148 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques ctx, cancel := context.WithCancel(ctx) defer cancel() - log := r.Log.WithValues("serviceexport", req.NamespacedName) + r.Log.Info("reconciling ServiceExport", "Namespace", req.Namespace, "Name", req.NamespacedName) - svcExport := v1alpha1.ServiceExport{} - if err := r.Client.Get(ctx, req.NamespacedName, &svcExport); err != nil { + serviceExport := v1alpha1.ServiceExport{} + if err := r.Client.Get(ctx, req.NamespacedName, &serviceExport); err != nil { + r.Log.Error(err, "error fetching ServiceExport", + "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) return ctrl.Result{}, client.IgnoreNotFound(err) } - if svcExport.DeletionTimestamp.IsZero() { - return r.handleUpdate(ctx, log, &svcExport) - } else { - return r.handleDelete(ctx, log, &svcExport) + // Mark ServiceExport to be deleted, which is indicated by the deletion timestamp being set. + isServiceExportMarkedForDelete := serviceExport.GetDeletionTimestamp() != nil + + service := v1.Service{} + namespacedName := types.NamespacedName{Namespace: serviceExport.Namespace, Name: serviceExport.Name} + if err := r.Client.Get(ctx, namespacedName, &service); err != nil { + if errors.IsNotFound(err) { + r.Log.Error(err, "no Service found for ServiceExport", + "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) + // Mark ServiceExport to be deleted, if the corresponding Service is not found + isServiceExportMarkedForDelete = true + } else { + r.Log.Error(err, "error fetching service", + "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) + return ctrl.Result{}, err + } + } + + // Check if the service export is marked to be deleted + if isServiceExportMarkedForDelete { + return r.handleDelete(ctx, &serviceExport) } + + return r.handleUpdate(ctx, &serviceExport, &service) } -func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, log logr.Logger, svcExport *v1alpha1.ServiceExport) (ctrl.Result, error) { - // add finalizer if not present - if !controllerutil.ContainsFinalizer(svcExport, serviceExportFinalizer) { - controllerutil.AddFinalizer(svcExport, serviceExportFinalizer) - if err := r.Update(ctx, svcExport); err != nil { +func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExport *v1alpha1.ServiceExport, service *v1.Service) (ctrl.Result, error) { + + // Add the finalizer to the service export if not present, ensures the ServiceExport won't be deleted + if !controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { + controllerutil.AddFinalizer(serviceExport, ServiceExportFinalizer) + if err := r.Update(ctx, serviceExport); err != nil { + r.Log.Error(err, "error adding finalizer", + "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) return ctrl.Result{}, err } } - svc := v1.Service{} - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: svcExport.Namespace, Name: svcExport.Name}, &svc); err != nil { - log.Error(err, "no service found for ServiceExport", - "Namespace", svcExport.GetNamespace(), "Name", svcExport.Name) - return ctrl.Result{}, client.IgnoreNotFound(err) - } - - endpoints, err := r.extractEndpoints(ctx, &svc) + r.Log.Info("updating Cloud Map service", "namespace", service.Namespace, "name", service.Name) + cmService, err := r.createOrGetCloudMapService(ctx, service) if err != nil { + r.Log.Error(err, "error fetching service from Cloud Map", + "namespace", service.Namespace, "name", service.Name) return ctrl.Result{}, err } - changes := model.Changes{ - Create: endpoints, - } - - log.Info("updating Cloud Map service", "namespace", svc.Namespace, "name", svc.Name) - srv, err := r.Cloudmap.GetService(ctx, svc.Namespace, svc.Name) + endpoints, err := r.extractEndpoints(ctx, service) if err != nil { - log.Error(err, "error when fetching service from Cloud Map API", "namespace", svc.Namespace, "name", svc.Name) + r.Log.Error(err, "error extracting endpoints", + "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) return ctrl.Result{}, err } - if srv == nil { - if err := r.Cloudmap.CreateService(ctx, svc.Namespace, svc.Name); err != nil { - log.Error(err, "error when creating new service in Cloud Map", "namespace", svc.Namespace, "name", svc.Name) - return ctrl.Result{}, err - } - } else { - // compute diff between Cloud Map and K8s and apply changes - plan := model.Plan{ - Current: srv.Endpoints, - Desired: endpoints, - } - changes = plan.CalculateChanges() - } - createRequired := len(changes.Create) > 0 - updateRequired := len(changes.Update) > 0 - deleteRequired := len(changes.Delete) > 0 + // Compute diff between Cloud Map and K8s endpoints, and apply changes + plan := model.Plan{ + Current: cmService.Endpoints, + Desired: endpoints, + } + changes := plan.CalculateChanges() - if createRequired || updateRequired { + if changes.HasUpdates() { // merge creates and updates (Cloud Map RegisterEndpoints can handle both) - upserts := append(changes.Create, changes.Update...) + upserts := changes.Create + upserts = append(upserts, changes.Update...) - if err := r.Cloudmap.RegisterEndpoints(ctx, svc.Namespace, svc.Name, upserts); err != nil { - log.Error(err, "error when registering endpoints to Cloud Map", - "namespace", svc.Namespace, "name", svc.Name) + if err := r.CloudMap.RegisterEndpoints(ctx, service.Namespace, service.Name, upserts); err != nil { + r.Log.Error(err, "error registering endpoints to Cloud Map", + "namespace", service.Namespace, "name", service.Name) return ctrl.Result{}, err } } - if deleteRequired { - if err := r.Cloudmap.DeleteEndpoints(ctx, svc.Namespace, svc.Name, changes.Delete); err != nil { - log.Error(err, "error when deleting endpoints from Cloud Map", - "namespace", srv.Namespace, "name", srv.Name) + if changes.HasDeletes() { + if err := r.CloudMap.DeleteEndpoints(ctx, service.Namespace, service.Name, changes.Delete); err != nil { + r.Log.Error(err, "error deleting endpoints from Cloud Map", + "namespace", cmService.Namespace, "name", cmService.Name) return ctrl.Result{}, err } } - if !createRequired && !updateRequired && !deleteRequired { - log.Info("no changes to export", "namespace", svc.Namespace, "name", svc.Name) + if changes.IsNone() { + r.Log.Info("no changes to export to Cloud Map", "namespace", service.Namespace, "name", service.Name) } return ctrl.Result{}, nil } -func (r *ServiceExportReconciler) handleDelete(ctx context.Context, log logr.Logger, svcExport *v1alpha1.ServiceExport) (ctrl.Result, error) { - if controllerutil.ContainsFinalizer(svcExport, serviceExportFinalizer) { +func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context, service *v1.Service) (*model.Service, error) { + cmService, err := r.CloudMap.GetService(ctx, service.Namespace, service.Name) + if err != nil { + return nil, err + } + + if cmService == nil { + if err := r.CloudMap.CreateService(ctx, service.Namespace, service.Name); err != nil { + r.Log.Error(err, "error creating a new service in Cloud Map", + "namespace", service.Namespace, "name", service.Name) + return nil, err + } + if cmService, err = r.CloudMap.GetService(ctx, service.Namespace, service.Name); err != nil { + return nil, err + } + } + + return cmService, nil +} + +func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExport *v1alpha1.ServiceExport) (ctrl.Result, error) { + if controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { - log.Info("removing Cloud Map service", "namespace", svcExport.Namespace, "name", svcExport.Name) + r.Log.Info("removing service export", "namespace", serviceExport.Namespace, "name", serviceExport.Name) - srv, err := r.Cloudmap.GetService(ctx, svcExport.Namespace, svcExport.Name) + cmService, err := r.CloudMap.GetService(ctx, serviceExport.Namespace, serviceExport.Name) if err != nil { - log.Error(err, "error when fetching service from Cloud Map API", - "namespace", svcExport.Namespace, "name", svcExport.Name) + r.Log.Error(err, "error fetching service from Cloud Map", + "namespace", serviceExport.Namespace, "name", serviceExport.Name) return ctrl.Result{}, err } - if srv != nil { - if err := r.Cloudmap.DeleteEndpoints(ctx, srv.Namespace, srv.Name, srv.Endpoints); err != nil { - log.Error(err, "error when deleting endpoints from Cloud Map", - "namespace", srv.Namespace, "name", srv.Name) + if cmService != nil { + if err := r.CloudMap.DeleteEndpoints(ctx, cmService.Namespace, cmService.Name, cmService.Endpoints); err != nil { + r.Log.Error(err, "error deleting endpoints from Cloud Map", + "namespace", cmService.Namespace, "name", cmService.Name) return ctrl.Result{}, err } } - // remove finalizer - controllerutil.RemoveFinalizer(svcExport, serviceExportFinalizer) - if err := r.Update(ctx, svcExport); err != nil { + // Remove finalizer. Once all finalizers have been + // removed, the ServiceExport object will be deleted. + controllerutil.RemoveFinalizer(serviceExport, ServiceExportFinalizer) + if err := r.Update(ctx, serviceExport); err != nil { return ctrl.Result{}, err } + } return ctrl.Result{}, nil @@ -223,5 +260,57 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. func (r *ServiceExportReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.ServiceExport{}). + // Watch for the changes to the EndpointSlice object. This object is bound to be + // updated when Service or Deployment are updated. There is also a filtering logic + // to enqueue those EndpointSlice event which have corresponding ServiceExport + Watches( + &source.Kind{Type: &discovery.EndpointSlice{}}, + handler.EnqueueRequestsFromMapFunc(r.endpointSliceEventHandler()), + builder.WithPredicates(r.endpointSliceFilter()), + ). Complete(r) } + +func (r *ServiceExportReconciler) endpointSliceEventHandler() handler.MapFunc { + return func(object client.Object) []reconcile.Request { + labels := object.GetLabels() + serviceName := labels[EndpointSliceServiceLabel] + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: serviceName, + Namespace: object.GetNamespace(), + }}, + } + } +} + +func (r *ServiceExportReconciler) endpointSliceFilter() predicate.Funcs { + return predicate.Funcs{ + GenericFunc: func(e event.GenericEvent) bool { + return r.doesEndpointSliceHaveServiceExport(e.Object) + }, + CreateFunc: func(e event.CreateEvent) bool { + return r.doesEndpointSliceHaveServiceExport(e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return r.doesEndpointSliceHaveServiceExport(e.ObjectNew) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return r.doesEndpointSliceHaveServiceExport(e.Object) + }, + } +} + +func (r *ServiceExportReconciler) doesEndpointSliceHaveServiceExport(object client.Object) bool { + labels := object.GetLabels() + serviceName := labels[EndpointSliceServiceLabel] + ns := types.NamespacedName{ + Name: serviceName, + Namespace: object.GetNamespace(), + } + svcExport := v1alpha1.ServiceExport{} + if err := r.Client.Get(context.TODO(), ns, &svcExport); err != nil { + return false + } + return true +} diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index badbec31..81e2c070 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -2,15 +2,15 @@ package controllers import ( "context" - cloudmapmock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" - "gotest.tools/assert" + + "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,28 +24,35 @@ import ( ) func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { + // create a fake controller client and add some objects + fakeClient := fake.NewClientBuilder(). + WithScheme(getServiceExportScheme()). + WithObjects(testServiceObj(), testServiceExportObj()). + WithLists(testEndpointSliceObj()). + Build() + + // create a mock cloudmap service discovery client mockController := gomock.NewController(t) defer mockController.Finish() - expectedService := model.Service{ - Namespace: "my-namespace", - Name: "exported-service", - Endpoints: []*model.Endpoint{test.GetTestEndpoint1()}, - } - - cloudmapMock := cloudmapmock.NewMockServiceDiscoveryClient(mockController) + mock := cloudmap.NewMockServiceDiscoveryClient(mockController) // expected interactions with the Cloud Map client - cloudmapMock.EXPECT().GetService(gomock.Any(), expectedService.Namespace, expectedService.Name).Return(nil, nil) - cloudmapMock.EXPECT().CreateService(gomock.Any(), expectedService.Namespace, expectedService.Name).Return(nil).Times(1) - cloudmapMock.EXPECT().RegisterEndpoints(gomock.Any(), expectedService.Namespace, expectedService.Name, - expectedService.Endpoints).Return(nil).Times(1) - - reconciler := setupServiceExportReconciler(t, cloudmapMock) + // The first get call is expected to return nil, then second call after the creation of service is + // supposed to return the value + first := mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName).Return(nil, nil) + second := mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). + Return(&model.Service{Namespace: test.NsName, Name: test.SvcName}, nil) + gomock.InOrder(first, second) + mock.EXPECT().CreateService(gomock.Any(), test.NsName, test.SvcName).Return(nil).Times(1) + mock.EXPECT().RegisterEndpoints(gomock.Any(), test.NsName, test.SvcName, + []*model.Endpoint{test.GetTestEndpoint1()}).Return(nil).Times(1) + + reconciler := getServiceExportReconciler(t, mock, fakeClient) request := ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: "my-namespace", - Name: "exported-service", + Namespace: test.NsName, + Name: test.SvcName, }, } @@ -55,38 +62,41 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { return } assert.Equal(t, ctrl.Result{}, got, "Result should be empty") + + serviceExport := &v1alpha1.ServiceExport{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceExport) + assert.NoError(t, err) + assert.Contains(t, serviceExport.Finalizers, ServiceExportFinalizer, "Finalizer added to the service export") } -func TestServiceExportReconciler_Reconcile_ExistingServiceNewEndpoint(t *testing.T) { +func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { + // create a fake controller client and add some objects + fakeClient := fake.NewClientBuilder(). + WithScheme(getServiceExportScheme()). + WithObjects(testServiceObj(), testServiceExportObj()). + WithLists(testEndpointSliceObj()). + Build() + mockController := gomock.NewController(t) defer mockController.Finish() - emptyService := model.Service{ - Namespace: "my-namespace", - Name: "exported-service", - } - - expectedService := model.Service{ - Namespace: "my-namespace", - Name: "exported-service", - Endpoints: []*model.Endpoint{test.GetTestEndpoint1()}, - } + mock := cloudmap.NewMockServiceDiscoveryClient(mockController) - cloudmapMock := cloudmapmock.NewMockServiceDiscoveryClient(mockController) - - // expected interactions with the Cloud Map client - cloudmapMock.EXPECT().GetService(gomock.Any(), gomock.Any(), gomock.Any()).Return(&emptyService, nil) - cloudmapMock.EXPECT().RegisterEndpoints(gomock.Any(), expectedService.Namespace, expectedService.Name, - expectedService.Endpoints).Return(nil).Times(1) + // GetService from Cloudmap returns endpoint1 and endpoint2 + mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). + Return(test.GetTestService(), nil) + // call to delete the endpoint not present in the k8s cluster + mock.EXPECT().DeleteEndpoints(gomock.Any(), test.NsName, test.SvcName, + []*model.Endpoint{test.GetTestEndpoint2()}).Return(nil).Times(1) request := ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: "my-namespace", - Name: "exported-service", + Namespace: test.NsName, + Name: test.SvcName, }, } - reconciler := setupServiceExportReconciler(t, cloudmapMock) + reconciler := getServiceExportReconciler(t, mock, fakeClient) got, err := reconciler.Reconcile(context.Background(), request) if err != nil { @@ -94,34 +104,103 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceNewEndpoint(t *testing return } assert.Equal(t, ctrl.Result{}, got, "Result should be empty") + + serviceExport := &v1alpha1.ServiceExport{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceExport) + assert.NoError(t, err) + assert.Contains(t, serviceExport.Finalizers, ServiceExportFinalizer, "Finalizer added to the service export") } -func setupServiceExportReconciler(t *testing.T, cloudmapMock cloudmap.ServiceDiscoveryClient) *ServiceExportReconciler { - k8sClient := setupK8sClient() +func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { + // create a fake controller client and add some objects + serviceExportObj := testServiceExportObj() + // Add finalizer string to the service + serviceExportObj.Finalizers = []string{ServiceExportFinalizer} + fakeClient := fake.NewClientBuilder(). + WithScheme(getServiceExportScheme()). + WithObjects(serviceExportObj). + WithLists(testEndpointSliceObj()). + Build() + + mockController := gomock.NewController(t) + defer mockController.Finish() + + mock := cloudmap.NewMockServiceDiscoveryClient(mockController) + + // GetService from Cloudmap returns endpoint1 and endpoint2 + mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). + Return(test.GetTestService(), nil) + // call to delete the endpoint in the cloudmap + mock.EXPECT().DeleteEndpoints(gomock.Any(), test.NsName, test.SvcName, + []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}).Return(nil).Times(1) + + request := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: test.NsName, + Name: test.SvcName, + }, + } + + reconciler := getServiceExportReconciler(t, mock, fakeClient) + + got, err := reconciler.Reconcile(context.Background(), request) + assert.NoError(t, err) + assert.Equal(t, ctrl.Result{}, got, "Result should be empty") + + serviceExport := &v1alpha1.ServiceExport{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceExport) + assert.NoError(t, err) + assert.Empty(t, serviceExport.Finalizers, "Finalizer removed from the service export") +} + +func getServiceExportScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + scheme.AddKnownTypes(v1alpha1.GroupVersion, &v1alpha1.ServiceExport{}) + scheme.AddKnownTypes(v1.SchemeGroupVersion, &v1.Service{}) + scheme.AddKnownTypes(discovery.SchemeGroupVersion, &discovery.EndpointSlice{}, &discovery.EndpointSliceList{}) + return scheme +} +func getServiceExportReconciler(t *testing.T, mockClient *cloudmap.MockServiceDiscoveryClient, client client.Client) *ServiceExportReconciler { return &ServiceExportReconciler{ - Client: k8sClient, + Client: client, Log: testing2.TestLogger{T: t}, - Scheme: k8sClient.Scheme(), - Cloudmap: cloudmapMock, + Scheme: client.Scheme(), + CloudMap: mockClient, } } -func setupK8sClient() client.Client { - // ServiceExport object - serviceExport := &v1alpha1.ServiceExport{ +func testEndpointSliceObj() *discovery.EndpointSliceList { + port := int32(test.Port1) + protocol := v1.ProtocolTCP + endpointSlice := &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: "exported-service", - Namespace: "my-namespace", + Namespace: test.NsName, + Name: test.SvcName + "-slice", + Labels: map[string]string{discovery.LabelServiceName: test.SvcName}, }, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{{ + Addresses: []string{test.EndptIp1}, + }}, + Ports: []discovery.EndpointPort{{ + Name: aws.String("http"), + Protocol: &protocol, + Port: &port, + }}, } + endpointSliceList := &discovery.EndpointSliceList{ + Items: []discovery.EndpointSlice{*endpointSlice}, + } + return endpointSliceList +} - // Service object - service := &v1.Service{ +func testServiceObj() *v1.Service { + return &v1.Service{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ - Name: "exported-service", - Namespace: "my-namespace", + Name: test.SvcName, + Namespace: test.NsName, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{{ @@ -133,39 +212,13 @@ func setupK8sClient() client.Client { }, Status: v1.ServiceStatus{}, } +} - // EndpointSlice object - port := int32(test.Port1) - protocol := v1.ProtocolTCP - endpointSlice := &discovery.EndpointSlice{ +func testServiceExportObj() *v1alpha1.ServiceExport { + return &v1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ - Namespace: "my-namespace", - Name: "slice-id", - Labels: map[string]string{discovery.LabelServiceName: "exported-service"}, + Name: test.SvcName, + Namespace: test.NsName, }, - AddressType: discovery.AddressTypeIPv4, - Endpoints: []discovery.Endpoint{{ - Addresses: []string{test.EndptIp1}, - }}, - Ports: []discovery.EndpointPort{{ - Name: aws.String("http"), - Protocol: &protocol, - Port: &port, - }}, - } - endpointSliceList := &discovery.EndpointSliceList{ - Items: []discovery.EndpointSlice{*endpointSlice}, } - - scheme := runtime.NewScheme() - scheme.AddKnownTypes(v1alpha1.GroupVersion, serviceExport) - scheme.AddKnownTypes(v1.SchemeGroupVersion, service) - scheme.AddKnownTypes(discovery.SchemeGroupVersion, endpointSlice) - scheme.AddKnownTypes(discovery.SchemeGroupVersion, endpointSliceList) - - return fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(serviceExport, service). - WithLists(endpointSliceList). - Build() } diff --git a/pkg/model/plan.go b/pkg/model/plan.go index 7ce29d5f..618b4df2 100644 --- a/pkg/model/plan.go +++ b/pkg/model/plan.go @@ -45,3 +45,15 @@ func (p *Plan) CalculateChanges() Changes { return changes } + +func (c *Changes) HasUpdates() bool { + return len(c.Create) > 0 || len(c.Update) > 0 +} + +func (c *Changes) HasDeletes() bool { + return len(c.Delete) > 0 +} + +func (c *Changes) IsNone() bool { + return len(c.Create) == 0 && len(c.Update) == 0 && len(c.Delete) == 0 +} diff --git a/test/test-constants.go b/test/test-constants.go index 8cd87441..f2f628d8 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -54,11 +54,11 @@ func GetTestService() *model.Service { } } -func GetTestServiceWithEndpoint1() *model.Service { +func GetTestServiceWithEndpoint(endpoints []*model.Endpoint) *model.Service { return &model.Service{ Namespace: NsName, Name: SvcName, - Endpoints: []*model.Endpoint{GetTestEndpoint1()}, + Endpoints: endpoints, } } From c3f4dd0398be9d1d1852c87f255c74fa4ca9060c Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 18 Nov 2021 20:59:30 -0800 Subject: [PATCH 043/163] Update endpoint slices during service import reconciliation (#91) --- integration/scripts/run-tests.sh | 10 +- pkg/controllers/cloudmap_controller.go | 160 ++++++++++++++++++------- pkg/controllers/utils.go | 21 ++++ 3 files changed, 141 insertions(+), 50 deletions(-) diff --git a/integration/scripts/run-tests.sh b/integration/scripts/run-tests.sh index 39919ba3..8430bff4 100755 --- a/integration/scripts/run-tests.sh +++ b/integration/scripts/run-tests.sh @@ -42,12 +42,10 @@ if [ "$exit_code" -eq 0 ] ; then go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" exit_code=$? - # TODO: The reconciliation during the import is not implemented yet, uncomment below once done - # - #if [ "$exit_code" -eq 0 ] ; then - # ./integration/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" - # exit_code=$? - #fi + if [ "$exit_code" -eq 0 ] ; then + ./integration/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" + exit_code=$? + fi fi echo "killing controller PID:$CTL_PID" diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 917a5eb6..27bc7c72 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/base32" + "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -24,6 +25,8 @@ const ( // TODO move to configuration syncPeriod = 2 * time.Second + maxEndpointsPerSlice = 100 + // DerivedServiceAnnotation annotates a ServiceImport with derived Service name DerivedServiceAnnotation = "multicluster.k8s.aws/derived-service" @@ -156,7 +159,7 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se return err } - err = r.updateEndpointSlices(ctx, svcImport, svc, derivedService) + err = r.updateEndpointSlices(ctx, svcImport, svc.Endpoints, derivedService) if err != nil { return err } @@ -199,7 +202,7 @@ func (r *CloudMapReconciler) getDerivedService(ctx context.Context, namespace st } func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svc *model.Service, svcImport *v1alpha1.ServiceImport) (*v1.Service, error) { - toCreate := createDerivedServiceStruct(svc, svcImport) + toCreate := createDerivedServiceStruct(svc.Endpoints, svcImport) if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err } @@ -208,25 +211,100 @@ func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svc return r.getDerivedService(ctx, svc.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) } -func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *v1alpha1.ServiceImport, cloudMapService *model.Service, svc *v1.Service) error { +func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *v1alpha1.ServiceImport, desiredEndpoints []*model.Endpoint, svc *v1.Service) error { existingSlicesList := discovery.EndpointSliceList{} - var existingSlices []*discovery.EndpointSlice if err := r.Client.List(ctx, &existingSlicesList, client.InNamespace(svc.Namespace), client.MatchingLabels{discovery.LabelServiceName: svc.Name}); err != nil { return err } - if len(existingSlicesList.Items) == 0 { - // create new endpoint slice - existingSlices = createEndpointSlicesStruct(svcImport, cloudMapService, svc) - for _, slice := range existingSlices { - if err := r.Client.Create(ctx, slice); err != nil { - return err + + desiredPorts := extractEndpointPorts(desiredEndpoints) + matchedEndpoints := make(map[string]*discovery.Endpoint) + endpointsToCreate := make([]discovery.Endpoint, 0) + + // populate map of existing endpoints in slices for lookup efficiency + existingEndpointMap := make(map[string]*discovery.Endpoint) + for _, existingSlice := range existingSlicesList.Items { + for _, existingEndpoint := range existingSlice.Endpoints { + ref := existingEndpoint + existingEndpointMap[ref.Addresses[0]] = &ref + } + } + + // check if all desired endpoints are in an endpoint slice already + for _, desiredEndpoint := range desiredEndpoints { + match, exists := existingEndpointMap[desiredEndpoint.IP] + if exists { + matchedEndpoints[desiredEndpoint.IP] = match + } else { + endpointsToCreate = append(endpointsToCreate, createEndpointForSlice(svc, desiredEndpoint.IP)) + } + } + + // check if all endpoints in slices match a desired endpoint, + for _, existingSlice := range existingSlicesList.Items { + updatedEndpointList := make([]discovery.Endpoint, 0) + for _, existingEndpoint := range existingSlice.Endpoints { + keep, found := matchedEndpoints[existingEndpoint.Addresses[0]] + if found { + updatedEndpointList = append(updatedEndpointList, *keep) + } + } + + endpointSliceNeedsUpdate := len(existingSlice.Endpoints) != len(updatedEndpointList) + + // fill endpoint slice with endpoints to create if necessary and there is sufficient room + for _, endpointToCreate := range endpointsToCreate { + if len(updatedEndpointList) >= maxEndpointsPerSlice { + break + } + endpointSliceNeedsUpdate = true + updatedEndpointList = append(updatedEndpointList, endpointToCreate) + endpointsToCreate = endpointsToCreate[1:] + } + + sliceToUpdate := existingSlice + sliceToUpdate.Endpoints = updatedEndpointList + + // delete empty endpoint slice + if len(updatedEndpointList) == 0 { + r.Logger.Info("deleting EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) + if err := r.Client.Delete(ctx, &sliceToUpdate); err != nil { + return fmt.Errorf("failed to delete EndpointSlice: %w", err) + } + continue + } + + // needsUpdate = true if ports don't match + if !EndpointPortsAreEqualIgnoreOrder(desiredPorts, sliceToUpdate.Ports) { + sliceToUpdate.Ports = desiredPorts + endpointSliceNeedsUpdate = true + } + + if endpointSliceNeedsUpdate { + r.Logger.Info("updating EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) + if err := r.Client.Update(ctx, &sliceToUpdate); err != nil { + return fmt.Errorf("failed to update EndpointSlice: %w", err) } - r.Logger.Info("created EndpointSlice", "namespace", slice.Namespace, "name", slice.Name) } } - // TODO check existing slices match Cloud Map endpoints + slicesToCreate := make([]*discovery.EndpointSlice, 0) + for len(endpointsToCreate) > maxEndpointsPerSlice { + slicesToCreate = append(slicesToCreate, createEndpointSliceStruct(svcImport, svc, endpointsToCreate[0:maxEndpointsPerSlice], desiredPorts)) + endpointsToCreate = endpointsToCreate[maxEndpointsPerSlice:] + } + + if len(endpointsToCreate) != 0 { + slicesToCreate = append(slicesToCreate, createEndpointSliceStruct(svcImport, svc, endpointsToCreate, desiredPorts)) + } + + for _, newSlice := range slicesToCreate { + r.Logger.Info("creating EndpointSlice", "namespace", newSlice.Namespace) + if err := r.Client.Create(ctx, newSlice); err != nil { + return fmt.Errorf("failed to create EndpointSlice: %w", err) + } + } return nil } @@ -238,7 +316,7 @@ func DerivedName(namespace string, name string) string { return "imported-" + strings.ToLower(base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(hash.Sum(nil)))[:10] } -func createDerivedServiceStruct(svc *model.Service, svcImport *v1alpha1.ServiceImport) *v1.Service { +func createDerivedServiceStruct(endpoints []*model.Endpoint, svcImport *v1alpha1.ServiceImport) *v1.Service { ownerRef := metav1.NewControllerRef(svcImport, schema.GroupVersionKind{ Version: svcImport.TypeMeta.APIVersion, Kind: svcImport.TypeMeta.Kind, @@ -252,37 +330,31 @@ func createDerivedServiceStruct(svc *model.Service, svcImport *v1alpha1.ServiceI }, Spec: v1.ServiceSpec{ Type: v1.ServiceTypeClusterIP, - Ports: extractServicePorts(svc), + Ports: extractServicePorts(endpoints), }, } } -func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc *model.Service, svc *v1.Service) []*discovery.EndpointSlice { - slices := make([]*discovery.EndpointSlice, 0) - +func createEndpointForSlice(svc *v1.Service, ip string) discovery.Endpoint { t := true - endpoints := make([]discovery.Endpoint, 0) - for _, ep := range cloudMapSvc.Endpoints { - endpoints = append(endpoints, discovery.Endpoint{ - Addresses: []string{ep.IP}, - Conditions: discovery.EndpointConditions{ - Ready: &t, - }, - TargetRef: &v1.ObjectReference{ - Kind: "Service", - Namespace: svc.Namespace, - Name: svc.Name, - UID: svc.ObjectMeta.UID, - ResourceVersion: svc.ObjectMeta.ResourceVersion, - }, - }) + return discovery.Endpoint{ + Addresses: []string{ip}, + Conditions: discovery.EndpointConditions{ + Ready: &t, + }, + TargetRef: &v1.ObjectReference{ + Kind: "Service", + Namespace: svc.Namespace, + Name: svc.Name, + UID: svc.ObjectMeta.UID, + ResourceVersion: svc.ObjectMeta.ResourceVersion, + }, } +} - // TODO split slices in case there are more than 1000 endpoints - // see https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0752-endpointslices/README.md - - slices = append(slices, &discovery.EndpointSlice{ +func createEndpointSliceStruct(svcImport *v1alpha1.ServiceImport, svc *v1.Service, endpoints []discovery.Endpoint, ports []discovery.EndpointPort) *discovery.EndpointSlice { + return &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ discovery.LabelServiceName: svc.Name, // derived Service name @@ -297,15 +369,13 @@ func createEndpointSlicesStruct(svcImport *v1alpha1.ServiceImport, cloudMapSvc * }, AddressType: discovery.AddressTypeIPv4, Endpoints: endpoints, - Ports: extractEndpointPorts(cloudMapSvc), - }) - - return slices + Ports: ports, + } } -func extractServicePorts(svc *model.Service) []v1.ServicePort { +func extractServicePorts(endpoints []*model.Endpoint) []v1.ServicePort { uniquePorts := make(map[string]model.Port) - for _, ep := range svc.Endpoints { + for _, ep := range endpoints { uniquePorts[ep.ServicePort.GetID()] = ep.ServicePort } @@ -317,9 +387,9 @@ func extractServicePorts(svc *model.Service) []v1.ServicePort { return servicePorts } -func extractEndpointPorts(svc *model.Service) []discovery.EndpointPort { +func extractEndpointPorts(endpoints []*model.Endpoint) []discovery.EndpointPort { uniquePorts := make(map[string]model.Port) - for _, ep := range svc.Endpoints { + for _, ep := range endpoints { uniquePorts[ep.EndpointPort.GetID()] = ep.EndpointPort } @@ -327,6 +397,7 @@ func extractEndpointPorts(svc *model.Service) []discovery.EndpointPort { for _, endpointPort := range uniquePorts { endpointPorts = append(endpointPorts, PortToEndpointPort(endpointPort)) } + return endpointPorts } @@ -356,6 +427,7 @@ func portsEqual(svcImport *v1alpha1.ServiceImport, svc *v1.Service) bool { svcPorts = append(svcPorts, servicePortToServiceImport(p)) } + // TODO: consider order return reflect.DeepEqual(impPorts, svcPorts) } diff --git a/pkg/controllers/utils.go b/pkg/controllers/utils.go index 00341972..57dc13e4 100644 --- a/pkg/controllers/utils.go +++ b/pkg/controllers/utils.go @@ -5,6 +5,7 @@ import ( v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" "k8s.io/apimachinery/pkg/util/intstr" + "reflect" ) func ServicePortToPort(svcPort v1.ServicePort) model.Port { @@ -67,3 +68,23 @@ func stringToProtocol(protocol string) v1.Protocol { return "" } } + +func EndpointPortsAreEqualIgnoreOrder(a, b []discovery.EndpointPort) (equal bool) { + if len(a) != len(b) { + return false + } + + for _, aPort := range a { + match := false + for _, bPort := range b { + if reflect.DeepEqual(aPort, bPort) { + match = true + break + } + } + if !match { + return false + } + } + return true +} From d9ca89e3e57bc93333dd0d8572c5bf7597ef0a94 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Thu, 18 Nov 2021 21:06:55 -0800 Subject: [PATCH 044/163] Add the support for kubectl version <= 1.18 (#92) --- config/controller_install_latest/kustomization.yaml | 2 +- config/controller_install_release/kustomization.yaml | 2 +- config/default/kustomization.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/controller_install_latest/kustomization.yaml b/config/controller_install_latest/kustomization.yaml index 38076ff6..1fbcbf37 100644 --- a/config/controller_install_latest/kustomization.yaml +++ b/config/controller_install_latest/kustomization.yaml @@ -1,4 +1,4 @@ -resources: +bases: - ../default images: diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index 45fec726..c30ae01a 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -1,4 +1,4 @@ -resources: +bases: - ../default images: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index e196a2b7..08158e36 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -12,7 +12,7 @@ namePrefix: cloud-map-mcs- #commonLabels: # someName: someValue -resources: +bases: - ../crd - ../rbac - ../manager From a74d4709e0a44371a91d05a518ad5d570c778daf Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Thu, 18 Nov 2021 21:21:42 -0800 Subject: [PATCH 045/163] Update release tag v0.2.0 (#93) --- config/controller_install_release/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index c30ae01a..f15bef5b 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -4,4 +4,4 @@ bases: images: - name: controller newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s - newTag: v0.1.1 + newTag: v0.2.0 From cca995a25f5eb94e46adefb23d4459e9c87718b2 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Fri, 19 Nov 2021 14:58:56 -0800 Subject: [PATCH 046/163] Change exported endpoint ID delimiters, add -ldflags to go run (#94) --- Makefile | 2 +- pkg/model/types.go | 4 +++- pkg/model/types_test.go | 4 ++-- test/test-constants.go | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 403440cb..287b914b 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ build: manifests generate generate-mocks fmt vet ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go run: manifests generate generate-mocks fmt vet ## Run a controller from your host. - go run ./main.go + go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go docker-build: test ## Build docker image with the manager. docker build $(ARGS) -t ${IMG} . diff --git a/pkg/model/types.go b/pkg/model/types.go index fc781e73..277a5185 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -189,7 +189,9 @@ func (e *Endpoint) String() string { // EndpointIdFromIPAddressAndPort converts an IP address to human-readable identifier. func EndpointIdFromIPAddressAndPort(address string, port Port) string { - return fmt.Sprintf("%s://%s:%d", strings.ToLower(port.Protocol), address, port.Port) + address = strings.Replace(address, ".", "_", -1) + address = strings.Replace(address, ":", "_", -1) + return fmt.Sprintf("%s-%s-%d", strings.ToLower(port.Protocol), address, port.Port) } func ConvertNamespaceType(nsType types.NamespaceType) (namespaceType NamespaceType) { diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index db6eb5ed..a041a079 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -176,13 +176,13 @@ func TestEndpointIdFromIPAddressAndPort(t *testing.T) { }{ { name: "happy case", - address: "192.168.0.1", + address: ip, port: Port{ Name: "http", Port: 80, Protocol: "TCP", }, - want: "tcp://192.168.0.1:80", + want: "tcp-192_168_0_1-80", }, } for _, tt := range tests { diff --git a/test/test-constants.go b/test/test-constants.go index f2f628d8..21738740 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -9,8 +9,8 @@ const ( NsId = "ns-id" SvcName = "svc-name" SvcId = "svc-id" - EndptId1 = "tcp://192.168.0.1:1" - EndptId2 = "tcp://192.168.0.2:2" + EndptId1 = "tcp-192_168_0_1-1" + EndptId2 = "tcp-192_168_0_2-2" EndptIp1 = "192.168.0.1" EndptIp2 = "192.168.0.2" Port1 = 1 From e081acdfee265d0e2df0c8244112cf7c1b8d0b25 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Fri, 19 Nov 2021 15:16:12 -0800 Subject: [PATCH 047/163] Patch release v0.2.1 (#95) --- config/controller_install_release/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index f15bef5b..ede95f93 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -4,4 +4,4 @@ bases: images: - name: controller newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s - newTag: v0.2.0 + newTag: v0.2.1 From b824b726aa27bf04b057ef1a69d6404c8780ed40 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Mon, 22 Nov 2021 14:36:18 -0800 Subject: [PATCH 048/163] Delete CHANGELOG.md (#97) --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 3894120e..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,4 +0,0 @@ -# CHANGELOG - -## v0.1.0 -Initial release to cover basic end-to-end functionality (ServiceExport and ServiceImport support). From 568f1bc4ebab1e27f219496d77c019388fd589e4 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Wed, 24 Nov 2021 14:40:56 -0800 Subject: [PATCH 049/163] Fix clusterole access (#99) --- config/rbac/role.yaml | 5 ++++- pkg/controllers/cloudmap_controller.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b6b86bea..923e655e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: manager-role rules: - apiGroups: @@ -22,6 +21,8 @@ rules: - get - list - watch + - update + - delete - apiGroups: - discovery.k8s.io resources: @@ -31,6 +32,8 @@ rules: - get - list - watch + - update + - delete - apiGroups: - multicluster.x-k8s.io resources: diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 27bc7c72..d6847e90 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -42,8 +42,8 @@ type CloudMapReconciler struct { } // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch -// +kubebuilder:rbac:groups="",resources=services,verbs=create;get;list;watch -// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;create;watch +// +kubebuilder:rbac:groups="",resources=services,verbs=create;get;list;watch;update;delete +// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;create;watch;update;delete // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceimports,verbs=create;get;list;watch;update;patch;delete // Start implements manager.Runnable From ed29d6d546947dec166375ac8073dc785f5b8705 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Thu, 25 Nov 2021 13:10:43 -0800 Subject: [PATCH 050/163] Simplify example-service.yaml (#106) --- samples/example-service.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/example-service.yaml b/samples/example-service.yaml index b204accb..16744d46 100644 --- a/samples/example-service.yaml +++ b/samples/example-service.yaml @@ -7,5 +7,4 @@ spec: selector: app: nginx ports: - - port: 8080 - targetPort: 80 + - port: 80 From 7214c2c458a36c3caaa43312d9dcc63b067d9740 Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Thu, 25 Nov 2021 13:18:11 -0800 Subject: [PATCH 051/163] Minor improvements/cleanup to the README.md and CONTRIBUTING.md (#107) --- CONTRIBUTING.md | 81 +++++++++++++++++++++---------------------------- Makefile | 7 +++-- README.md | 6 ---- 3 files changed, 40 insertions(+), 54 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c86c8c0..2b078c6d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,6 @@ # Contributing Guidelines +* [Architecture Overview](#architecture-overview) * [Getting Started](#getting-started) + [Local Setup](#local-setup) - [Prerequisites](#prerequisites) @@ -9,10 +10,9 @@ - [Run unit tests](#run-unit-tests) - [Cleanup](#cleanup) + [Deploying to a cluster](#deploying-to-a-cluster) - + [Build and push docker image to ECR](#build-and-push-docker-image-to-ecr) - - [Deployment](#deployment) - - [Uninstallation](#uninstallation) -* [Reporting Bugs/Feature Requests](#reporting-bugsfeature-requests) +* [Integration testing](#integration-testing) +* [Build and push docker image to ECR](#build-and-push-docker-image-to-ecr) +* [Reporting Bugs/Feature Requests](#reporting-bugs-feature-requests) * [Contributing via Pull Requests](#contributing-via-pull-requests) * [Finding contributions to work on](#finding-contributions-to-work-on) * [Code of Conduct](#code-of-conduct) @@ -40,7 +40,7 @@ information to effectively respond to your bug report or contribution. In order to build and run locally: -* Make sure to have `kubectl` [installed](https://kubernetes.io/docs/tasks/tools/install-kubectl/), at least version `1.15` or above. +* Make sure to have `kubectl` [installed](https://kubernetes.io/docs/tasks/tools/install-kubectl/), at least version `1.17` or above. * Make sure to have `kind` [installed](https://kind.sigs.k8s.io/docs/user/quick-start/#installation). * Make sure, you have access to AWS Cloud Map. As per exercise below, AWS Cloud Map namespace `example` of the type [HttpNamespace](https://docs.aws.amazon.com/cloud-map/latest/api/API_CreateHttpNamespace.html) will be automatically created. @@ -52,7 +52,7 @@ export AWS_REGION=us-west-2 #### Cluster Setup -Spin up a local Kubernetes cluster using `kind` +Spin up a local Kubernetes cluster using `kind`: ```sh kind create cluster --name my-cluster @@ -60,15 +60,13 @@ kind create cluster --name my-cluster # ... ``` -When completed, set the kubectl context - +When completed, set the kubectl context: ```sh kind export kubeconfig --name my-cluster # Set kubectl context to "kind-my-cluster" ``` -Create `example` namespace in the cluster - +Create `example` namespace in the cluster: ```sh kubectl create namespace example # namespace/example created @@ -76,8 +74,7 @@ kubectl create namespace example #### Run the controller from outside the cluster -To register the custom CRDs (`ServiceImport`, `ServiceExport`) in the cluster and create installers - +To register the custom CRDs (`ServiceImport`, `ServiceExport`) in the cluster and create installers: ```sh make install # ... @@ -86,13 +83,11 @@ make install ``` To run the controller, run the following command. The controller runs in an infinite loop so open another terminal to create CRDs. - ```sh make run ``` -Apply deployment, service and serviceexport configs - +Apply deployment, service and serviceexport configs: ```sh kubectl apply -f samples/example-deployment.yaml # deployment.apps/nginx-deployment created @@ -102,8 +97,7 @@ kubectl apply -f samples/example-serviceexport.yaml # serviceexport.multicluster.x-k8s.io/my-service created ``` -Check running controller if it correctly detects newly created resources - +Check running controller if it correctly detects newly created resources: ``` controllers.ServiceExport updating Cloud Map service {"serviceexport": "example/my-service", "namespace": "example", "name": "my-service"} cloudmap fetching a service {"namespaceName": "example", "serviceName": "my-service"} @@ -112,26 +106,25 @@ cloudmap creating a new service {"namespace": "example", "name": #### Build and deploy to the cluster -Build a `controller:local` (`--no-cache` args can be set to update the existing docker image) - -```shell -make docker-build IMG=controller:local ARGS="--no-cache" +Build local `controller` docker image: +```sh +make docker-build IMG=controller:local # ... # docker build --no-cache -t controller:local . # ... # ``` -Load the controller docker image into the kind cluster `my-cluster` -```shell +Load the controller docker image into the kind cluster `my-cluster`: +```sh kind load docker-image controller:local --name my-cluster # Image: "controller:local" with ID "sha256:xxx" not yet present on node "my-cluster-control-plane", loading... ``` > ⚠ **The controller still needs credentials to interact to AWS SDK.** We are not supporting this configuration out of box. There are multiple ways to achieve this within the cluster. -Finally, create the controller resources in the cluster. -```shell +Finally, create the controller resources in the cluster: +```sh make deploy IMG=controller:local AWS_REGION=us-west-2 # customresourcedefinition.apiextensions.k8s.io/serviceexports.multicluster.x-k8s.io configured # customresourcedefinition.apiextensions.k8s.io/serviceimports.multicluster.x-k8s.io created @@ -139,35 +132,36 @@ make deploy IMG=controller:local AWS_REGION=us-west-2 # deployment.apps/cloud-map-mcs-controller-manager created ``` -Stream the controller logs +Stream the controller logs: ```shell kubectl logs -f -l control-plane=controller-manager -c manager -n cloud-map-mcs-system ``` -#### Run unit tests +To remove the controller from your cluster, run: +```sh +make undeploy +``` -Use command below to run the unit test +#### Run unit tests +Use command below to run the unit test: ```sh make test ``` #### Cleanup -Use the command below to clean all the generated files - +Use the command below to perform cleanup: ```sh make clean ``` -Use the command below to remove the CRDs from the cluster - +Use the command below to remove the CRDs from the cluster: ```sh make uninstall ``` -Use the command below to delete the cluster `my-cluster` - +Use the command below to delete the cluster `my-cluster`: ```sh kind delete cluster --name my-cluster ``` @@ -178,27 +172,22 @@ You must first push a Docker image containing the changes to a Docker repository If you are deploying to cluster using kustomize templates from the `config` directory, you will need to override the image URI away from `ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s` in order to use your own docker images. -### Build and push docker image to ECR +## Integration testing +The end-to-end integration test suite can be run locally to validate controller core functionality. This will provision a local Kind cluster and build and run the AWS Cloud Map MCS Controller for K8s. The test will verify service endpoints sync with AWS Cloud Map. If successful, the suite will then de-provision the local test cluster and delete AWS Cloud Map namespace `aws-cloud-map-mcs-e2e` along with test service and service instance resources: ```sh -make docker-build docker-push IMG=.dkr.ecr..amazonaws.com/ +make integration-suite ``` -#### Deployment - -You must setup the AWS access credentials. Also, set the AWS_REGION environment variable like `export AWS_REGION=us-west-2` - +If integration test suite fails for some reason, you can perform a cleanup: ```sh -# With an IAM user. -make deploy +make integration-cleanup ``` -#### Uninstallation - -To remove the controller from your cluster, run +## Build and push docker image to ECR ```sh -make undeploy +make docker-build docker-push IMG=.dkr.ecr..amazonaws.com/ ``` ## Reporting Bugs/Feature Requests diff --git a/Makefile b/Makefile index 287b914b..8c5e9c3a 100644 --- a/Makefile +++ b/Makefile @@ -87,12 +87,15 @@ run: manifests generate generate-mocks fmt vet ## Run a controller from your hos go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go docker-build: test ## Build docker image with the manager. - docker build $(ARGS) -t ${IMG} . + docker build --no-cache -t ${IMG} . docker-push: ## Push docker image with the manager. - docker push $(ARGS) ${IMG} + docker push ${IMG} +.PHONY: clean clean: + @echo Cleaning... + go clean rm -rf $(MOCKS_DESTINATION) bin/ testbin/ cover.out ##@ Deployment diff --git a/README.md b/README.md index f0c53689..2bf882c8 100644 --- a/README.md +++ b/README.md @@ -98,12 +98,6 @@ To install from `latest` tag run kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" ``` -## Integration testing -The end-to-end integration test suite can be run locally to validate controller core functionality. This will provision a local Kind cluster and build and run the AWS Cloud Map MCS Controller for K8s. The test will verify service endpoints sync with AWS Cloud Map. If successful, the suite will then de-provision the local test cluster and delete AWS Cloud Map namespace `aws-cloud-map-mcs-e2e` along with test service and service instance resources. -```sh -make integration-suite -``` - ## Slack community We have an open Slack community where users may get support with integration, discuss controller functionality and provide input on our feature roadmap. https://awsappmesh.slack.com/#k8s-mcs-controller Join the channel with this [invite](https://join.slack.com/t/awsappmesh/shared_invite/zt-dwgbt85c-Sj_md92__quV8YADKfsQSA). From 30416ac987e3d02f38cbf48273234ff2e0db218e Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Fri, 26 Nov 2021 10:24:15 -0800 Subject: [PATCH 052/163] Improve Logging (#109) * Reduce the noise in the logging by adding a wrapper around the logr.Logger to take the advantage of zapr logging levels. zapr provides development mode which sets to logLevel=Debug, and the production mode to logLevel=Info. By default, production mode is enabled, unless explicit argument to set the development mode. * Another iteration of refactorings to cleanup the init method of the common.Logger --- .github/.codecov.yml | 4 +- Makefile | 2 +- main.go | 39 ++++++++-------- pkg/cloudmap/api.go | 7 ++- pkg/cloudmap/api_test.go | 3 +- pkg/cloudmap/cache.go | 7 ++- pkg/cloudmap/client.go | 9 ++-- pkg/cloudmap/client_test.go | 3 +- pkg/cloudmap/operation_collector.go | 7 ++- pkg/cloudmap/operation_poller.go | 7 ++- pkg/cloudmap/operation_poller_test.go | 3 +- pkg/common/logger.go | 44 +++++++++++++++++++ pkg/controllers/cloudmap_controller.go | 38 ++++++++-------- pkg/controllers/cloudmap_controller_test.go | 3 +- pkg/controllers/serviceexport_controller.go | 12 ++--- .../serviceexport_controller_test.go | 3 +- 16 files changed, 120 insertions(+), 71 deletions(-) create mode 100644 pkg/common/logger.go diff --git a/.github/.codecov.yml b/.github/.codecov.yml index e7ef3b00..4e479a10 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -23,4 +23,6 @@ comment: ignore: - "config/**/*" - "pkg/api/**/*" - - "mocks/**/*" \ No newline at end of file + - "mocks/**/*" + - "integration/scenarios/**/*" + - "pkg/common/logger.go" diff --git a/Makefile b/Makefile index 8c5e9c3a..5a3f1c02 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ build: manifests generate generate-mocks fmt vet ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go run: manifests generate generate-mocks fmt vet ## Run a controller from your host. - go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go + go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go --zap-devel=true docker-build: test ## Build docker image with the manager. docker build --no-cache -t ${IMG} . diff --git a/main.go b/main.go index cafc203c..bee55ede 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ package main import ( "context" "flag" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "os" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" @@ -42,8 +43,8 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + log = ctrl.Log.WithName("main") ) func init() { @@ -62,16 +63,18 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - opts := zap.Options{ - Development: true, - } + + // Add the zap logger flag set to the CLI. The flag set must + // be added before calling flag.Parse(). + opts := zap.Options{} opts.BindFlags(flag.CommandLine) + flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) v := version.GetVersion() - setupLog.Info("starting AWS Cloud Map MCS Controller for K8s", "version", v) + log.Info("starting AWS Cloud Map MCS Controller for K8s", "version", v) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, @@ -82,56 +85,56 @@ func main() { LeaderElectionID: "db692913.x-k8s.io", }) if err != nil { - setupLog.Error(err, "unable to start manager") + log.Error(err, "unable to start manager") os.Exit(1) } - setupLog.Info("configuring AWS session") + log.Info("configuring AWS session") // GO sdk will look for region in order 1) AWS_REGION env var, 2) ~/.aws/config file, 3) EC2 IMDS awsCfg, err := config.LoadDefaultConfig(context.TODO(), config.WithEC2IMDSRegion()) if err != nil || awsCfg.Region == "" { - setupLog.Error(err, "unable to configure AWS session", "AWS_REGION", awsCfg.Region) + log.Error(err, "unable to configure AWS session", "AWS_REGION", awsCfg.Region) os.Exit(1) } - setupLog.Info("Running with AWS region", "AWS_REGION", awsCfg.Region) + log.Info("Running with AWS region", "AWS_REGION", awsCfg.Region) serviceDiscoveryClient := cloudmap.NewDefaultServiceDiscoveryClient(&awsCfg) if err = (&controllers.ServiceExportReconciler{ Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("ServiceExport"), + Log: common.NewLogger("controllers", "ServiceExport"), Scheme: mgr.GetScheme(), CloudMap: serviceDiscoveryClient, }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ServiceExport") + log.Error(err, "unable to create controller", "controller", "ServiceExport") os.Exit(1) } cloudMapReconciler := &controllers.CloudMapReconciler{ Client: mgr.GetClient(), Cloudmap: serviceDiscoveryClient, - Logger: ctrl.Log.WithName("controllers").WithName("CloudMap"), + Log: common.NewLogger("controllers", "Cloudmap"), } if err = mgr.Add(cloudMapReconciler); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "CloudMap") + log.Error(err, "unable to create controller", "controller", "CloudMap") os.Exit(1) } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") + log.Error(err, "unable to set up health check") os.Exit(1) } if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") + log.Error(err, "unable to set up ready check") os.Exit(1) } - setupLog.Info("starting manager") + log.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") + log.Error(err, "problem running manager") os.Exit(1) } } diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index 1198adbb..ce81b27f 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -4,13 +4,12 @@ import ( "context" "errors" "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/wait" - ctrl "sigs.k8s.io/controller-runtime" ) const ( @@ -52,14 +51,14 @@ type ServiceDiscoveryApi interface { } type serviceDiscoveryApi struct { - log logr.Logger + log common.Logger awsFacade AwsFacade } // NewServiceDiscoveryApiFromConfig creates a new AWS Cloud Map API connection manager from an AWS client config. func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { return &serviceDiscoveryApi{ - log: ctrl.Log.WithName("cloudmap"), + log: common.NewLogger("cloudmap"), awsFacade: NewAwsFacadeFromConfig(cfg), } } diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index a36f5980..55a4f53f 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" @@ -319,7 +320,7 @@ func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmap.MockAwsFacade) ServiceDiscoveryApi { return &serviceDiscoveryApi{ - log: testingLogger.TestLogger{T: t}, + log: common.NewLoggerWithLogr(testingLogger.TestLogger{T: t}), awsFacade: awsFacade, } } diff --git a/pkg/cloudmap/cache.go b/pkg/cloudmap/cache.go index 640d157e..b70f1b62 100644 --- a/pkg/cloudmap/cache.go +++ b/pkg/cloudmap/cache.go @@ -3,10 +3,9 @@ package cloudmap import ( "errors" "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" - "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/cache" - ctrl "sigs.k8s.io/controller-runtime" "time" ) @@ -33,7 +32,7 @@ type ServiceDiscoveryClientCache interface { } type sdCache struct { - log logr.Logger + log common.Logger cache *cache.LRUExpireCache config *SdCacheConfig } @@ -46,7 +45,7 @@ type SdCacheConfig struct { func NewServiceDiscoveryClientCache(cacheConfig *SdCacheConfig) ServiceDiscoveryClientCache { return &sdCache{ - log: ctrl.Log.WithName("cloudmap"), + log: common.NewLogger("cloudmap"), cache: cache.NewLRUExpireCache(defaultCacheSize), config: cacheConfig, } diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 6c78675d..4910277f 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -3,10 +3,9 @@ package cloudmap import ( "context" "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/go-logr/logr" - ctrl "sigs.k8s.io/controller-runtime" ) // ServiceDiscoveryClient provides the service endpoint management functionality required by the AWS Cloud Map @@ -29,7 +28,7 @@ type ServiceDiscoveryClient interface { } type serviceDiscoveryClient struct { - log logr.Logger + log common.Logger sdApi ServiceDiscoveryApi cache ServiceDiscoveryClientCache } @@ -38,7 +37,7 @@ type serviceDiscoveryClient struct { // from a given AWS client config. func NewDefaultServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: ctrl.Log.WithName("cloudmap"), + log: common.NewLogger("cloudmap"), sdApi: NewServiceDiscoveryApiFromConfig(cfg), cache: NewDefaultServiceDiscoveryClientCache(), } @@ -46,7 +45,7 @@ func NewDefaultServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCacheConfig) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: ctrl.Log.WithName("cloudmap"), + log: common.NewLogger("cloudmap"), sdApi: NewServiceDiscoveryApiFromConfig(cfg), cache: NewServiceDiscoveryClientCache(cacheConfig), } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 5ab71be0..c31b5b91 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" @@ -410,7 +411,7 @@ func getTestSdClient(t *testing.T) *testSdClient { mockApi := cloudmap.NewMockServiceDiscoveryApi(mockController) return &testSdClient{ client: &serviceDiscoveryClient{ - log: testing2.TestLogger{T: t}, + log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), sdApi: mockApi, cache: mockCache, }, diff --git a/pkg/cloudmap/operation_collector.go b/pkg/cloudmap/operation_collector.go index e38c5257..1e3239e1 100644 --- a/pkg/cloudmap/operation_collector.go +++ b/pkg/cloudmap/operation_collector.go @@ -1,8 +1,7 @@ package cloudmap import ( - "github.com/go-logr/logr" - ctrl "sigs.k8s.io/controller-runtime" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "sync" ) @@ -22,7 +21,7 @@ type OperationCollector interface { } type opCollector struct { - log logr.Logger + log common.Logger opChan chan opResult wg sync.WaitGroup startTime int64 @@ -36,7 +35,7 @@ type opResult struct { func NewOperationCollector() OperationCollector { return &opCollector{ - log: ctrl.Log.WithName("cloudmap"), + log: common.NewLogger("cloudmap"), opChan: make(chan opResult), startTime: Now(), createOpsSuccess: true, diff --git a/pkg/cloudmap/operation_poller.go b/pkg/cloudmap/operation_poller.go index 61467821..3680b30b 100644 --- a/pkg/cloudmap/operation_poller.go +++ b/pkg/cloudmap/operation_poller.go @@ -3,11 +3,10 @@ package cloudmap import ( "context" "errors" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/wait" - ctrl "sigs.k8s.io/controller-runtime" "strconv" "time" ) @@ -29,7 +28,7 @@ type OperationPoller interface { } type operationPoller struct { - log logr.Logger + log common.Logger sdApi ServiceDiscoveryApi timeout time.Duration @@ -41,7 +40,7 @@ type operationPoller struct { func newOperationPoller(sdApi ServiceDiscoveryApi, svcId string, opIds []string, startTime int64) operationPoller { return operationPoller{ - log: ctrl.Log.WithName("cloudmap"), + log: common.NewLogger("cloudmap"), sdApi: sdApi, timeout: defaultOperationPollTimeout, diff --git a/pkg/cloudmap/operation_poller_test.go b/pkg/cloudmap/operation_poller_test.go index ccff3d93..6e8cc56c 100644 --- a/pkg/cloudmap/operation_poller_test.go +++ b/pkg/cloudmap/operation_poller_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" testing2 "github.com/go-logr/logr/testing" @@ -185,7 +186,7 @@ func TestOperationPoller_PollTimeout(t *testing.T) { sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) p := operationPoller{ - log: testing2.TestLogger{T: t}, + log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), sdApi: sdApi, timeout: 2 * time.Millisecond, opIds: []string{test.OpId1, test.OpId2}, diff --git a/pkg/common/logger.go b/pkg/common/logger.go new file mode 100644 index 00000000..a9dfc38c --- /dev/null +++ b/pkg/common/logger.go @@ -0,0 +1,44 @@ +package common + +import ( + "github.com/go-logr/logr" + ctrl "sigs.k8s.io/controller-runtime" +) + +type Logger interface { + Info(msg string, keysAndValues ...interface{}) + Debug(msg string, keysAndValues ...interface{}) + Error(err error, msg string, keysAndValues ...interface{}) +} + +type logger struct { + log logr.Logger +} + +func NewLogger(name string, names ...string) Logger { + l := ctrl.Log.WithName(name) + for _, n := range names { + l = l.WithName(n) + } + return logger{log: l} +} + +func NewLoggerWithLogr(l logr.Logger) Logger { + return logger{log: l} +} + +func (l logger) Info(msg string, keysAndValues ...interface{}) { + l.log.Info(msg, keysAndValues...) +} + +func (l logger) Debug(msg string, keysAndValues ...interface{}) { + l.log.V(1).Info(msg, keysAndValues...) +} + +func (l logger) Error(err error, msg string, keysAndValues ...interface{}) { + l.log.Error(err, msg, keysAndValues...) +} + +func (l logger) WithValues(keysAndValues ...interface{}) Logger { + return logger{log: l.log.WithValues(keysAndValues...)} +} diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index d6847e90..0c209dde 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -7,8 +7,8 @@ import ( "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" - "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" "k8s.io/apimachinery/pkg/api/errors" @@ -36,9 +36,9 @@ const ( // CloudMapReconciler reconciles state of Cloud Map services with local ServiceImport objects type CloudMapReconciler struct { - client.Client + Client client.Client Cloudmap cloudmap.ServiceDiscoveryClient - logr.Logger + Log common.Logger } // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch @@ -53,12 +53,12 @@ func (r *CloudMapReconciler) Start(ctx context.Context) error { for { if err := r.Reconcile(ctx); err != nil { // just log the error and continue running - r.Logger.Error(err, "Cloud Map reconciliation error") + r.Log.Error(err, "Cloud Map reconciliation error") } select { case <-ticker.C: case <-ctx.Done(): - r.Logger.Info("terminating CloudMapReconciler") + r.Log.Info("terminating CloudMapReconciler") return nil } } @@ -69,7 +69,7 @@ func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { namespaces := v1.NamespaceList{} if err := r.Client.List(ctx, &namespaces); err != nil { - r.Logger.Error(err, "unable to list namespaces") + r.Log.Error(err, "unable to list namespaces") return err } @@ -85,7 +85,7 @@ func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { } func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceName string) error { - r.Logger.Info("syncing namespace", "namespace", namespaceName) + r.Log.Debug("syncing namespace", "namespace", namespaceName) desiredServices, err := r.Cloudmap.ListServices(ctx, namespaceName) if err != nil { @@ -94,7 +94,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa serviceImports := v1alpha1.ServiceImportList{} if err := r.Client.List(ctx, &serviceImports, client.InNamespace(namespaceName)); err != nil { - r.Logger.Error(err, "failed to reconcile namespace", "namespace", namespaceName) + r.Log.Error(err, "failed to reconcile namespace", "namespace", namespaceName) return nil } @@ -110,7 +110,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa } if err := r.reconcileService(ctx, svc); err != nil { - r.Logger.Error(err, "error when syncing service", "namespace", svc.Namespace, "name", svc.Name) + r.Log.Error(err, "error when syncing service", "namespace", svc.Namespace, "name", svc.Name) } delete(existingImportsMap, svc.Namespace+"/"+svc.Name) } @@ -118,17 +118,17 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa // delete remaining imports that have not been matched for _, i := range existingImportsMap { if err := r.Client.Delete(ctx, &i); err != nil { - r.Logger.Error(err, "error deleting ServiceImport", "namespace", i.Namespace, "name", i.Name) + r.Log.Error(err, "error deleting ServiceImport", "namespace", i.Namespace, "name", i.Name) continue } - r.Logger.Info("delete ServiceImport", "namespace", i.Namespace, "name", i.Name) + r.Log.Info("delete ServiceImport", "namespace", i.Namespace, "name", i.Name) } return nil } func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Service) error { - r.Logger.Info("syncing service", "namespace", svc.Namespace, "service", svc.Name) + r.Log.Info("syncing service", "namespace", svc.Namespace, "service", svc.Name) svcImport, err := r.getServiceImport(ctx, svc.Namespace, svc.Name) if err != nil { @@ -190,7 +190,7 @@ func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, name if err := r.Client.Create(ctx, imp); err != nil { return nil, err } - r.Logger.Info("created ServiceImport", "namespace", imp.Namespace, "name", imp.Name) + r.Log.Info("created ServiceImport", "namespace", imp.Namespace, "name", imp.Name) return r.getServiceImport(ctx, namespace, name) } @@ -206,7 +206,7 @@ func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svc if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err } - r.Logger.Info("created derived Service", "namespace", toCreate.Namespace, "name", toCreate.Name) + r.Log.Info("created derived Service", "namespace", toCreate.Namespace, "name", toCreate.Name) return r.getDerivedService(ctx, svc.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) } @@ -268,7 +268,7 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport // delete empty endpoint slice if len(updatedEndpointList) == 0 { - r.Logger.Info("deleting EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) + r.Log.Info("deleting EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) if err := r.Client.Delete(ctx, &sliceToUpdate); err != nil { return fmt.Errorf("failed to delete EndpointSlice: %w", err) } @@ -282,7 +282,7 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport } if endpointSliceNeedsUpdate { - r.Logger.Info("updating EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) + r.Log.Info("updating EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) if err := r.Client.Update(ctx, &sliceToUpdate); err != nil { return fmt.Errorf("failed to update EndpointSlice: %w", err) } @@ -300,7 +300,7 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport } for _, newSlice := range slicesToCreate { - r.Logger.Info("creating EndpointSlice", "namespace", newSlice.Namespace) + r.Log.Info("creating EndpointSlice", "namespace", newSlice.Namespace) if err := r.Client.Create(ctx, newSlice); err != nil { return fmt.Errorf("failed to create EndpointSlice: %w", err) } @@ -409,10 +409,10 @@ func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport for _, p := range svc.Spec.Ports { svcImport.Spec.Ports = append(svcImport.Spec.Ports, servicePortToServiceImport(p)) } - if err := r.Update(ctx, svcImport); err != nil { + if err := r.Client.Update(ctx, svcImport); err != nil { return err } - r.Logger.Info("updated ServiceImport", + r.Log.Info("updated ServiceImport", "namespace", svcImport.Namespace, "name", svcImport.Name, "IP", svcImport.Spec.IPs, "ports", svcImport.Spec.Ports) } diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index 3eca59c0..bf3daa98 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -4,6 +4,7 @@ import ( "context" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" testingLogger "github.com/go-logr/logr/testing" @@ -84,6 +85,6 @@ func getReconciler(t *testing.T, mockSDClient *cloudmap.MockServiceDiscoveryClie return &CloudMapReconciler{ Client: client, Cloudmap: mockSDClient, - Logger: testingLogger.TestLogger{T: t}, + Log: common.NewLoggerWithLogr(testingLogger.TestLogger{T: t}), } } diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index 74db7571..689e2e59 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" v1 "k8s.io/api/core/v1" @@ -34,7 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -50,8 +50,8 @@ const ( // ServiceExportReconciler reconciles a ServiceExport object type ServiceExportReconciler struct { - client.Client - Log logr.Logger + Client client.Client + Log common.Logger Scheme *runtime.Scheme CloudMap cloudmap.ServiceDiscoveryClient } @@ -65,7 +65,7 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques ctx, cancel := context.WithCancel(ctx) defer cancel() - r.Log.Info("reconciling ServiceExport", "Namespace", req.Namespace, "Name", req.NamespacedName) + r.Log.Debug("reconciling ServiceExport", "Namespace", req.Namespace, "Name", req.NamespacedName) serviceExport := v1alpha1.ServiceExport{} if err := r.Client.Get(ctx, req.NamespacedName, &serviceExport); err != nil { @@ -105,7 +105,7 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor // Add the finalizer to the service export if not present, ensures the ServiceExport won't be deleted if !controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { controllerutil.AddFinalizer(serviceExport, ServiceExportFinalizer) - if err := r.Update(ctx, serviceExport); err != nil { + if err := r.Client.Update(ctx, serviceExport); err != nil { r.Log.Error(err, "error adding finalizer", "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) return ctrl.Result{}, err @@ -203,7 +203,7 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor // Remove finalizer. Once all finalizers have been // removed, the ServiceExport object will be deleted. controllerutil.RemoveFinalizer(serviceExport, ServiceExportFinalizer) - if err := r.Update(ctx, serviceExport); err != nil { + if err := r.Client.Update(ctx, serviceExport); err != nil { return ctrl.Result{}, err } diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index 81e2c070..4c544dd9 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -4,6 +4,7 @@ import ( "context" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" @@ -164,7 +165,7 @@ func getServiceExportScheme() *runtime.Scheme { func getServiceExportReconciler(t *testing.T, mockClient *cloudmap.MockServiceDiscoveryClient, client client.Client) *ServiceExportReconciler { return &ServiceExportReconciler{ Client: client, - Log: testing2.TestLogger{T: t}, + Log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), Scheme: client.Scheme(), CloudMap: mockClient, } From cd633bc369b805ec154ca752c3fca3340a04942e Mon Sep 17 00:00:00 2001 From: Ben Du <5668844+bendu@users.noreply.github.com> Date: Tue, 30 Nov 2021 17:33:57 -0800 Subject: [PATCH 053/163] Track v1 release for configure aws creds (#112) --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 9000bbe2..512635e2 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -12,7 +12,7 @@ jobs: id-token: write steps: - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@master + uses: aws-actions/configure-aws-credentials@v1 with: aws-region: us-west-2 role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} From 14b1c369aa5c99bd518485bfc3569e89ddbcb64b Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Wed, 1 Dec 2021 19:27:10 -0800 Subject: [PATCH 054/163] Integrate golangci-lint (#114) --- .golangci.yaml | 29 ++++++ Makefile | 23 ++++- config/rbac/role.yaml | 9 +- go.mod | 2 +- go.sum | 15 ++- integration/janitor/api.go | 1 + integration/janitor/api_test.go | 9 +- integration/janitor/aws_facade.go | 1 + integration/janitor/janitor.go | 3 +- integration/janitor/janitor_test.go | 3 +- integration/janitor/runner/main.go | 3 +- integration/scenarios/export_service.go | 7 +- integration/scenarios/runner/main.go | 3 +- main.go | 3 +- pkg/cloudmap/api.go | 5 +- pkg/cloudmap/api_test.go | 7 +- pkg/cloudmap/aws_facade.go | 1 + pkg/cloudmap/cache.go | 3 +- pkg/cloudmap/cache_test.go | 31 +++++-- pkg/cloudmap/client.go | 4 +- pkg/cloudmap/client_test.go | 92 +++++++------------ pkg/cloudmap/operation_collector.go | 3 +- pkg/cloudmap/operation_collector_test.go | 26 ++++-- pkg/cloudmap/operation_poller.go | 5 +- pkg/cloudmap/operation_poller_test.go | 7 +- pkg/controllers/cloudmap_controller.go | 8 +- pkg/controllers/cloudmap_controller_test.go | 5 +- pkg/controllers/serviceexport_controller.go | 7 +- .../serviceexport_controller_test.go | 4 +- pkg/controllers/suite_test.go | 2 +- pkg/controllers/utils.go | 3 +- pkg/controllers/utils_test.go | 5 +- pkg/model/types.go | 7 +- pkg/model/types_test.go | 3 +- 34 files changed, 206 insertions(+), 133 deletions(-) create mode 100644 .golangci.yaml diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 00000000..d0dd0006 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,29 @@ +linters-settings: + errcheck: + check-type-assertions: true + goconst: + min-len: 2 + min-occurrences: 3 + govet: + check-shadowing: true + nolintlint: + require-explanation: true + require-specific: true + +linters: + enable: + - dupl + - goconst + - gocritic + - gofmt + - goimports + - misspell + - whitespace + +run: + issues-exit-code: 1 + concurrency: 4 + skip: + - .*_mock.go + - mocks/ + - pkg/api/ diff --git a/Makefile b/Makefile index 5a3f1c02..20ca1e4d 100644 --- a/Makefile +++ b/Makefile @@ -54,11 +54,29 @@ fmt: ## Run go fmt against code. vet: ## Run go vet against code. go vet ./... +mod: + go mod download + +tidy: + go mod tidy + +golangci-lint: ## Download golangci-lint + @mkdir -p $(shell pwd)/bin + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.43.0 + +.PHONY: lint +lint: golangci-lint ## Run linter + $(shell pwd)/bin/golangci-lint run + +.PHONY: goimports +goimports: ## run goimports updating files in place + goimports -w . + ENVTEST_ASSETS_DIR=$(shell pwd)/testbin test: manifests generate generate-mocks fmt vet test-setup ## Run tests. source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -covermode=atomic -test-setup: # setup test environment +test-setup: ## setup test environment mkdir -p ${ENVTEST_ASSETS_DIR} test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR) @@ -80,7 +98,8 @@ integration-cleanup: kind ## Cleanup integration test resources in Cloud Map an ##@ Build -build: manifests generate generate-mocks fmt vet ## Build manager binary. +.DEFAULT: build +build: manifests generate generate-mocks fmt vet lint ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go run: manifests generate generate-mocks fmt vet ## Run a controller from your host. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 923e655e..921e22ba 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -3,6 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + creationTimestamp: null name: manager-role rules: - apiGroups: @@ -18,22 +19,22 @@ rules: - services verbs: - create + - delete - get - list - - watch - update - - delete + - watch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - create + - delete - get - list - - watch - update - - delete + - watch - apiGroups: - multicluster.x-k8s.io resources: diff --git a/go.mod b/go.mod index be498bf5..b9aa25c4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 github.com/stretchr/testify v1.6.1 - gotest.tools v2.2.0+incompatible + golang.org/x/tools v0.1.7 // indirect k8s.io/api v0.20.2 k8s.io/apimachinery v0.20.2 k8s.io/client-go v0.20.2 diff --git a/go.sum b/go.sum index 0e7d1fae..4319e422 100644 --- a/go.sum +++ b/go.sum @@ -394,6 +394,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -488,8 +489,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -541,16 +543,19 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -598,8 +603,9 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -689,7 +695,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/integration/janitor/api.go b/integration/janitor/api.go index d74561d7..6e00a400 100644 --- a/integration/janitor/api.go +++ b/integration/janitor/api.go @@ -2,6 +2,7 @@ package janitor import ( "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" diff --git a/integration/janitor/api_test.go b/integration/janitor/api_test.go index f0d84fbc..8f19b886 100644 --- a/integration/janitor/api_test.go +++ b/integration/janitor/api_test.go @@ -2,13 +2,14 @@ package janitor import ( "context" + "testing" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "testing" ) func TestNewServiceDiscoveryJanitorApiFromConfig(t *testing.T) { @@ -20,7 +21,7 @@ func TestServiceDiscoveryJanitorApi_DeleteNamespace_HappyCase(t *testing.T) { defer mockController.Finish() mocksdk := janitor.NewMockSdkJanitorFacade(mockController) - jApi := getJanitorApi(t, mocksdk) + jApi := getJanitorApi(mocksdk) mocksdk.EXPECT().DeleteNamespace(context.TODO(), &sd.DeleteNamespaceInput{Id: aws.String(test.NsId)}). Return(&sd.DeleteNamespaceOutput{OperationId: aws.String(test.OpId1)}, nil) @@ -35,7 +36,7 @@ func TestServiceDiscoveryJanitorApi_DeleteService_HappyCase(t *testing.T) { defer mockController.Finish() mocksdk := janitor.NewMockSdkJanitorFacade(mockController) - jApi := getJanitorApi(t, mocksdk) + jApi := getJanitorApi(mocksdk) mocksdk.EXPECT().DeleteService(context.TODO(), &sd.DeleteServiceInput{Id: aws.String(test.SvcId)}). Return(&sd.DeleteServiceOutput{}, nil) @@ -44,7 +45,7 @@ func TestServiceDiscoveryJanitorApi_DeleteService_HappyCase(t *testing.T) { assert.Nil(t, err, "No error for happy case") } -func getJanitorApi(t *testing.T, sdk *janitor.MockSdkJanitorFacade) ServiceDiscoveryJanitorApi { +func getJanitorApi(sdk *janitor.MockSdkJanitorFacade) ServiceDiscoveryJanitorApi { return &serviceDiscoveryJanitorApi{ janitorFacade: sdk, } diff --git a/integration/janitor/aws_facade.go b/integration/janitor/aws_facade.go index 00752cd0..5a2d3bce 100644 --- a/integration/janitor/aws_facade.go +++ b/integration/janitor/aws_facade.go @@ -2,6 +2,7 @@ package janitor import ( "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index 9a6252ce..0ac9a416 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -3,10 +3,11 @@ package janitor import ( "context" "fmt" + "os" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "os" ) // CloudMapJanitor handles AWS Cloud Map resource cleanup during integration tests. diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index 99c1b782..6c10b91b 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -2,6 +2,8 @@ package janitor import ( "context" + "testing" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -9,7 +11,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "testing" ) type testJanitor struct { diff --git a/integration/janitor/runner/main.go b/integration/janitor/runner/main.go index 66e3aa08..9711bfed 100644 --- a/integration/janitor/runner/main.go +++ b/integration/janitor/runner/main.go @@ -3,8 +3,9 @@ package main import ( "context" "fmt" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/janitor" "os" + + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/janitor" ) func main() { diff --git a/integration/scenarios/export_service.go b/integration/scenarios/export_service.go index 75527cdd..183e9901 100644 --- a/integration/scenarios/export_service.go +++ b/integration/scenarios/export_service.go @@ -3,14 +3,15 @@ package scenarios import ( "context" "fmt" + "strconv" + "strings" + "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" "k8s.io/apimachinery/pkg/util/wait" - "strconv" - "strings" - "time" ) const ( diff --git a/integration/scenarios/runner/main.go b/integration/scenarios/runner/main.go index 6fe27428..e2448773 100644 --- a/integration/scenarios/runner/main.go +++ b/integration/scenarios/runner/main.go @@ -3,10 +3,11 @@ package main import ( "context" "fmt" + "os" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/scenarios" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "os" ) func main() { diff --git a/main.go b/main.go index bee55ede..6480f8e4 100644 --- a/main.go +++ b/main.go @@ -19,9 +19,10 @@ package main import ( "context" "flag" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "os" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" "github.com/aws/aws-sdk-go-v2/config" diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index ce81b27f..f2caaf46 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" @@ -87,7 +88,6 @@ func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) (namespace } func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) (svcs []*model.Resource, err error) { - filter := types.ServiceFilter{ Name: types.ServiceFilterNameNamespaceId, Values: []string{nsId}, @@ -128,7 +128,7 @@ func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName } func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters []types.OperationFilter) (opStatusMap map[string]types.OperationStatus, err error) { - opStatusMap = make(map[string]types.OperationStatus, 0) + opStatusMap = make(map[string]types.OperationStatus) pages := sd.NewListOperationsPaginator(sdApi.awsFacade, &sd.ListOperationsInput{ Filters: opFilters, @@ -231,7 +231,6 @@ func (sdApi *serviceDiscoveryApi) DeregisterInstance(ctx context.Context, svcId } return aws.ToString(deregResp.OperationId), err - } func (sdApi *serviceDiscoveryApi) PollNamespaceOperation(ctx context.Context, opId string) (nsId string, err error) { diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 55a4f53f..72f8003b 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "testing" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -14,7 +16,6 @@ import ( testingLogger "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "testing" ) func TestNewServiceDiscoveryApi(t *testing.T) { @@ -132,7 +133,6 @@ func TestServiceDiscoveryApi_ListOperations_HappyCase(t *testing.T) { assert.Nil(t, err, "No error for happy case") assert.True(t, len(ops) == 1) assert.Equal(t, ops[test.OpId1], types.OperationStatusSuccess) - } func TestServiceDiscoveryApi_GetOperation_HappyCase(t *testing.T) { @@ -164,7 +164,6 @@ func TestServiceDiscoveryApi_CreateHttNamespace_HappyCase(t *testing.T) { opId, err := sdApi.CreateHttpNamespace(context.TODO(), test.NsName) assert.Nil(t, err, "No error for happy case") assert.Equal(t, test.OpId1, opId) - } func TestServiceDiscoveryApi_CreateService_CreateForHttpNamespace(t *testing.T) { @@ -233,7 +232,7 @@ func TestServiceDiscoveryApi_CreateService_ThrowError(t *testing.T) { retSvcId, err := sdApi.CreateService(context.TODO(), *test.GetTestHttpNamespace(), svcName) assert.Empty(t, retSvcId) - assert.Equal(t, "dummy error", fmt.Sprint(err), "Got error") + assert.Equal(t, "dummy error", err.Error(), "Got error") } func TestServiceDiscoveryApi_RegisterInstance_HappyCase(t *testing.T) { diff --git a/pkg/cloudmap/aws_facade.go b/pkg/cloudmap/aws_facade.go index fb174d43..5edf4312 100644 --- a/pkg/cloudmap/aws_facade.go +++ b/pkg/cloudmap/aws_facade.go @@ -2,6 +2,7 @@ package cloudmap import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" ) diff --git a/pkg/cloudmap/cache.go b/pkg/cloudmap/cache.go index b70f1b62..ccad3691 100644 --- a/pkg/cloudmap/cache.go +++ b/pkg/cloudmap/cache.go @@ -3,10 +3,11 @@ package cloudmap import ( "errors" "fmt" + "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "k8s.io/apimachinery/pkg/util/cache" - "time" ) const ( diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index 03473544..2f448ecf 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -1,19 +1,23 @@ package cloudmap import ( + "testing" + "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/stretchr/testify/assert" - "testing" - "time" ) func TestNewServiceDiscoveryClientCache(t *testing.T) { - sdc := NewServiceDiscoveryClientCache(&SdCacheConfig{ + sdc, ok := NewServiceDiscoveryClientCache(&SdCacheConfig{ NsTTL: 3 * time.Second, SvcTTL: 3 * time.Second, EndptTTL: 3 * time.Second, }).(*sdCache) + if !ok { + t.Fatalf("failed to create cache") + } assert.Equal(t, 3*time.Second, sdc.config.NsTTL) assert.Equal(t, 3*time.Second, sdc.config.SvcTTL) @@ -21,7 +25,10 @@ func TestNewServiceDiscoveryClientCache(t *testing.T) { } func TestNewDefaultServiceDiscoveryClientCache(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) + sdc, ok := NewDefaultServiceDiscoveryClientCache().(*sdCache) + if !ok { + t.Fatalf("failed to create cache") + } assert.Equal(t, defaultNsTTL, sdc.config.NsTTL) assert.Equal(t, defaultSvcTTL, sdc.config.SvcTTL) @@ -55,7 +62,10 @@ func TestServiceDiscoveryClientCacheGetNamespace_Nil(t *testing.T) { } func TestServiceDiscoveryClientCacheGetNamespace_Corrupt(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) + sdc, ok := NewDefaultServiceDiscoveryClientCache().(*sdCache) + if !ok { + t.Fatalf("failed to create cache") + } sdc.cache.Add(sdc.buildNsKey(test.NsName), &model.Resource{}, time.Minute) ns, found := sdc.GetNamespace(test.NsName) @@ -81,8 +91,10 @@ func TestServiceDiscoveryClientCacheGetServiceId_NotFound(t *testing.T) { } func TestServiceDiscoveryClientCacheGetServiceId_Corrupt(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) - + sdc, ok := NewDefaultServiceDiscoveryClientCache().(*sdCache) + if !ok { + t.Fatalf("failed to create cache") + } sdc.cache.Add(sdc.buildSvcKey(test.NsName, test.SvcName), &model.Resource{}, time.Minute) svcId, found := sdc.GetServiceId(test.NsName, test.SvcName) assert.False(t, found) @@ -107,7 +119,10 @@ func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { } func TestServiceDiscoveryClientCacheGetEndpoints_Corrupt(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClientCache().(*sdCache) + sdc, ok := NewDefaultServiceDiscoveryClientCache().(*sdCache) + if !ok { + t.Fatalf("failed to create cache") + } sdc.cache.Add(sdc.buildEndptsKey(test.NsName, test.SvcName), &model.Resource{}, time.Minute) endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 4910277f..e10b891c 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -3,6 +3,7 @@ package cloudmap import ( "context" "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" @@ -220,7 +221,8 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s } func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, nsName string, svcName string) (endpts []*model.Endpoint, err error) { - if endpts, found := sdc.cache.GetEndpoints(nsName, svcName); found { + endpts, found := sdc.cache.GetEndpoints(nsName, svcName) + if found { return endpts, nil } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index c31b5b91..9f1711bc 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -3,6 +3,8 @@ package cloudmap import ( "context" "errors" + "testing" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -12,7 +14,6 @@ import ( testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "testing" ) type testSdClient struct { @@ -42,34 +43,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return(nil, false) tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). - Return([]types.HttpInstanceSummary{ - { - InstanceId: aws.String(test.EndptId1), - Attributes: map[string]string{ - model.EndpointIpv4Attr: test.EndptIp1, - model.EndpointPortAttr: test.PortStr1, - model.EndpointPortNameAttr: test.PortName1, - model.EndpointProtocolAttr: test.Protocol1, - model.ServicePortNameAttr: test.PortName1, - model.ServicePortAttr: test.ServicePortStr1, - model.ServiceProtocolAttr: test.Protocol1, - model.ServiceTargetPortAttr: test.PortStr1, - }, - }, - { - InstanceId: aws.String(test.EndptId2), - Attributes: map[string]string{ - model.EndpointIpv4Attr: test.EndptIp2, - model.EndpointPortAttr: test.PortStr2, - model.EndpointPortNameAttr: test.PortName2, - model.EndpointProtocolAttr: test.Protocol2, - model.ServicePortNameAttr: test.PortName2, - model.ServicePortAttr: test.ServicePortStr2, - model.ServiceProtocolAttr: test.Protocol2, - model.ServiceTargetPortAttr: test.PortStr2, - }, - }, - }, nil) + Return(testHttpInstanceSummary(), nil) tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) @@ -292,34 +266,7 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return([]*model.Endpoint{}, false) tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). - Return([]types.HttpInstanceSummary{ - { - InstanceId: aws.String(test.EndptId1), - Attributes: map[string]string{ - model.EndpointIpv4Attr: test.EndptIp1, - model.EndpointPortAttr: test.PortStr1, - model.EndpointPortNameAttr: test.PortName1, - model.EndpointProtocolAttr: test.Protocol1, - model.ServicePortNameAttr: test.PortName1, - model.ServicePortAttr: test.ServicePortStr1, - model.ServiceProtocolAttr: test.Protocol1, - model.ServiceTargetPortAttr: test.PortStr1, - }, - }, - { - InstanceId: aws.String(test.EndptId2), - Attributes: map[string]string{ - model.EndpointIpv4Attr: test.EndptIp2, - model.EndpointPortAttr: test.PortStr2, - model.EndpointPortNameAttr: test.PortName2, - model.EndpointProtocolAttr: test.Protocol2, - model.ServicePortNameAttr: test.PortName2, - model.ServicePortAttr: test.ServicePortStr2, - model.ServiceProtocolAttr: test.Protocol2, - model.ServiceTargetPortAttr: test.PortStr2, - }, - }, - }, nil) + Return(testHttpInstanceSummary(), nil) tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) @@ -420,3 +367,34 @@ func getTestSdClient(t *testing.T) *testSdClient { close: func() { mockController.Finish() }, } } + +func testHttpInstanceSummary() []types.HttpInstanceSummary { + return []types.HttpInstanceSummary{ + { + InstanceId: aws.String(test.EndptId1), + Attributes: map[string]string{ + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, + }, + }, + { + InstanceId: aws.String(test.EndptId2), + Attributes: map[string]string{ + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, + }, + }, + } +} diff --git a/pkg/cloudmap/operation_collector.go b/pkg/cloudmap/operation_collector.go index 1e3239e1..93293be6 100644 --- a/pkg/cloudmap/operation_collector.go +++ b/pkg/cloudmap/operation_collector.go @@ -1,8 +1,9 @@ package cloudmap import ( - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "sync" + + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" ) // OperationCollector collects a list of operation IDs asynchronously with thread safety. diff --git a/pkg/cloudmap/operation_collector_test.go b/pkg/cloudmap/operation_collector_test.go index 1b8d6bab..cb6e6f5f 100644 --- a/pkg/cloudmap/operation_collector_test.go +++ b/pkg/cloudmap/operation_collector_test.go @@ -2,27 +2,33 @@ package cloudmap import ( "errors" - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/stretchr/testify/assert" +) + +const ( + op1 = "one" + op2 = "two" ) func TestOpCollector_HappyCase(t *testing.T) { oc := NewOperationCollector() - oc.Add(func() (opId string, err error) { return "one", nil }) - oc.Add(func() (opId string, err error) { return "two", nil }) + oc.Add(func() (opId string, err error) { return op1, nil }) + oc.Add(func() (opId string, err error) { return op2, nil }) result := oc.Collect() assert.True(t, oc.IsAllOperationsCreated()) assert.Equal(t, 2, len(result)) - assert.Contains(t, result, "one") - assert.Contains(t, result, "two") + assert.Contains(t, result, op1) + assert.Contains(t, result, op2) } func TestOpCollector_AllFail(t *testing.T) { oc := NewOperationCollector() - oc.Add(func() (opId string, err error) { return "one", errors.New("fail one") }) - oc.Add(func() (opId string, err error) { return "two", errors.New("fail two") }) + oc.Add(func() (opId string, err error) { return op1, errors.New("fail one") }) + oc.Add(func() (opId string, err error) { return op2, errors.New("fail two") }) result := oc.Collect() assert.False(t, oc.IsAllOperationsCreated()) @@ -31,12 +37,12 @@ func TestOpCollector_AllFail(t *testing.T) { func TestOpCollector_MixedSuccess(t *testing.T) { oc := NewOperationCollector() - oc.Add(func() (opId string, err error) { return "one", errors.New("fail one") }) - oc.Add(func() (opId string, err error) { return "two", nil }) + oc.Add(func() (opId string, err error) { return op1, errors.New("fail one") }) + oc.Add(func() (opId string, err error) { return op2, nil }) result := oc.Collect() assert.False(t, oc.IsAllOperationsCreated()) - assert.Equal(t, []string{"two"}, result) + assert.Equal(t, []string{op2}, result) } func TestOpCollector_GetStartTime(t *testing.T) { diff --git a/pkg/cloudmap/operation_poller.go b/pkg/cloudmap/operation_poller.go index 3680b30b..aacf4a3c 100644 --- a/pkg/cloudmap/operation_poller.go +++ b/pkg/cloudmap/operation_poller.go @@ -3,12 +3,13 @@ package cloudmap import ( "context" "errors" + "strconv" + "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "k8s.io/apimachinery/pkg/util/wait" - "strconv" - "time" ) const ( diff --git a/pkg/cloudmap/operation_poller_test.go b/pkg/cloudmap/operation_poller_test.go index 6e8cc56c..f96adf7b 100644 --- a/pkg/cloudmap/operation_poller_test.go +++ b/pkg/cloudmap/operation_poller_test.go @@ -3,6 +3,10 @@ package cloudmap import ( "context" "errors" + "strconv" + "testing" + "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -10,9 +14,6 @@ import ( testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "strconv" - "testing" - "time" ) func TestOperationPoller_HappyCases(t *testing.T) { diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 0c209dde..4a0e7d1a 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -5,6 +5,10 @@ import ( "crypto/sha256" "encoding/base32" "fmt" + "reflect" + "strings" + "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" @@ -15,10 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" - "strings" - "time" ) const ( @@ -66,7 +67,6 @@ func (r *CloudMapReconciler) Start(ctx context.Context) error { // Reconcile triggers a single reconciliation round func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { - namespaces := v1.NamespaceList{} if err := r.Client.List(ctx, &namespaces); err != nil { r.Log.Error(err, "unable to list namespaces") diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index bf3daa98..3ed9b506 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -2,6 +2,9 @@ package controllers import ( "context" + "strings" + "testing" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" @@ -18,8 +21,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "strings" - "testing" ) func TestCloudMapReconciler_Reconcile(t *testing.T) { diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index 689e2e59..ecc81a55 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -101,7 +102,6 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques } func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExport *v1alpha1.ServiceExport, service *v1.Service) (ctrl.Result, error) { - // Add the finalizer to the service export if not present, ensures the ServiceExport won't be deleted if !controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { controllerutil.AddFinalizer(serviceExport, ServiceExportFinalizer) @@ -168,7 +168,8 @@ func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context } if cmService == nil { - if err := r.CloudMap.CreateService(ctx, service.Namespace, service.Name); err != nil { + err = r.CloudMap.CreateService(ctx, service.Namespace, service.Name) + if err != nil { r.Log.Error(err, "error creating a new service in Cloud Map", "namespace", service.Namespace, "name", service.Name) return nil, err @@ -183,7 +184,6 @@ func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExport *v1alpha1.ServiceExport) (ctrl.Result, error) { if controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { - r.Log.Info("removing service export", "namespace", serviceExport.Namespace, "name", serviceExport.Name) cmService, err := r.CloudMap.GetService(ctx, serviceExport.Namespace, serviceExport.Name) @@ -206,7 +206,6 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor if err := r.Client.Update(ctx, serviceExport); err != nil { return ctrl.Result{}, err } - } return ctrl.Result{}, nil diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index 4c544dd9..49611bc3 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -2,6 +2,7 @@ package controllers import ( "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" @@ -11,6 +12,8 @@ import ( testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" + "testing" + "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" @@ -21,7 +24,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" ) func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index e4127751..45192419 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -37,7 +37,7 @@ import ( // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. -var cfg *rest.Config +var _ *rest.Config var k8sClient client.Client var testEnv *envtest.Environment diff --git a/pkg/controllers/utils.go b/pkg/controllers/utils.go index 57dc13e4..cb2e8d68 100644 --- a/pkg/controllers/utils.go +++ b/pkg/controllers/utils.go @@ -1,11 +1,12 @@ package controllers import ( + "reflect" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" "k8s.io/apimachinery/pkg/util/intstr" - "reflect" ) func ServicePortToPort(svcPort v1.ServicePort) model.Port { diff --git a/pkg/controllers/utils_test.go b/pkg/controllers/utils_test.go index ec970b31..093509fd 100644 --- a/pkg/controllers/utils_test.go +++ b/pkg/controllers/utils_test.go @@ -1,12 +1,13 @@ package controllers import ( + "reflect" + "testing" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" v1 "k8s.io/api/core/v1" "k8s.io/api/discovery/v1beta1" "k8s.io/apimachinery/pkg/util/intstr" - "reflect" - "testing" ) func TestServicePortToPort(t *testing.T) { diff --git a/pkg/model/types.go b/pkg/model/types.go index 277a5185..25c8a3ae 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -3,10 +3,11 @@ package model import ( "encoding/json" "fmt" - "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "reflect" "strconv" "strings" + + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" ) // Resource encapsulates a ID/name pair. @@ -189,8 +190,8 @@ func (e *Endpoint) String() string { // EndpointIdFromIPAddressAndPort converts an IP address to human-readable identifier. func EndpointIdFromIPAddressAndPort(address string, port Port) string { - address = strings.Replace(address, ".", "_", -1) - address = strings.Replace(address, ":", "_", -1) + address = strings.ReplaceAll(address, ".", "_") + address = strings.ReplaceAll(address, ":", "_") return fmt.Sprintf("%s-%s-%d", strings.ToLower(port.Protocol), address, port.Port) } diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index a041a079..019308fa 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -1,9 +1,10 @@ package model import ( - "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "reflect" "testing" + + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" ) var instId = "my-instance" From e860c5c891685c96fcc2efd02abe2fc72c46e6de Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Fri, 3 Dec 2021 11:29:44 -0800 Subject: [PATCH 055/163] Set the Service as ServiceExport's OwnerReference (#115) --- Makefile | 2 +- go.mod | 1 - go.sum | 14 ++---- pkg/common/logger.go | 2 +- pkg/controllers/cloudmap_controller.go | 3 +- pkg/controllers/serviceexport_controller.go | 53 ++++++++++++++------- 6 files changed, 44 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 20ca1e4d..456bfb96 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,7 @@ build: manifests generate generate-mocks fmt vet lint ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go run: manifests generate generate-mocks fmt vet ## Run a controller from your host. - go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go --zap-devel=true + go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go --zap-devel=true $(ARGS) docker-build: test ## Build docker image with the manager. docker build --no-cache -t ${IMG} . diff --git a/go.mod b/go.mod index b9aa25c4..ff8a208b 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 github.com/stretchr/testify v1.6.1 - golang.org/x/tools v0.1.7 // indirect k8s.io/api v0.20.2 k8s.io/apimachinery v0.20.2 k8s.io/client-go v0.20.2 diff --git a/go.sum b/go.sum index 4319e422..01799e08 100644 --- a/go.sum +++ b/go.sum @@ -394,7 +394,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -489,9 +488,8 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -543,19 +541,16 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -603,9 +598,8 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/common/logger.go b/pkg/common/logger.go index a9dfc38c..194f011f 100644 --- a/pkg/common/logger.go +++ b/pkg/common/logger.go @@ -28,7 +28,7 @@ func NewLoggerWithLogr(l logr.Logger) Logger { } func (l logger) Info(msg string, keysAndValues ...interface{}) { - l.log.Info(msg, keysAndValues...) + l.log.V(0).Info(msg, keysAndValues...) } func (l logger) Debug(msg string, keysAndValues ...interface{}) { diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 4a0e7d1a..038e2941 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -89,6 +89,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa desiredServices, err := r.Cloudmap.ListServices(ctx, namespaceName) if err != nil { + r.Log.Error(err, "failed to fetch the list Services") return err } @@ -128,7 +129,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa } func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Service) error { - r.Log.Info("syncing service", "namespace", svc.Namespace, "service", svc.Name) + r.Log.Debug("syncing service", "namespace", svc.Namespace, "service", svc.Name) svcImport, err := r.getServiceImport(ctx, svc.Namespace, svc.Name) if err != nil { diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index ecc81a55..9a9765e7 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -40,7 +40,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - v1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" ) const ( @@ -66,13 +66,20 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques ctx, cancel := context.WithCancel(ctx) defer cancel() - r.Log.Debug("reconciling ServiceExport", "Namespace", req.Namespace, "Name", req.NamespacedName) + namespace := req.Namespace + name := req.NamespacedName + r.Log.Debug("reconciling ServiceExport", "Namespace", namespace, "Name", name) serviceExport := v1alpha1.ServiceExport{} - if err := r.Client.Get(ctx, req.NamespacedName, &serviceExport); err != nil { - r.Log.Error(err, "error fetching ServiceExport", - "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) - return ctrl.Result{}, client.IgnoreNotFound(err) + if err := r.Client.Get(ctx, name, &serviceExport); err != nil { + if errors.IsNotFound(err) { + r.Log.Debug("no ServiceExport found", + "Namespace", namespace, "Name", name) + } else { + r.Log.Error(err, "error fetching ServiceExport", + "Namespace", namespace, "Name", name) + } + return ctrl.Result{}, nil } // Mark ServiceExport to be deleted, which is indicated by the deletion timestamp being set. @@ -82,14 +89,14 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques namespacedName := types.NamespacedName{Namespace: serviceExport.Namespace, Name: serviceExport.Name} if err := r.Client.Get(ctx, namespacedName, &service); err != nil { if errors.IsNotFound(err) { - r.Log.Error(err, "no Service found for ServiceExport", + r.Log.Info("no Service found, deleting the ServiceExport", "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) // Mark ServiceExport to be deleted, if the corresponding Service is not found isServiceExportMarkedForDelete = true } else { - r.Log.Error(err, "error fetching service", + r.Log.Error(err, "error fetching Service", "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) - return ctrl.Result{}, err + return ctrl.Result{}, nil } } @@ -112,18 +119,30 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor } } + if len(serviceExport.GetOwnerReferences()) == 0 { + err := controllerutil.SetControllerReference(service, serviceExport, r.Scheme) + if err == nil { + err = r.Client.Update(ctx, serviceExport) + } + if err != nil { + r.Log.Error(err, "error setting Service as an owner of the ServiceExport", + "namespace", service.Namespace, "name", service.Name) + return ctrl.Result{}, err + } + } + r.Log.Info("updating Cloud Map service", "namespace", service.Namespace, "name", service.Name) cmService, err := r.createOrGetCloudMapService(ctx, service) if err != nil { - r.Log.Error(err, "error fetching service from Cloud Map", + r.Log.Error(err, "error fetching Service from Cloud Map", "namespace", service.Namespace, "name", service.Name) return ctrl.Result{}, err } endpoints, err := r.extractEndpoints(ctx, service) if err != nil { - r.Log.Error(err, "error extracting endpoints", - "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) + r.Log.Error(err, "error extracting Endpoints", + "namespace", serviceExport.Namespace, "name", serviceExport.Name) return ctrl.Result{}, err } @@ -140,7 +159,7 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor upserts = append(upserts, changes.Update...) if err := r.CloudMap.RegisterEndpoints(ctx, service.Namespace, service.Name, upserts); err != nil { - r.Log.Error(err, "error registering endpoints to Cloud Map", + r.Log.Error(err, "error registering Endpoints to Cloud Map", "namespace", service.Namespace, "name", service.Name) return ctrl.Result{}, err } @@ -148,7 +167,7 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor if changes.HasDeletes() { if err := r.CloudMap.DeleteEndpoints(ctx, service.Namespace, service.Name, changes.Delete); err != nil { - r.Log.Error(err, "error deleting endpoints from Cloud Map", + r.Log.Error(err, "error deleting Endpoints from Cloud Map", "namespace", cmService.Namespace, "name", cmService.Name) return ctrl.Result{}, err } @@ -170,7 +189,7 @@ func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context if cmService == nil { err = r.CloudMap.CreateService(ctx, service.Namespace, service.Name) if err != nil { - r.Log.Error(err, "error creating a new service in Cloud Map", + r.Log.Error(err, "error creating a new Service in Cloud Map", "namespace", service.Namespace, "name", service.Name) return nil, err } @@ -188,13 +207,13 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor cmService, err := r.CloudMap.GetService(ctx, serviceExport.Namespace, serviceExport.Name) if err != nil { - r.Log.Error(err, "error fetching service from Cloud Map", + r.Log.Error(err, "error fetching Service from Cloud Map", "namespace", serviceExport.Namespace, "name", serviceExport.Name) return ctrl.Result{}, err } if cmService != nil { if err := r.CloudMap.DeleteEndpoints(ctx, cmService.Namespace, cmService.Name, cmService.Endpoints); err != nil { - r.Log.Error(err, "error deleting endpoints from Cloud Map", + r.Log.Error(err, "error deleting Endpoints from Cloud Map", "namespace", cmService.Namespace, "name", cmService.Name) return ctrl.Result{}, err } From 24f199614053e7d0106f20a4b3d1e48d195b9fb3 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Fri, 3 Dec 2021 14:24:26 -0800 Subject: [PATCH 056/163] Improve import updates with slice change plans and port deltas (#116) --- .golangci.yaml | 6 + integration/scenarios/export_service.go | 5 +- pkg/controllers/cloudmap_controller.go | 305 +++++------------ pkg/controllers/cloudmap_controller_test.go | 12 +- pkg/controllers/controllers_common_test.go | 85 +++++ pkg/controllers/endpointslice_plan.go | 174 ++++++++++ pkg/controllers/endpointslice_plan_test.go | 218 +++++++++++++ .../serviceexport_controller_test.go | 74 +---- pkg/controllers/utils.go | 195 +++++++++-- pkg/controllers/utils_test.go | 307 ++++++++++++++++++ pkg/model/types.go | 12 +- test/test-constants.go | 15 +- 12 files changed, 1072 insertions(+), 336 deletions(-) create mode 100644 pkg/controllers/controllers_common_test.go create mode 100644 pkg/controllers/endpointslice_plan.go create mode 100644 pkg/controllers/endpointslice_plan_test.go diff --git a/.golangci.yaml b/.golangci.yaml index d0dd0006..376f5fee 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -20,6 +20,12 @@ linters: - misspell - whitespace +issues: + exclude-rules: + - path: _test\.go # disable some linters on test files + linters: + - dupl + run: issues-exit-code: 1 concurrency: 4 diff --git a/integration/scenarios/export_service.go b/integration/scenarios/export_service.go index 183e9901..51826cf1 100644 --- a/integration/scenarios/export_service.go +++ b/integration/scenarios/export_service.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" ) @@ -46,7 +47,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po for _, ip := range strings.Split(ips, ",") { endpointPort := model.Port{ Port: int32(port), - Protocol: model.TCPProtocol, + Protocol: string(v1.ProtocolTCP), } endpts = append(endpts, &model.Endpoint{ Id: model.EndpointIdFromIPAddressAndPort(ip, endpointPort), @@ -54,7 +55,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po ServicePort: model.Port{ Port: int32(servicePort), TargetPort: portStr, - Protocol: model.TCPProtocol, + Protocol: string(v1.ProtocolTCP), }, EndpointPort: endpointPort, Attributes: make(map[string]string), diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 038e2941..4235b40f 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -2,11 +2,7 @@ package controllers import ( "context" - "crypto/sha256" - "encoding/base32" "fmt" - "reflect" - "strings" "time" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" @@ -16,8 +12,6 @@ import ( v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -25,14 +19,6 @@ import ( const ( // TODO move to configuration syncPeriod = 2 * time.Second - - maxEndpointsPerSlice = 100 - - // DerivedServiceAnnotation annotates a ServiceImport with derived Service name - DerivedServiceAnnotation = "multicluster.k8s.aws/derived-service" - - // LabelServiceImportName indicates the name of the multi-cluster service that an EndpointSlice belongs to. - LabelServiceImportName = "multicluster.kubernetes.io/service-name" ) // CloudMapReconciler reconciles state of Cloud Map services with local ServiceImport objects @@ -94,7 +80,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa } serviceImports := v1alpha1.ServiceImportList{} - if err := r.Client.List(ctx, &serviceImports, client.InNamespace(namespaceName)); err != nil { + if err = r.Client.List(ctx, &serviceImports, client.InNamespace(namespaceName)); err != nil { r.Log.Error(err, "failed to reconcile namespace", "namespace", namespaceName) return nil } @@ -110,7 +96,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa continue } - if err := r.reconcileService(ctx, svc); err != nil { + if err = r.reconcileService(ctx, svc); err != nil { r.Log.Error(err, "error when syncing service", "namespace", svc.Namespace, "name", svc.Name) } delete(existingImportsMap, svc.Namespace+"/"+svc.Name) @@ -118,7 +104,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa // delete remaining imports that have not been matched for _, i := range existingImportsMap { - if err := r.Client.Delete(ctx, &i); err != nil { + if err = r.Client.Delete(ctx, &i); err != nil { r.Log.Error(err, "error deleting ServiceImport", "namespace", i.Namespace, "name", i.Name) continue } @@ -131,6 +117,8 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Service) error { r.Log.Debug("syncing service", "namespace", svc.Namespace, "service", svc.Name) + importedSvcPorts := ExtractServicePorts(svc.Endpoints) + svcImport, err := r.getServiceImport(ctx, svc.Namespace, svc.Name) if err != nil { if !errors.IsNotFound(err) { @@ -138,7 +126,7 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se } // create ServiceImport if it doesn't exist - if svcImport, err = r.createAndGetServiceImport(ctx, svc.Namespace, svc.Name); err != nil { + if svcImport, err = r.createAndGetServiceImport(ctx, svc.Namespace, svc.Name, importedSvcPorts); err != nil { return err } } @@ -150,22 +138,22 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se } // create derived Service if it doesn't exist - if derivedService, err = r.createAndGetDerivedService(ctx, svc, svcImport); err != nil { + if derivedService, err = r.createAndGetDerivedService(ctx, svcImport, importedSvcPorts); err != nil { return err } } - // update ServiceImport to match IP and port of previously created service - if err = r.updateServiceImport(ctx, svcImport, derivedService); err != nil { + // update service import to match derived service cluster IP and imported ports if necessary + if err = r.updateServiceImport(ctx, svcImport, derivedService, importedSvcPorts); err != nil { return err } - err = r.updateEndpointSlices(ctx, svcImport, svc.Endpoints, derivedService) - if err != nil { + // update derived service ports to match imported ports if necessary + if err = r.updateDerivedService(ctx, derivedService, importedSvcPorts); err != nil { return err } - return nil + return r.updateEndpointSlices(ctx, svcImport, svc.Endpoints, derivedService) } func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace string, name string) (*v1alpha1.ServiceImport, error) { @@ -174,24 +162,12 @@ func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace str return existingServiceImport, err } -func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, namespace string, name string) (*v1alpha1.ServiceImport, error) { - imp := &v1alpha1.ServiceImport{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(namespace, name)}, - }, - Spec: v1alpha1.ServiceImportSpec{ - IPs: []string{}, - Type: v1alpha1.ClusterSetIP, - Ports: []v1alpha1.ServicePort{}, - }, - } - - if err := r.Client.Create(ctx, imp); err != nil { +func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, namespace string, name string, servicePorts []*model.Port) (*v1alpha1.ServiceImport, error) { + toCreate := CreateServiceImportStruct(namespace, name, servicePorts) + if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err } - r.Log.Info("created ServiceImport", "namespace", imp.Namespace, "name", imp.Name) + r.Log.Info("created ServiceImport", "namespace", namespace, "name", name) return r.getServiceImport(ctx, namespace, name) } @@ -202,14 +178,14 @@ func (r *CloudMapReconciler) getDerivedService(ctx context.Context, namespace st return existingService, err } -func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svc *model.Service, svcImport *v1alpha1.ServiceImport) (*v1.Service, error) { - toCreate := createDerivedServiceStruct(svc.Endpoints, svcImport) +func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svcImport *v1alpha1.ServiceImport, svcPorts []*model.Port) (*v1.Service, error) { + toCreate := CreateDerivedServiceStruct(svcImport, svcPorts) if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err } r.Log.Info("created derived Service", "namespace", toCreate.Namespace, "name", toCreate.Name) - return r.getDerivedService(ctx, svc.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) + return r.getDerivedService(ctx, toCreate.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) } func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *v1alpha1.ServiceImport, desiredEndpoints []*model.Endpoint, svc *v1.Service) error { @@ -219,90 +195,37 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport return err } - desiredPorts := extractEndpointPorts(desiredEndpoints) - matchedEndpoints := make(map[string]*discovery.Endpoint) - endpointsToCreate := make([]discovery.Endpoint, 0) - - // populate map of existing endpoints in slices for lookup efficiency - existingEndpointMap := make(map[string]*discovery.Endpoint) + existingSlices := make([]*discovery.EndpointSlice, 0) for _, existingSlice := range existingSlicesList.Items { - for _, existingEndpoint := range existingSlice.Endpoints { - ref := existingEndpoint - existingEndpointMap[ref.Addresses[0]] = &ref - } + existingSlices = append(existingSlices, &existingSlice) } - // check if all desired endpoints are in an endpoint slice already - for _, desiredEndpoint := range desiredEndpoints { - match, exists := existingEndpointMap[desiredEndpoint.IP] - if exists { - matchedEndpoints[desiredEndpoint.IP] = match - } else { - endpointsToCreate = append(endpointsToCreate, createEndpointForSlice(svc, desiredEndpoint.IP)) - } + plan := EndpointSlicePlan{ + Current: existingSlices, + Desired: desiredEndpoints, + Service: svc, + ServiceImportName: svcImport.Name, } - // check if all endpoints in slices match a desired endpoint, - for _, existingSlice := range existingSlicesList.Items { - updatedEndpointList := make([]discovery.Endpoint, 0) - for _, existingEndpoint := range existingSlice.Endpoints { - keep, found := matchedEndpoints[existingEndpoint.Addresses[0]] - if found { - updatedEndpointList = append(updatedEndpointList, *keep) - } - } - - endpointSliceNeedsUpdate := len(existingSlice.Endpoints) != len(updatedEndpointList) + changes := plan.CalculateChanges() - // fill endpoint slice with endpoints to create if necessary and there is sufficient room - for _, endpointToCreate := range endpointsToCreate { - if len(updatedEndpointList) >= maxEndpointsPerSlice { - break - } - endpointSliceNeedsUpdate = true - updatedEndpointList = append(updatedEndpointList, endpointToCreate) - endpointsToCreate = endpointsToCreate[1:] - } - - sliceToUpdate := existingSlice - sliceToUpdate.Endpoints = updatedEndpointList - - // delete empty endpoint slice - if len(updatedEndpointList) == 0 { - r.Log.Info("deleting EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) - if err := r.Client.Delete(ctx, &sliceToUpdate); err != nil { - return fmt.Errorf("failed to delete EndpointSlice: %w", err) - } - continue - } - - // needsUpdate = true if ports don't match - if !EndpointPortsAreEqualIgnoreOrder(desiredPorts, sliceToUpdate.Ports) { - sliceToUpdate.Ports = desiredPorts - endpointSliceNeedsUpdate = true - } - - if endpointSliceNeedsUpdate { - r.Log.Info("updating EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) - if err := r.Client.Update(ctx, &sliceToUpdate); err != nil { - return fmt.Errorf("failed to update EndpointSlice: %w", err) - } + for _, sliceToUpdate := range changes.Update { + r.Log.Debug("updating EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) + if err := r.Client.Update(ctx, sliceToUpdate); err != nil { + return fmt.Errorf("failed to update EndpointSlice: %w", err) } } - slicesToCreate := make([]*discovery.EndpointSlice, 0) - for len(endpointsToCreate) > maxEndpointsPerSlice { - slicesToCreate = append(slicesToCreate, createEndpointSliceStruct(svcImport, svc, endpointsToCreate[0:maxEndpointsPerSlice], desiredPorts)) - endpointsToCreate = endpointsToCreate[maxEndpointsPerSlice:] - } - - if len(endpointsToCreate) != 0 { - slicesToCreate = append(slicesToCreate, createEndpointSliceStruct(svcImport, svc, endpointsToCreate, desiredPorts)) + for _, sliceToDelete := range changes.Delete { + r.Log.Debug("deleting EndpointSlice", "namespace", sliceToDelete.Namespace, "name", sliceToDelete.Name) + if err := r.Client.Delete(ctx, sliceToDelete); err != nil { + return fmt.Errorf("failed to delete EndpointSlice: %w", err) + } } - for _, newSlice := range slicesToCreate { - r.Log.Info("creating EndpointSlice", "namespace", newSlice.Namespace) - if err := r.Client.Create(ctx, newSlice); err != nil { + for _, sliceToCreate := range changes.Create { + r.Log.Debug("creating EndpointSlice", "namespace", sliceToCreate.Namespace) + if err := r.Client.Create(ctx, sliceToCreate); err != nil { return fmt.Errorf("failed to create EndpointSlice: %w", err) } } @@ -310,106 +233,41 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport return nil } -// DerivedName computes the "placeholder" name for the imported service -func DerivedName(namespace string, name string) string { - hash := sha256.New() - hash.Write([]byte(namespace + name)) - return "imported-" + strings.ToLower(base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(hash.Sum(nil)))[:10] -} - -func createDerivedServiceStruct(endpoints []*model.Endpoint, svcImport *v1alpha1.ServiceImport) *v1.Service { - ownerRef := metav1.NewControllerRef(svcImport, schema.GroupVersionKind{ - Version: svcImport.TypeMeta.APIVersion, - Kind: svcImport.TypeMeta.Kind, - }) - - return &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: svcImport.Namespace, - Name: svcImport.Annotations[DerivedServiceAnnotation], - OwnerReferences: []metav1.OwnerReference{*ownerRef}, - }, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeClusterIP, - Ports: extractServicePorts(endpoints), - }, - } -} - -func createEndpointForSlice(svc *v1.Service, ip string) discovery.Endpoint { - t := true - - return discovery.Endpoint{ - Addresses: []string{ip}, - Conditions: discovery.EndpointConditions{ - Ready: &t, - }, - TargetRef: &v1.ObjectReference{ - Kind: "Service", - Namespace: svc.Namespace, - Name: svc.Name, - UID: svc.ObjectMeta.UID, - ResourceVersion: svc.ObjectMeta.ResourceVersion, - }, - } -} - -func createEndpointSliceStruct(svcImport *v1alpha1.ServiceImport, svc *v1.Service, endpoints []discovery.Endpoint, ports []discovery.EndpointPort) *discovery.EndpointSlice { - return &discovery.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - discovery.LabelServiceName: svc.Name, // derived Service name - LabelServiceImportName: svcImport.Name, // original ServiceImport name - }, - GenerateName: svc.Name + "-", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(svc, schema.GroupVersionKind{ - Version: svc.TypeMeta.APIVersion, - Kind: svc.TypeMeta.Kind, - })}, - Namespace: svc.Namespace, - }, - AddressType: discovery.AddressTypeIPv4, - Endpoints: endpoints, - Ports: ports, - } -} - -func extractServicePorts(endpoints []*model.Endpoint) []v1.ServicePort { - uniquePorts := make(map[string]model.Port) - for _, ep := range endpoints { - uniquePorts[ep.ServicePort.GetID()] = ep.ServicePort +func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport *v1alpha1.ServiceImport, svc *v1.Service, importedSvcPorts []*model.Port) error { + updateRequired := false + if len(svcImport.Spec.IPs) != 1 || svcImport.Spec.IPs[0] != svc.Spec.ClusterIP { + r.Log.Debug("ServiceImport IP need update", "ServiceImport IPs", svcImport.Spec.IPs, "cluster IP", svc.Spec.ClusterIP) + svcImport.Spec.IPs = []string{svc.Spec.ClusterIP} + updateRequired = true } - servicePorts := make([]v1.ServicePort, 0, len(uniquePorts)) - for _, servicePort := range uniquePorts { - servicePorts = append(servicePorts, PortToServicePort(servicePort)) + // ServiceImport ports do not have TargetPort, exclude field for purpose of comparison + simplifiedSvcPorts := make([]*model.Port, 0) + for _, svcPort := range importedSvcPorts { + simplifiedSvcPorts = append(simplifiedSvcPorts, &model.Port{ + Name: svcPort.Name, + Port: svcPort.Port, + Protocol: svcPort.Protocol, + }) } - return servicePorts -} - -func extractEndpointPorts(endpoints []*model.Endpoint) []discovery.EndpointPort { - uniquePorts := make(map[string]model.Port) - for _, ep := range endpoints { - uniquePorts[ep.EndpointPort.GetID()] = ep.EndpointPort + svcImportPorts := make([]*model.Port, 0) + for _, importPort := range svcImport.Spec.Ports { + port := ServiceImportPortToPort(importPort) + svcImportPorts = append(svcImportPorts, &port) } - endpointPorts := make([]discovery.EndpointPort, 0, len(uniquePorts)) - for _, endpointPort := range uniquePorts { - endpointPorts = append(endpointPorts, PortToEndpointPort(endpointPort)) + if !PortsEqualIgnoreOrder(svcImportPorts, simplifiedSvcPorts) { + r.Log.Debug("ServiceImport ports need update", "ServiceImport Ports", svcImport.Spec.Ports, "imported ports", importedSvcPorts) + serviceImportPorts := make([]v1alpha1.ServicePort, 0) + for _, port := range importedSvcPorts { + serviceImportPorts = append(serviceImportPorts, PortToServiceImportPort(*port)) + } + svcImport.Spec.Ports = serviceImportPorts + updateRequired = true } - return endpointPorts -} - -func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport *v1alpha1.ServiceImport, svc *v1.Service) error { - if len(svcImport.Spec.IPs) != 1 || svcImport.Spec.IPs[0] != svc.Spec.ClusterIP || !portsEqual(svcImport, svc) { - svcImport.Spec.IPs = []string{svc.Spec.ClusterIP} - - svcImport.Spec.Ports = make([]v1alpha1.ServicePort, 0) - for _, p := range svc.Spec.Ports { - svcImport.Spec.Ports = append(svcImport.Spec.Ports, servicePortToServiceImport(p)) - } + if updateRequired { if err := r.Client.Update(ctx, svcImport); err != nil { return err } @@ -421,22 +279,27 @@ func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport return nil } -func portsEqual(svcImport *v1alpha1.ServiceImport, svc *v1.Service) bool { - impPorts := svcImport.Spec.Ports - svcPorts := make([]v1alpha1.ServicePort, 0) +func (r *CloudMapReconciler) updateDerivedService(ctx context.Context, svc *v1.Service, importedSvcPorts []*model.Port) error { + svcPorts := make([]*model.Port, 0) for _, p := range svc.Spec.Ports { - svcPorts = append(svcPorts, servicePortToServiceImport(p)) + port := ServicePortToPort(p) + svcPorts = append(svcPorts, &port) } - // TODO: consider order - return reflect.DeepEqual(impPorts, svcPorts) -} + portsMatch := PortsEqualIgnoreOrder(importedSvcPorts, svcPorts) + if !portsMatch { + newSvcPorts := make([]v1.ServicePort, 0) + for _, importPort := range importedSvcPorts { + newSvcPorts = append(newSvcPorts, PortToServicePort(*importPort)) + } -func servicePortToServiceImport(port v1.ServicePort) v1alpha1.ServicePort { - return v1alpha1.ServicePort{ - Name: port.Name, - Protocol: port.Protocol, - AppProtocol: port.AppProtocol, - Port: port.Port, + svc.Spec.Ports = newSvcPorts + if err := r.Client.Update(ctx, svc); err != nil { + return err + } + r.Log.Info("updated derived Service", + "namespace", svc.Namespace, "name", svc.Name, "ports", svc.Spec.Ports) } + + return nil } diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index 3ed9b506..e29f35a6 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" "k8s.io/api/discovery/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" @@ -25,7 +24,7 @@ import ( func TestCloudMapReconciler_Reconcile(t *testing.T) { // create a fake controller client and add some objects - objs := []runtime.Object{testNamespace()} + objs := []runtime.Object{k8sNamespaceForTest()} s := scheme.Scheme s.AddKnownTypes(v1alpha1.GroupVersion, &v1alpha1.ServiceImportList{}, &v1alpha1.ServiceImport{}) @@ -73,15 +72,6 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { assert.Equal(t, test.EndptIp1, endpointSlice.Endpoints[0].Addresses[0]) } -func testNamespace() *v1.Namespace { - return &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: test.NsName, - Namespace: test.NsName, - }, - } -} - func getReconciler(t *testing.T, mockSDClient *cloudmap.MockServiceDiscoveryClient, client client.Client) *CloudMapReconciler { return &CloudMapReconciler{ Client: client, diff --git a/pkg/controllers/controllers_common_test.go b/pkg/controllers/controllers_common_test.go new file mode 100644 index 00000000..b002e06c --- /dev/null +++ b/pkg/controllers/controllers_common_test.go @@ -0,0 +1,85 @@ +package controllers + +import ( + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/aws/aws-sdk-go-v2/aws" + v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// Factory functions for testing + +func k8sNamespaceForTest() *v1.Namespace { + return &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.NsName, + Namespace: test.NsName, + }, + } +} + +func k8sServiceForTest() *v1.Service { + return &v1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: test.SvcName, + Namespace: test.NsName, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Name: test.PortName1, + Protocol: test.Protocol1, + Port: test.ServicePort1, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: test.Port1}, + }}, + }, + Status: v1.ServiceStatus{}, + } +} + +func serviceExportForTest() *v1alpha1.ServiceExport { + return &v1alpha1.ServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.SvcName, + Namespace: test.NsName, + }, + } +} + +func endpointSliceForTest() *discovery.EndpointSlice { + port := int32(test.Port1) + protocol := v1.ProtocolTCP + return &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: test.NsName, + Name: test.SvcName + "-slice", + Labels: map[string]string{discovery.LabelServiceName: test.SvcName}, + }, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{{ + Addresses: []string{test.EndptIp1}, + }}, + Ports: []discovery.EndpointPort{{ + Name: aws.String(test.PortName1), + Protocol: &protocol, + Port: &port, + }}, + } +} + +func endpointSliceWithIpsAndPortsForTest(ips []string, ports []discovery.EndpointPort) *discovery.EndpointSlice { + svc := k8sServiceForTest() + slice := CreateEndpointSliceStruct(svc, test.SvcName) + slice.Ports = ports + + testEndpoints := make([]discovery.Endpoint, 0) + for _, ip := range ips { + testEndpoints = append(testEndpoints, CreateEndpointForSlice(svc, ip)) + } + slice.Endpoints = testEndpoints + + return slice +} diff --git a/pkg/controllers/endpointslice_plan.go b/pkg/controllers/endpointslice_plan.go new file mode 100644 index 00000000..ff47bdad --- /dev/null +++ b/pkg/controllers/endpointslice_plan.go @@ -0,0 +1,174 @@ +package controllers + +import ( + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1beta1" +) + +const defaultMaxEndpointsPerSlice = 100 + +type EndpointSliceChanges struct { + // Create: List of EndpointSlices that need to be created + Create []*discovery.EndpointSlice + // Update: List of EndpointSlices that need to be updated + Update []*discovery.EndpointSlice + // Delete: List of EndpointSlices that need to be deleted + Delete []*discovery.EndpointSlice + // Unmodified: List of EndpointSlices that do not need to be changed + Unmodified []*discovery.EndpointSlice +} + +type EndpointSlicePlan struct { + // maxEndpointsPerSlice defaults to 100 + maxEndpointsPerSlice int + + // Service to reconcile endpoints in + Service *v1.Service + + // ServiceImportName name used to create new EndpointSlices + ServiceImportName string + + // Current EndpontSlices + Current []*discovery.EndpointSlice + + // Desired Endpoints + Desired []*model.Endpoint +} + +// CalculateChanges returns list of EndpointSlice Changes that need to applied +func (p *EndpointSlicePlan) CalculateChanges() EndpointSliceChanges { + // populate map of desired endpoints for lookup efficiency + desiredEndpoints := make(map[string]*model.Endpoint) + for _, desiredEndpoint := range p.Desired { + desiredEndpoints[desiredEndpoint.IP] = desiredEndpoint + } + + desiredPorts := ExtractEndpointPorts(p.Desired) + + // Remove unwanted endpoints from slices + changes := p.trimSlices(desiredEndpoints, desiredPorts) + + // Add new endpoints to slices + for len(desiredEndpoints) > 0 { + sliceWithRoom, needsPortUpdate := p.getOrCreateUnfilledEndpointSlice(&changes, len(desiredEndpoints)) + + for key, endpointToAdd := range desiredEndpoints { + roomInSlice := p.getMaxEndpointsPerSlice() - len(sliceWithRoom.Endpoints) + if roomInSlice <= 0 { + // stop adding to slice once it is full + break + } + sliceWithRoom.Endpoints = append(sliceWithRoom.Endpoints, CreateEndpointForSlice(p.Service, endpointToAdd.IP)) + delete(desiredEndpoints, key) + } + + if needsPortUpdate { + newPorts := portSliceToEndpointPortSlice(desiredPorts) + sliceWithRoom.Ports = newPorts + } + } + + return changes +} + +func (p *EndpointSlicePlan) trimSlices(desiredEndpoints map[string]*model.Endpoint, desiredPorts []*model.Port) (changes EndpointSliceChanges) { + // remove all undesired existing endpoints in slices + for _, existingSlice := range p.Current { + updatedEndpointList := make([]discovery.Endpoint, 0) + for _, existingEndpoint := range existingSlice.Endpoints { + key := existingEndpoint.Addresses[0] + if _, found := desiredEndpoints[key]; found { + updatedEndpointList = append(updatedEndpointList, existingEndpoint) + delete(desiredEndpoints, key) + } + } + + // mark slice for deletion if all endpoints were removed + if len(updatedEndpointList) == 0 { + changes.Delete = append(changes.Delete, existingSlice) + continue + } + + sliceNeedsUpdate := false + + // slice needs to be updated if ports do not match + if !PortsEqualIgnoreOrder(desiredPorts, endpointPortSliceToPortSlice(existingSlice.Ports)) { + existingSlice.Ports = portSliceToEndpointPortSlice(desiredPorts) + sliceNeedsUpdate = true + } + + // slice needs to be updated if endpoint list changed + if len(updatedEndpointList) != len(existingSlice.Endpoints) { + existingSlice.Endpoints = updatedEndpointList + sliceNeedsUpdate = true + } + + if sliceNeedsUpdate { + changes.Update = append(changes.Update, existingSlice) + } else { + changes.Unmodified = append(changes.Unmodified, existingSlice) + } + } + + return changes +} + +func (p *EndpointSlicePlan) getOrCreateUnfilledEndpointSlice(changes *EndpointSliceChanges, requiredCapacity int) (sliceWithRoom *discovery.EndpointSlice, needsPortUpdate bool) { + // Prefer slices we are already updating + for _, sliceToUpdate := range changes.Update { + if len(sliceToUpdate.Endpoints) < p.getMaxEndpointsPerSlice() { + return sliceToUpdate, false + } + } + + // Update a slice marked for deletion if possible + if len(changes.Delete) > 0 { + sliceToReuse := changes.Delete[0] + changes.Delete = changes.Delete[1:] + changes.Update = append(changes.Update, sliceToReuse) + + // clear endpoint list that was marked for deletion before reusing + sliceToReuse.Endpoints = []discovery.Endpoint{} + return sliceToReuse, true + } + + // Update an unmodified slice if it has capacity to add all endpoints + for i, unmodifiedSlice := range changes.Unmodified { + proposedSliceLength := len(unmodifiedSlice.Endpoints) + requiredCapacity + if proposedSliceLength <= p.getMaxEndpointsPerSlice() { + changes.Unmodified = append(changes.Unmodified[:i], changes.Unmodified[i+1:]...) + changes.Update = append(changes.Update, unmodifiedSlice) + return unmodifiedSlice, false + } + } + + // No existing slices can fill new endpoint requirements so create a new slice + sliceToCreate := CreateEndpointSliceStruct(p.Service, p.ServiceImportName) + changes.Create = append(changes.Create, sliceToCreate) + return sliceToCreate, true +} + +func (p *EndpointSlicePlan) getMaxEndpointsPerSlice() int { + if p.maxEndpointsPerSlice != 0 { + return p.maxEndpointsPerSlice + } + + return defaultMaxEndpointsPerSlice +} + +func endpointPortSliceToPortSlice(endpointPorts []discovery.EndpointPort) (ports []*model.Port) { + for _, endpointPort := range endpointPorts { + port := EndpointPortToPort(endpointPort) + ports = append(ports, &port) + } + return ports +} + +func portSliceToEndpointPortSlice(ports []*model.Port) (endpointPorts []discovery.EndpointPort) { + for _, port := range ports { + endpointPort := PortToEndpointPort(*port) + endpointPorts = append(endpointPorts, endpointPort) + } + return endpointPorts +} diff --git a/pkg/controllers/endpointslice_plan_test.go b/pkg/controllers/endpointslice_plan_test.go new file mode 100644 index 00000000..354a1150 --- /dev/null +++ b/pkg/controllers/endpointslice_plan_test.go @@ -0,0 +1,218 @@ +package controllers + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/stretchr/testify/assert" + discovery "k8s.io/api/discovery/v1beta1" +) + +func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { + type fields struct { + Current []*discovery.EndpointSlice + Desired []*model.Endpoint + } + tests := []struct { + name string + fields fields + want EndpointSliceChanges + }{ + { + name: "no changes", + fields: fields{ + Current: []*discovery.EndpointSlice{endpointSliceForTest()}, + Desired: []*model.Endpoint{test.GetTestEndpoint1()}, + }, + want: EndpointSliceChanges{ + Unmodified: []*discovery.EndpointSlice{endpointSliceForTest()}, + }, + }, + { + name: "delete slice", + fields: fields{ + Current: []*discovery.EndpointSlice{endpointSliceForTest()}, + Desired: []*model.Endpoint{}, + }, + want: EndpointSliceChanges{ + Delete: []*discovery.EndpointSlice{endpointSliceForTest()}, + }, + }, + { + name: "new slice", + fields: fields{ + Current: []*discovery.EndpointSlice{}, + Desired: []*model.Endpoint{test.GetTestEndpoint1()}, + }, + want: EndpointSliceChanges{ + Create: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp1}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), + }, + ), + }, + }, + }, + { + name: "removed endpoint needs slice update", + fields: fields{ + Current: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp1, test.EndptIp2}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), + }, + ), + }, + Desired: []*model.Endpoint{test.GetTestEndpoint2()}, + }, + want: EndpointSliceChanges{ + Update: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp2}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), + }, + ), + }, + }, + }, + { + name: "added endpoint needs slice update", + fields: fields{ + Current: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp1}, + []discovery.EndpointPort{ + PortToEndpointPort(model.Port{Name: test.PortName1, Port: test.Port1, Protocol: test.Protocol1}), + }, + ), + }, + Desired: []*model.Endpoint{ + test.GetTestEndpoint1(), + { + Id: test.EndptId2, + IP: test.EndptIp2, + EndpointPort: model.Port{ + Name: test.PortName1, + Port: test.Port1, + Protocol: test.Protocol1, + }, + }, + }, + }, + want: EndpointSliceChanges{ + Update: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp1, test.EndptIp2}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), + }, + ), + }, + Unmodified: []*discovery.EndpointSlice{}, + }, + }, + { + name: "swapped endpoints need slice update", + fields: fields{ + Current: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp1}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), + }, + ), + }, + Desired: []*model.Endpoint{ + test.GetTestEndpoint2(), + }, + }, + want: EndpointSliceChanges{ + Update: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp2}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), + }, + ), + }, + Delete: []*discovery.EndpointSlice{}, + }, + }, + { + name: "changed ports need slice update", + fields: fields{ + Current: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp2}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), + }, + ), + }, + Desired: []*model.Endpoint{ + test.GetTestEndpoint2(), + }, + }, + want: EndpointSliceChanges{ + Update: []*discovery.EndpointSlice{ + endpointSliceWithIpsAndPortsForTest( + []string{test.EndptIp2}, + []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), + }, + ), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &EndpointSlicePlan{ + Service: k8sServiceForTest(), + ServiceImportName: test.SvcName, + Current: tt.fields.Current, + Desired: tt.fields.Desired, + } + if got := p.CalculateChanges(); !reflect.DeepEqual(got, tt.want) { + gotJson, _ := json.MarshalIndent(got, "", " ") + wantJson, _ := json.MarshalIndent(tt.want, "", " ") + t.Errorf("CalculateChanges() = \n%s\nwant = \n%s", gotJson, wantJson) + } + }) + } +} + +func TestEndpointSlicePlan_MultipleSliceCreation(t *testing.T) { + p := &EndpointSlicePlan{ + maxEndpointsPerSlice: 2, + Service: k8sServiceForTest(), + ServiceImportName: test.SvcName, + Current: []*discovery.EndpointSlice{}, + Desired: test.GetTestEndpoints(43), + } + changes := p.CalculateChanges() + assert.Equal(t, 22, len(changes.Create)) +} + +func TestEndpointSlicePlan_PreferCreateOverMultipleSliceUpdate(t *testing.T) { + p := &EndpointSlicePlan{ + maxEndpointsPerSlice: 2, + Service: k8sServiceForTest(), + ServiceImportName: test.SvcName, + Current: []*discovery.EndpointSlice{endpointSliceForTest()}, + Desired: []*model.Endpoint{test.GetTestEndpoint1()}, + } + p.Desired = append(p.Desired, test.GetTestEndpoints(2)...) + changes := p.CalculateChanges() + assert.Equal(t, 1, len(changes.Create)) + assert.Equal(t, 1, len(changes.Unmodified)) + assert.Equal(t, 0, len(changes.Update)) + assert.Equal(t, 0, len(changes.Delete)) +} diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index 49611bc3..db1495a4 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -8,7 +8,6 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" - "github.com/aws/aws-sdk-go-v2/aws" testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" @@ -17,10 +16,8 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -30,8 +27,10 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { // create a fake controller client and add some objects fakeClient := fake.NewClientBuilder(). WithScheme(getServiceExportScheme()). - WithObjects(testServiceObj(), testServiceExportObj()). - WithLists(testEndpointSliceObj()). + WithObjects(k8sServiceForTest(), serviceExportForTest()). + WithLists(&discovery.EndpointSliceList{ + Items: []discovery.EndpointSlice{*endpointSliceForTest()}, + }). Build() // create a mock cloudmap service discovery client @@ -76,8 +75,10 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { // create a fake controller client and add some objects fakeClient := fake.NewClientBuilder(). WithScheme(getServiceExportScheme()). - WithObjects(testServiceObj(), testServiceExportObj()). - WithLists(testEndpointSliceObj()). + WithObjects(k8sServiceForTest(), serviceExportForTest()). + WithLists(&discovery.EndpointSliceList{ + Items: []discovery.EndpointSlice{*endpointSliceForTest()}, + }). Build() mockController := gomock.NewController(t) @@ -116,13 +117,15 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { // create a fake controller client and add some objects - serviceExportObj := testServiceExportObj() + serviceExportObj := serviceExportForTest() // Add finalizer string to the service serviceExportObj.Finalizers = []string{ServiceExportFinalizer} fakeClient := fake.NewClientBuilder(). WithScheme(getServiceExportScheme()). WithObjects(serviceExportObj). - WithLists(testEndpointSliceObj()). + WithLists(&discovery.EndpointSliceList{ + Items: []discovery.EndpointSlice{*endpointSliceForTest()}, + }). Build() mockController := gomock.NewController(t) @@ -172,56 +175,3 @@ func getServiceExportReconciler(t *testing.T, mockClient *cloudmap.MockServiceDi CloudMap: mockClient, } } - -func testEndpointSliceObj() *discovery.EndpointSliceList { - port := int32(test.Port1) - protocol := v1.ProtocolTCP - endpointSlice := &discovery.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: test.NsName, - Name: test.SvcName + "-slice", - Labels: map[string]string{discovery.LabelServiceName: test.SvcName}, - }, - AddressType: discovery.AddressTypeIPv4, - Endpoints: []discovery.Endpoint{{ - Addresses: []string{test.EndptIp1}, - }}, - Ports: []discovery.EndpointPort{{ - Name: aws.String("http"), - Protocol: &protocol, - Port: &port, - }}, - } - endpointSliceList := &discovery.EndpointSliceList{ - Items: []discovery.EndpointSlice{*endpointSlice}, - } - return endpointSliceList -} - -func testServiceObj() *v1.Service { - return &v1.Service{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: test.SvcName, - Namespace: test.NsName, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{{ - Name: "http", - Protocol: test.Protocol1, - Port: test.ServicePort1, - TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: test.Port1}, - }}, - }, - Status: v1.ServiceStatus{}, - } -} - -func testServiceExportObj() *v1alpha1.ServiceExport { - return &v1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: test.SvcName, - Namespace: test.NsName, - }, - } -} diff --git a/pkg/controllers/utils.go b/pkg/controllers/utils.go index cb2e8d68..517a8f92 100644 --- a/pkg/controllers/utils.go +++ b/pkg/controllers/utils.go @@ -1,42 +1,77 @@ package controllers import ( - "reflect" + "crypto/sha256" + "encoding/base32" + "strings" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" ) +const ( + // DerivedServiceAnnotation annotates a ServiceImport with derived Service name + DerivedServiceAnnotation = "multicluster.k8s.aws/derived-service" + + // LabelServiceImportName indicates the name of the multi-cluster service that an EndpointSlice belongs to. + LabelServiceImportName = "multicluster.kubernetes.io/service-name" +) + +// ServicePortToPort converts a k8s service port to internal model port func ServicePortToPort(svcPort v1.ServicePort) model.Port { return model.Port{ Name: svcPort.Name, Port: svcPort.Port, TargetPort: svcPort.TargetPort.String(), - Protocol: protocolToString(svcPort.Protocol), + Protocol: string(svcPort.Protocol), } } +// ServiceImportPortToPort converts a service import port to an internal model port +func ServiceImportPortToPort(svcPort v1alpha1.ServicePort) model.Port { + return model.Port{ + Name: svcPort.Name, + Port: svcPort.Port, + Protocol: string(svcPort.Protocol), + } +} + +// EndpointPortToPort converts a k8s endpoint port to an internal model port func EndpointPortToPort(port discovery.EndpointPort) model.Port { return model.Port{ Name: *port.Name, Port: *port.Port, - Protocol: protocolToString(*port.Protocol), + Protocol: string(*port.Protocol), } } +// PortToServicePort converts an internal model port to a k8s service port func PortToServicePort(port model.Port) v1.ServicePort { return v1.ServicePort{ Name: port.Name, - Protocol: stringToProtocol(port.Protocol), + Protocol: v1.Protocol(port.Protocol), Port: port.Port, TargetPort: intstr.Parse(port.TargetPort), } } +// PortToServiceImportPort converts an internal model port to a service import port +func PortToServiceImportPort(port model.Port) v1alpha1.ServicePort { + return v1alpha1.ServicePort{ + Name: port.Name, + Protocol: v1.Protocol(port.Protocol), + Port: port.Port, + } +} + +// PortToEndpointPort converts an internal model port to a k8s endpoint port func PortToEndpointPort(port model.Port) discovery.EndpointPort { - protocol := stringToProtocol(port.Protocol) + protocol := v1.Protocol(port.Protocol) return discovery.EndpointPort{ Name: &port.Name, Protocol: &protocol, @@ -44,48 +79,140 @@ func PortToEndpointPort(port model.Port) discovery.EndpointPort { } } -func protocolToString(protocol v1.Protocol) string { - switch protocol { - case v1.ProtocolTCP: - return model.TCPProtocol - case v1.ProtocolUDP: - return model.UDPProtocol - case v1.ProtocolSCTP: - return model.SCTPProtocol - default: - return "" +// ExtractServicePorts extracts all unique service ports from a slice of endpoints +func ExtractServicePorts(endpoints []*model.Endpoint) (servicePorts []*model.Port) { + uniquePorts := make(map[string]model.Port) + for _, ep := range endpoints { + uniquePorts[ep.ServicePort.GetID()] = ep.ServicePort + } + for _, servicePort := range uniquePorts { + portRef := servicePort + servicePorts = append(servicePorts, &portRef) } + return servicePorts } -func stringToProtocol(protocol string) v1.Protocol { - switch protocol { - case model.TCPProtocol: - return v1.ProtocolTCP - case model.UDPProtocol: - return v1.ProtocolUDP - case model.SCTPProtocol: - return v1.ProtocolSCTP - default: - return "" +// ExtractEndpointPorts extracts all unique endpoint ports from a slice of endpoints +func ExtractEndpointPorts(endpoints []*model.Endpoint) (endpointPorts []*model.Port) { + uniquePorts := make(map[string]model.Port) + for _, ep := range endpoints { + uniquePorts[ep.EndpointPort.GetID()] = ep.EndpointPort + } + for _, endpointPort := range uniquePorts { + portRef := endpointPort + endpointPorts = append(endpointPorts, &portRef) } + return endpointPorts } -func EndpointPortsAreEqualIgnoreOrder(a, b []discovery.EndpointPort) (equal bool) { +func PortsEqualIgnoreOrder(a, b []*model.Port) (equal bool) { if len(a) != len(b) { return false } - for _, aPort := range a { - match := false - for _, bPort := range b { - if reflect.DeepEqual(aPort, bPort) { - match = true - break - } + aMap := make(map[string]*model.Port) + for _, portA := range a { + aMap[portA.GetID()] = portA + } + + for _, portB := range b { + portA, found := aMap[portB.GetID()] + if !found { + return false } - if !match { + + if !portB.Equals(portA) { return false } } return true } + +// DerivedName computes the "placeholder" name for an imported service +func DerivedName(namespace string, name string) string { + hash := sha256.New() + hash.Write([]byte(namespace + name)) + return "imported-" + strings.ToLower(base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(hash.Sum(nil)))[:10] +} + +// CreateServiceImportStruct creates struct representation of a ServiceImport +func CreateServiceImportStruct(namespace string, name string, servicePorts []*model.Port) *v1alpha1.ServiceImport { + serviceImportPorts := make([]v1alpha1.ServicePort, 0) + for _, port := range servicePorts { + serviceImportPorts = append(serviceImportPorts, PortToServiceImportPort(*port)) + } + + return &v1alpha1.ServiceImport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(namespace, name)}, + }, + Spec: v1alpha1.ServiceImportSpec{ + IPs: []string{}, + Type: v1alpha1.ClusterSetIP, + Ports: serviceImportPorts, + }, + } +} + +// CreateDerivedServiceStruct creates struct representation of a derived service +func CreateDerivedServiceStruct(svcImport *v1alpha1.ServiceImport, importedSvcPorts []*model.Port) *v1.Service { + ownerRef := metav1.NewControllerRef(svcImport, schema.GroupVersionKind{ + Version: svcImport.TypeMeta.APIVersion, + Kind: svcImport.TypeMeta.Kind, + }) + + svcPorts := make([]v1.ServicePort, 0) + for _, svcPort := range importedSvcPorts { + svcPorts = append(svcPorts, PortToServicePort(*svcPort)) + } + + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: svcImport.Namespace, + Name: svcImport.Annotations[DerivedServiceAnnotation], + OwnerReferences: []metav1.OwnerReference{*ownerRef}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + Ports: svcPorts, + }, + } +} + +func CreateEndpointForSlice(svc *v1.Service, ip string) discovery.Endpoint { + t := true + + return discovery.Endpoint{ + Addresses: []string{ip}, + Conditions: discovery.EndpointConditions{ + Ready: &t, + }, + TargetRef: &v1.ObjectReference{ + Kind: "Service", + Namespace: svc.Namespace, + Name: svc.Name, + UID: svc.ObjectMeta.UID, + ResourceVersion: svc.ObjectMeta.ResourceVersion, + }, + } +} + +func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string) *discovery.EndpointSlice { + return &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + discovery.LabelServiceName: svc.Name, // derived Service name + LabelServiceImportName: svcImportName, // original ServiceImport name + }, + GenerateName: svc.Name + "-", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(svc, schema.GroupVersionKind{ + Version: svc.TypeMeta.APIVersion, + Kind: svc.TypeMeta.Kind, + })}, + Namespace: svc.Namespace, + }, + AddressType: discovery.AddressTypeIPv4, + } +} diff --git a/pkg/controllers/utils_test.go b/pkg/controllers/utils_test.go index 093509fd..b06d4f37 100644 --- a/pkg/controllers/utils_test.go +++ b/pkg/controllers/utils_test.go @@ -4,9 +4,12 @@ import ( "reflect" "testing" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" v1 "k8s.io/api/core/v1" "k8s.io/api/discovery/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -49,6 +52,40 @@ func TestServicePortToPort(t *testing.T) { } } +func TestServiceImportPortToPort(t *testing.T) { + type args struct { + svcImportPort v1alpha1.ServicePort + } + tests := []struct { + name string + args args + want model.Port + }{ + { + name: "happy case", + args: args{ + svcImportPort: v1alpha1.ServicePort{ + Name: test.PortName1, + Protocol: v1.ProtocolTCP, + Port: 80, + }, + }, + want: model.Port{ + Name: test.PortName1, + Port: 80, + Protocol: test.Protocol1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ServiceImportPortToPort(tt.args.svcImportPort); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ServiceImportPortToPort() = %v, want %v", got, tt.want) + } + }) + } +} + func TestEndpointPortToPort(t *testing.T) { type args struct { port v1beta1.EndpointPort @@ -146,6 +183,41 @@ func TestPortToServicePort(t *testing.T) { } } +func TestPortToServiceImportPort(t *testing.T) { + type args struct { + port model.Port + } + tests := []struct { + name string + args args + want v1alpha1.ServicePort + }{ + { + name: "happy case", + args: args{ + port: model.Port{ + Name: test.PortName1, + Port: test.Port1, + TargetPort: test.PortStr2, // ignored + Protocol: test.Protocol1, + }, + }, + want: v1alpha1.ServicePort{ + Name: test.PortName1, + Protocol: v1.ProtocolTCP, + Port: test.Port1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PortToServiceImportPort(tt.args.port); !reflect.DeepEqual(got, tt.want) { + t.Errorf("PortToServiceImportPort() = %v, want %v", got, tt.want) + } + }) + } +} + func TestPortToEndpointPort(t *testing.T) { name := "http" protocolTCP := v1.ProtocolTCP @@ -182,3 +254,238 @@ func TestPortToEndpointPort(t *testing.T) { }) } } + +func TestExtractServicePorts(t *testing.T) { + type args struct { + endpoints []*model.Endpoint + } + tests := []struct { + name string + args args + want []*model.Port + }{ + { + name: "unique service ports extracted", + args: args{ + endpoints: []*model.Endpoint{ + { + ServicePort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + }, + { + ServicePort: model.Port{Protocol: test.Protocol1, Port: test.Port2}, + }, + { + ServicePort: model.Port{Protocol: test.Protocol2, Port: test.Port2}, + }, + }, + }, + want: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol1, Port: test.Port2}, + {Protocol: test.Protocol2, Port: test.Port2}, + }, + }, + { + name: "duplicate and endpoint ports ignored", + args: args{ + endpoints: []*model.Endpoint{ + { + ServicePort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + EndpointPort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + }, + { + ServicePort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + EndpointPort: model.Port{Protocol: test.Protocol2, Port: test.Port2}, + }, + }, + }, + want: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ExtractServicePorts(tt.args.endpoints); !PortsEqualIgnoreOrder(got, tt.want) { + t.Errorf("ServicePortToPort() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExtractEndpointPorts(t *testing.T) { + type args struct { + endpoints []*model.Endpoint + } + tests := []struct { + name string + args args + want []*model.Port + }{ + { + name: "unique endpoint ports extracted", + args: args{ + endpoints: []*model.Endpoint{ + { + EndpointPort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + }, + { + EndpointPort: model.Port{Protocol: test.Protocol1, Port: test.Port2}, + }, + { + EndpointPort: model.Port{Protocol: test.Protocol2, Port: test.Port2}, + }, + }, + }, + want: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol1, Port: test.Port2}, + {Protocol: test.Protocol2, Port: test.Port2}, + }, + }, + { + name: "duplicate and service ports ignored", + args: args{ + endpoints: []*model.Endpoint{ + { + EndpointPort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + ServicePort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + }, + { + EndpointPort: model.Port{Protocol: test.Protocol1, Port: test.Port1}, + ServicePort: model.Port{Protocol: test.Protocol2, Port: test.Port2}, + }, + }, + }, + want: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ExtractEndpointPorts(tt.args.endpoints); !PortsEqualIgnoreOrder(got, tt.want) { + t.Errorf("ServicePortToPort() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPortsEqualIgnoreOrder(t *testing.T) { + type args struct { + portsA []*model.Port + portsB []*model.Port + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "ports equal same order", + args: args{ + portsA: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol2, Port: test.Port2}, + }, + portsB: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol2, Port: test.Port2}, + }, + }, + want: true, + }, + { + name: "ports equal different order", + args: args{ + portsA: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol2, Port: test.Port2}, + }, + portsB: []*model.Port{ + {Protocol: test.Protocol2, Port: test.Port2}, + {Protocol: test.Protocol1, Port: test.Port1}, + }, + }, + want: true, + }, + { + name: "ports not equal", + args: args{ + portsA: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol2, Port: test.Port2}, + }, + portsB: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol2, Port: 3}, + }, + }, + want: false, + }, + { + name: "protocols not equal", + args: args{ + portsA: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol2, Port: test.Port2}, + }, + portsB: []*model.Port{ + {Protocol: test.Protocol1, Port: test.Port1}, + {Protocol: test.Protocol1, Port: test.Port2}, + }, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PortsEqualIgnoreOrder(tt.args.portsA, tt.args.portsB); !(got == tt.want) { + t.Errorf("PortsEqualIgnoreOrder() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCreateServiceImportStruct(t *testing.T) { + type args struct { + servicePorts []*model.Port + } + tests := []struct { + name string + args args + want v1alpha1.ServiceImport + }{ + { + name: "happy case", + args: args{ + servicePorts: []*model.Port{ + {Name: test.PortName1, Protocol: test.Protocol1, Port: test.Port1}, + {Name: test.PortName2, Protocol: test.Protocol1, Port: test.Port2}, + }, + }, + want: v1alpha1.ServiceImport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: test.NsName, + Name: test.SvcName, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.NsName, test.SvcName)}, + }, + Spec: v1alpha1.ServiceImportSpec{ + IPs: []string{}, + Type: v1alpha1.ClusterSetIP, + Ports: []v1alpha1.ServicePort{ + {Name: test.PortName1, Protocol: v1.ProtocolTCP, Port: test.Port1}, + {Name: test.PortName2, Protocol: v1.ProtocolTCP, Port: test.Port2}, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CreateServiceImportStruct(test.NsName, test.SvcName, tt.args.servicePorts); !reflect.DeepEqual(*got, tt.want) { + t.Errorf("CreateServiceImportStruct() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/model/types.go b/pkg/model/types.go index 25c8a3ae..04c761a0 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -66,9 +66,6 @@ const ( ServicePortAttr = "SERVICE_PORT" ServiceTargetPortAttr = "SERVICE_TARGET_PORT" ServiceProtocolAttr = "SERVICE_PROTOCOL" - TCPProtocol = "TCP" - UDPProtocol = "UDP" - SCTPProtocol = "SCTP" ) // NewEndpointFromInstance converts a Cloud Map HttpInstanceSummary to an endpoint. @@ -210,6 +207,11 @@ func (namespaceType *NamespaceType) IsUnsupported() bool { return *namespaceType == UnsupportedNamespaceType } -func (port *Port) GetID() string { - return fmt.Sprintf("%s:%d", port.Protocol, port.Port) +func (p *Port) GetID() string { + return fmt.Sprintf("%s:%d", p.Protocol, p.Port) +} + +// Equals evaluates if two Ports are "deeply equal" (including all fields). +func (p *Port) Equals(other *Port) bool { + return reflect.DeepEqual(p, other) } diff --git a/test/test-constants.go b/test/test-constants.go index 21738740..c51a2bae 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -1,6 +1,8 @@ package test import ( + "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" ) @@ -22,7 +24,7 @@ const ( Port2 = 2 PortStr2 = "2" PortName2 = "http" - Protocol2 = "TCP" + Protocol2 = "UDP" ServicePort2 = 22 ServicePortStr2 = "22" OpId1 = "operation-id-1" @@ -99,3 +101,14 @@ func GetTestEndpoint2() *model.Endpoint { Attributes: make(map[string]string), } } + +func GetTestEndpoints(count int) (endpts []*model.Endpoint) { + // use +3 offset go avoid collision with test endpoint 1 and 2 + for i := 3; i < count+3; i++ { + e := GetTestEndpoint1() + e.Id = fmt.Sprintf("tcp-192_168_0_%d-1", i) + e.IP = fmt.Sprintf("192.168.0.%d", i) + endpts = append(endpts, e) + } + return endpts +} From 9160b6e832242fbd617c24d72f4ba559b28e30da Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Fri, 3 Dec 2021 14:46:44 -0800 Subject: [PATCH 057/163] Patch release v0.2.2 (#117) --- config/controller_install_release/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index ede95f93..6c3c71b9 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -4,4 +4,4 @@ bases: images: - name: controller newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s - newTag: v0.2.1 + newTag: v0.2.2 From e5cd011726dba8dfc924b71b1cc759f06db242ec Mon Sep 17 00:00:00 2001 From: Akash Rungta <850306+runakash@users.noreply.github.com> Date: Mon, 6 Dec 2021 20:48:24 -0800 Subject: [PATCH 058/163] Integrate golangci-lint Github Action (#118) --- .github/workflows/build.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23fa678a..72f04bad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,3 +25,12 @@ jobs: uses: codecov/codecov-action@v2 with: files: cover.out + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: v1.43 + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + skip-pkg-cache: true + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + skip-build-cache: true From 2ce8f28ac99032bf4061e29443f16c60899a4af3 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Tue, 7 Dec 2021 11:29:40 -0800 Subject: [PATCH 059/163] Move mocks to mock packages (#120) --- Makefile | 16 ++++----- integration/janitor/api_test.go | 8 ++--- integration/janitor/janitor_test.go | 6 ++-- pkg/cloudmap/api_test.go | 34 +++++++++---------- pkg/cloudmap/client_test.go | 10 +++--- pkg/cloudmap/operation_poller_test.go | 14 ++++---- pkg/controllers/cloudmap_controller_test.go | 6 ++-- .../serviceexport_controller_test.go | 10 +++--- 8 files changed, 52 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index 456bfb96..214de487 100644 --- a/Makefile +++ b/Makefile @@ -134,14 +134,14 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi MOCKS_DESTINATION=mocks generate-mocks: mockgen - $(MOCKGEN) --source pkg/cloudmap/client.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/client_mock.go --package cloudmap - $(MOCKGEN) --source pkg/cloudmap/cache.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/cache_mock.go --package cloudmap - $(MOCKGEN) --source pkg/cloudmap/operation_poller.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_poller_mock.go --package cloudmap - $(MOCKGEN) --source pkg/cloudmap/operation_collector.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_collector_mock.go --package cloudmap - $(MOCKGEN) --source pkg/cloudmap/api.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/api_mock.go --package cloudmap - $(MOCKGEN) --source pkg/cloudmap/aws_facade.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/aws_facade_mock.go --package cloudmap - $(MOCKGEN) --source integration/janitor/api.go --destination $(MOCKS_DESTINATION)/integration/janitor/api_mock.go --package janitor - $(MOCKGEN) --source integration/janitor/aws_facade.go --destination $(MOCKS_DESTINATION)/integration/janitor/aws_facade_mock.go --package janitor + $(MOCKGEN) --source pkg/cloudmap/client.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/client_mock.go --package cloudmap_mock + $(MOCKGEN) --source pkg/cloudmap/cache.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/cache_mock.go --package cloudmap_mock + $(MOCKGEN) --source pkg/cloudmap/operation_poller.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_poller_mock.go --package cloudmap_mock + $(MOCKGEN) --source pkg/cloudmap/operation_collector.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_collector_mock.go --package cloudmap_mock + $(MOCKGEN) --source pkg/cloudmap/api.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/api_mock.go --package cloudmap_mock + $(MOCKGEN) --source pkg/cloudmap/aws_facade.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/aws_facade_mock.go --package cloudmap_mock + $(MOCKGEN) --source integration/janitor/api.go --destination $(MOCKS_DESTINATION)/integration/janitor/api_mock.go --package janitor_mock + $(MOCKGEN) --source integration/janitor/aws_facade.go --destination $(MOCKS_DESTINATION)/integration/janitor/aws_facade_mock.go --package janitor_mock CONTROLLER_GEN = $(shell pwd)/bin/controller-gen diff --git a/integration/janitor/api_test.go b/integration/janitor/api_test.go index 8f19b886..ce10330e 100644 --- a/integration/janitor/api_test.go +++ b/integration/janitor/api_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" + janitorMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" @@ -20,7 +20,7 @@ func TestServiceDiscoveryJanitorApi_DeleteNamespace_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - mocksdk := janitor.NewMockSdkJanitorFacade(mockController) + mocksdk := janitorMock.NewMockSdkJanitorFacade(mockController) jApi := getJanitorApi(mocksdk) mocksdk.EXPECT().DeleteNamespace(context.TODO(), &sd.DeleteNamespaceInput{Id: aws.String(test.NsId)}). @@ -35,7 +35,7 @@ func TestServiceDiscoveryJanitorApi_DeleteService_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - mocksdk := janitor.NewMockSdkJanitorFacade(mockController) + mocksdk := janitorMock.NewMockSdkJanitorFacade(mockController) jApi := getJanitorApi(mocksdk) mocksdk.EXPECT().DeleteService(context.TODO(), &sd.DeleteServiceInput{Id: aws.String(test.SvcId)}). @@ -45,7 +45,7 @@ func TestServiceDiscoveryJanitorApi_DeleteService_HappyCase(t *testing.T) { assert.Nil(t, err, "No error for happy case") } -func getJanitorApi(sdk *janitor.MockSdkJanitorFacade) ServiceDiscoveryJanitorApi { +func getJanitorApi(sdk *janitorMock.MockSdkJanitorFacade) ServiceDiscoveryJanitorApi { return &serviceDiscoveryJanitorApi{ janitorFacade: sdk, } diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index 6c10b91b..9dd0d276 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" + janitorMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" @@ -15,7 +15,7 @@ import ( type testJanitor struct { janitor *cloudMapJanitor - mockApi *janitor.MockServiceDiscoveryJanitorApi + mockApi *janitorMock.MockServiceDiscoveryJanitorApi failed *bool close func() } @@ -63,7 +63,7 @@ func TestCleanupNothingToClean(t *testing.T) { func getTestJanitor(t *testing.T) *testJanitor { mockController := gomock.NewController(t) - api := janitor.NewMockServiceDiscoveryJanitorApi(mockController) + api := janitorMock.NewMockServiceDiscoveryJanitorApi(mockController) failed := false return &testJanitor{ janitor: &cloudMapJanitor{ diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 72f8003b..117d2f91 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -6,7 +6,7 @@ import ( "fmt" "testing" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -27,7 +27,7 @@ func TestServiceDiscoveryApi_ListNamespaces_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) id, name := test.NsId, test.NsName @@ -48,7 +48,7 @@ func TestServiceDiscoveryApi_ListNamespaces_SkipPublicDNSNotSupported(t *testing mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) id, name := test.NsId, test.NsName @@ -68,7 +68,7 @@ func TestServiceDiscoveryApi_ListServices_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) filter := types.ServiceFilter{ @@ -91,7 +91,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) awsFacade.EXPECT().DiscoverInstances(context.TODO(), @@ -119,7 +119,7 @@ func TestServiceDiscoveryApi_ListOperations_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) filters := make([]types.OperationFilter, 0) @@ -139,7 +139,7 @@ func TestServiceDiscoveryApi_GetOperation_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) expectedOp := &types.Operation{Id: aws.String(test.OpId1), Status: types.OperationStatusPending} @@ -155,7 +155,7 @@ func TestServiceDiscoveryApi_CreateHttNamespace_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) awsFacade.EXPECT().CreateHttpNamespace(context.TODO(), &sd.CreateHttpNamespaceInput{Name: aws.String(test.NsName)}). @@ -170,7 +170,7 @@ func TestServiceDiscoveryApi_CreateService_CreateForHttpNamespace(t *testing.T) mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) nsId, svcId, svcName := test.NsId, test.SvcId, test.SvcName @@ -192,7 +192,7 @@ func TestServiceDiscoveryApi_CreateService_CreateForDnsNamespace(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) nsId, svcId, svcName := test.NsId, test.SvcId, test.SvcName @@ -220,7 +220,7 @@ func TestServiceDiscoveryApi_CreateService_ThrowError(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) nsId, svcName := test.NsId, test.SvcName @@ -241,7 +241,7 @@ func TestServiceDiscoveryApi_RegisterInstance_HappyCase(t *testing.T) { attrs := map[string]string{"a": "b"} - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) awsFacade.EXPECT().RegisterInstance(context.TODO(), &sd.RegisterInstanceInput{ ServiceId: aws.String(test.SvcId), @@ -260,7 +260,7 @@ func TestServiceDiscoveryApi_RegisterInstance_Error(t *testing.T) { defer mockController.Finish() sdkErr := errors.New("fail") - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) awsFacade.EXPECT().RegisterInstance(context.TODO(), gomock.Any()).Return(nil, sdkErr) sdApi := getServiceDiscoveryApi(t, awsFacade) @@ -272,7 +272,7 @@ func TestServiceDiscoveryApi_DeregisterInstance_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) awsFacade.EXPECT().DeregisterInstance(context.TODO(), &sd.DeregisterInstanceInput{ ServiceId: aws.String(test.SvcId), @@ -290,7 +290,7 @@ func TestServiceDiscoveryApi_DeregisterInstance_Error(t *testing.T) { defer mockController.Finish() sdkErr := errors.New("fail") - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) awsFacade.EXPECT().DeregisterInstance(context.TODO(), gomock.Any()).Return(nil, sdkErr) sdApi := getServiceDiscoveryApi(t, awsFacade) @@ -302,7 +302,7 @@ func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - awsFacade := cloudmap.NewMockAwsFacade(mockController) + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) awsFacade.EXPECT().GetOperation(context.TODO(), &sd.GetOperationInput{OperationId: aws.String(test.OpId1)}). Return(&sd.GetOperationOutput{Operation: &types.Operation{Status: types.OperationStatusPending}}, nil) @@ -317,7 +317,7 @@ func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { assert.Equal(t, test.NsId, nsId) } -func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmap.MockAwsFacade) ServiceDiscoveryApi { +func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmapMock.MockAwsFacade) ServiceDiscoveryApi { return &serviceDiscoveryApi{ log: common.NewLoggerWithLogr(testingLogger.TestLogger{T: t}), awsFacade: awsFacade, diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 9f1711bc..a8802a94 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -5,7 +5,7 @@ import ( "errors" "testing" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -18,8 +18,8 @@ import ( type testSdClient struct { client *serviceDiscoveryClient - mockApi cloudmap.MockServiceDiscoveryApi - mockCache cloudmap.MockServiceDiscoveryClientCache + mockApi cloudmapMock.MockServiceDiscoveryApi + mockCache cloudmapMock.MockServiceDiscoveryClientCache close func() } @@ -354,8 +354,8 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { func getTestSdClient(t *testing.T) *testSdClient { mockController := gomock.NewController(t) - mockCache := cloudmap.NewMockServiceDiscoveryClientCache(mockController) - mockApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + mockCache := cloudmapMock.NewMockServiceDiscoveryClientCache(mockController) + mockApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) return &testSdClient{ client: &serviceDiscoveryClient{ log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), diff --git a/pkg/cloudmap/operation_poller_test.go b/pkg/cloudmap/operation_poller_test.go index f96adf7b..2e81c9bc 100644 --- a/pkg/cloudmap/operation_poller_test.go +++ b/pkg/cloudmap/operation_poller_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" @@ -20,7 +20,7 @@ func TestOperationPoller_HappyCases(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) pollerTypes := []struct { constructor func() OperationPoller @@ -105,7 +105,7 @@ func TestOperationPoller_PollEmpty(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{}, test.OpStart) err := p.Poll(context.TODO()) @@ -116,7 +116,7 @@ func TestOperationPoller_PollFailure(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) @@ -134,7 +134,7 @@ func TestOperationPoller_PollOpFailure(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) @@ -160,7 +160,7 @@ func TestOperationPoller_PollOpFailureAndMessageFailure(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) @@ -184,7 +184,7 @@ func TestOperationPoller_PollTimeout(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - sdApi := cloudmap.NewMockServiceDiscoveryApi(mockController) + sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) p := operationPoller{ log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index e29f35a6..3e244a63 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -35,7 +35,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - mockSDClient := cloudmap.NewMockServiceDiscoveryClient(mockController) + mockSDClient := cloudmapMock.NewMockServiceDiscoveryClient(mockController) // The service model in the Cloudmap mockSDClient.EXPECT().ListServices(context.TODO(), test.NsName). Return([]*model.Service{test.GetTestServiceWithEndpoint([]*model.Endpoint{test.GetTestEndpoint1()})}, nil) @@ -72,7 +72,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { assert.Equal(t, test.EndptIp1, endpointSlice.Endpoints[0].Addresses[0]) } -func getReconciler(t *testing.T, mockSDClient *cloudmap.MockServiceDiscoveryClient, client client.Client) *CloudMapReconciler { +func getReconciler(t *testing.T, mockSDClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *CloudMapReconciler { return &CloudMapReconciler{ Client: client, Cloudmap: mockSDClient, diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index db1495a4..901f4af1 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -3,7 +3,7 @@ package controllers import ( "context" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -37,7 +37,7 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - mock := cloudmap.NewMockServiceDiscoveryClient(mockController) + mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) // expected interactions with the Cloud Map client // The first get call is expected to return nil, then second call after the creation of service is // supposed to return the value @@ -84,7 +84,7 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - mock := cloudmap.NewMockServiceDiscoveryClient(mockController) + mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) // GetService from Cloudmap returns endpoint1 and endpoint2 mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). @@ -131,7 +131,7 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() - mock := cloudmap.NewMockServiceDiscoveryClient(mockController) + mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) // GetService from Cloudmap returns endpoint1 and endpoint2 mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). @@ -167,7 +167,7 @@ func getServiceExportScheme() *runtime.Scheme { return scheme } -func getServiceExportReconciler(t *testing.T, mockClient *cloudmap.MockServiceDiscoveryClient, client client.Client) *ServiceExportReconciler { +func getServiceExportReconciler(t *testing.T, mockClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *ServiceExportReconciler { return &ServiceExportReconciler{ Client: client, Log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), From 490dce56dbbb255dd6366fe72045cc9e12da817a Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Tue, 7 Dec 2021 13:25:46 -0800 Subject: [PATCH 060/163] Remove redundant apache license headers (#121) --- main.go | 16 ---------------- pkg/api/v1alpha1/groupversion_info.go | 16 ---------------- pkg/api/v1alpha1/serviceexport_types.go | 16 ---------------- pkg/api/v1alpha1/serviceimport_types.go | 16 ---------------- pkg/controllers/serviceexport_controller.go | 16 ---------------- pkg/controllers/suite_test.go | 16 ---------------- 6 files changed, 96 deletions(-) diff --git a/main.go b/main.go index 6480f8e4..757383f7 100644 --- a/main.go +++ b/main.go @@ -1,19 +1,3 @@ -/* -Copyright 2021. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package main import ( diff --git a/pkg/api/v1alpha1/groupversion_info.go b/pkg/api/v1alpha1/groupversion_info.go index dba40d29..44459d03 100644 --- a/pkg/api/v1alpha1/groupversion_info.go +++ b/pkg/api/v1alpha1/groupversion_info.go @@ -1,19 +1,3 @@ -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - // Package v1alpha1 contains API Schema definitions for the multicluster v1alpha1 API group // +kubebuilder:object:generate=true // +groupName=multicluster.x-k8s.io diff --git a/pkg/api/v1alpha1/serviceexport_types.go b/pkg/api/v1alpha1/serviceexport_types.go index 7bc8beb6..125ea5c0 100644 --- a/pkg/api/v1alpha1/serviceexport_types.go +++ b/pkg/api/v1alpha1/serviceexport_types.go @@ -1,19 +1,3 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package v1alpha1 import ( diff --git a/pkg/api/v1alpha1/serviceimport_types.go b/pkg/api/v1alpha1/serviceimport_types.go index d13e7213..40989fdf 100644 --- a/pkg/api/v1alpha1/serviceimport_types.go +++ b/pkg/api/v1alpha1/serviceimport_types.go @@ -1,19 +1,3 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package v1alpha1 import ( diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/serviceexport_controller.go index 9a9765e7..0d77d265 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/serviceexport_controller.go @@ -1,19 +1,3 @@ -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package controllers import ( diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index 45192419..56672340 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -1,19 +1,3 @@ -/* -Copyright 2021. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package controllers import ( From 795dd804dae8d3c85bf2ee45f9bf18cef351a1f4 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 9 Dec 2021 11:21:01 -0800 Subject: [PATCH 061/163] Differentiate dns and http ns test constants (#123) --- integration/janitor/api_test.go | 4 +- integration/janitor/janitor_test.go | 14 +- pkg/cloudmap/api_test.go | 26 ++-- pkg/cloudmap/cache_test.go | 38 ++--- pkg/cloudmap/client_test.go | 134 +++++++++--------- pkg/controllers/cloudmap_controller_test.go | 8 +- pkg/controllers/controllers_common_test.go | 10 +- .../serviceexport_controller_test.go | 30 ++-- pkg/controllers/utils_test.go | 6 +- test/test-constants.go | 18 +-- 10 files changed, 145 insertions(+), 143 deletions(-) diff --git a/integration/janitor/api_test.go b/integration/janitor/api_test.go index ce10330e..55a3b5b8 100644 --- a/integration/janitor/api_test.go +++ b/integration/janitor/api_test.go @@ -23,10 +23,10 @@ func TestServiceDiscoveryJanitorApi_DeleteNamespace_HappyCase(t *testing.T) { mocksdk := janitorMock.NewMockSdkJanitorFacade(mockController) jApi := getJanitorApi(mocksdk) - mocksdk.EXPECT().DeleteNamespace(context.TODO(), &sd.DeleteNamespaceInput{Id: aws.String(test.NsId)}). + mocksdk.EXPECT().DeleteNamespace(context.TODO(), &sd.DeleteNamespaceInput{Id: aws.String(test.HttpNsId)}). Return(&sd.DeleteNamespaceOutput{OperationId: aws.String(test.OpId1)}, nil) - opId, err := jApi.DeleteNamespace(context.TODO(), test.NsId) + opId, err := jApi.DeleteNamespace(context.TODO(), test.HttpNsId) assert.Nil(t, err, "No error for happy case") assert.Equal(t, test.OpId1, opId) } diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index 9dd0d276..38c0c7a4 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -29,10 +29,10 @@ func TestCleanupHappyCase(t *testing.T) { defer tj.close() tj.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{{Id: test.NsId, Name: test.NsName}}, nil) - tj.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). + Return([]*model.Namespace{{Id: test.HttpNsId, Name: test.HttpNsName}}, nil) + tj.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) - tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). Return([]types.HttpInstanceSummary{{InstanceId: aws.String(test.EndptId1)}}, nil) tj.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1). @@ -41,12 +41,12 @@ func TestCleanupHappyCase(t *testing.T) { Return(map[string]types.OperationStatus{test.OpId1: types.OperationStatusSuccess}, nil) tj.mockApi.EXPECT().DeleteService(context.TODO(), test.SvcId). Return(nil) - tj.mockApi.EXPECT().DeleteNamespace(context.TODO(), test.NsId). + tj.mockApi.EXPECT().DeleteNamespace(context.TODO(), test.HttpNsId). Return(test.OpId2, nil) tj.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId2). - Return(test.NsId, nil) + Return(test.HttpNsId, nil) - tj.janitor.Cleanup(context.TODO(), test.NsName) + tj.janitor.Cleanup(context.TODO(), test.HttpNsName) assert.False(t, *tj.failed) } @@ -57,7 +57,7 @@ func TestCleanupNothingToClean(t *testing.T) { tj.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - tj.janitor.Cleanup(context.TODO(), test.NsName) + tj.janitor.Cleanup(context.TODO(), test.HttpNsName) assert.False(t, *tj.failed) } diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 117d2f91..28a4d6f1 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -30,7 +30,7 @@ func TestServiceDiscoveryApi_ListNamespaces_HappyCase(t *testing.T) { awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) - id, name := test.NsId, test.NsName + id, name := test.DnsNsId, test.DnsNsName ns := types.NamespaceSummary{ Name: &name, Id: &id, @@ -51,7 +51,7 @@ func TestServiceDiscoveryApi_ListNamespaces_SkipPublicDNSNotSupported(t *testing awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) - id, name := test.NsId, test.NsName + id, name := test.DnsNsId, test.DnsNsName ns := types.NamespaceSummary{ Name: &name, Id: &id, @@ -73,7 +73,7 @@ func TestServiceDiscoveryApi_ListServices_HappyCase(t *testing.T) { filter := types.ServiceFilter{ Name: types.ServiceFilterNameNamespaceId, - Values: []string{test.NsId}, + Values: []string{test.HttpNsId}, } awsFacade.EXPECT().ListServices(context.TODO(), &sd.ListServicesInput{Filters: []types.ServiceFilter{filter}}). @@ -81,7 +81,7 @@ func TestServiceDiscoveryApi_ListServices_HappyCase(t *testing.T) { {Id: aws.String(test.SvcId), Name: aws.String(test.SvcName)}, }}, nil) - svcs, err := sdApi.ListServices(context.TODO(), test.NsId) + svcs, err := sdApi.ListServices(context.TODO(), test.HttpNsId) assert.Nil(t, err, "No error for happy case") assert.True(t, len(svcs) == 1) assert.Equal(t, svcs[0], &model.Resource{Id: test.SvcId, Name: test.SvcName}) @@ -96,7 +96,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { awsFacade.EXPECT().DiscoverInstances(context.TODO(), &sd.DiscoverInstancesInput{ - NamespaceName: aws.String(test.NsName), + NamespaceName: aws.String(test.HttpNsName), ServiceName: aws.String(test.SvcName), HealthStatus: types.HealthStatusFilterAll, MaxResults: aws.Int32(1000), @@ -108,7 +108,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { }, }, nil) - insts, err := sdApi.DiscoverInstances(context.TODO(), test.NsName, test.SvcName) + insts, err := sdApi.DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err, "No error for happy case") assert.True(t, len(insts) == 2) assert.Equal(t, test.EndptId1, *insts[0].InstanceId) @@ -158,10 +158,10 @@ func TestServiceDiscoveryApi_CreateHttNamespace_HappyCase(t *testing.T) { awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) - awsFacade.EXPECT().CreateHttpNamespace(context.TODO(), &sd.CreateHttpNamespaceInput{Name: aws.String(test.NsName)}). + awsFacade.EXPECT().CreateHttpNamespace(context.TODO(), &sd.CreateHttpNamespaceInput{Name: aws.String(test.HttpNsName)}). Return(&sd.CreateHttpNamespaceOutput{OperationId: aws.String(test.OpId1)}, nil) - opId, err := sdApi.CreateHttpNamespace(context.TODO(), test.NsName) + opId, err := sdApi.CreateHttpNamespace(context.TODO(), test.HttpNsName) assert.Nil(t, err, "No error for happy case") assert.Equal(t, test.OpId1, opId) } @@ -173,7 +173,7 @@ func TestServiceDiscoveryApi_CreateService_CreateForHttpNamespace(t *testing.T) awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) - nsId, svcId, svcName := test.NsId, test.SvcId, test.SvcName + nsId, svcId, svcName := test.HttpNsId, test.SvcId, test.SvcName awsFacade.EXPECT().CreateService(context.TODO(), &sd.CreateServiceInput{ Name: &svcName, NamespaceId: &nsId, @@ -195,7 +195,7 @@ func TestServiceDiscoveryApi_CreateService_CreateForDnsNamespace(t *testing.T) { awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) - nsId, svcId, svcName := test.NsId, test.SvcId, test.SvcName + nsId, svcId, svcName := test.DnsNsId, test.SvcId, test.SvcName awsFacade.EXPECT().CreateService(context.TODO(), &sd.CreateServiceInput{ Name: &svcName, NamespaceId: &nsId, @@ -223,7 +223,7 @@ func TestServiceDiscoveryApi_CreateService_ThrowError(t *testing.T) { awsFacade := cloudmapMock.NewMockAwsFacade(mockController) sdApi := getServiceDiscoveryApi(t, awsFacade) - nsId, svcName := test.NsId, test.SvcName + nsId, svcName := test.HttpNsId, test.SvcName awsFacade.EXPECT().CreateService(context.TODO(), &sd.CreateServiceInput{ Name: &svcName, NamespaceId: &nsId, @@ -308,13 +308,13 @@ func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { awsFacade.EXPECT().GetOperation(context.TODO(), &sd.GetOperationInput{OperationId: aws.String(test.OpId1)}). Return(&sd.GetOperationOutput{Operation: &types.Operation{Status: types.OperationStatusSuccess, - Targets: map[string]string{string(types.OperationTargetTypeNamespace): test.NsId}}}, nil) + Targets: map[string]string{string(types.OperationTargetTypeNamespace): test.HttpNsId}}}, nil) sdApi := getServiceDiscoveryApi(t, awsFacade) nsId, err := sdApi.PollNamespaceOperation(context.TODO(), test.OpId1) assert.Nil(t, err) - assert.Equal(t, test.NsId, nsId) + assert.Equal(t, test.HttpNsId, nsId) } func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmapMock.MockAwsFacade) ServiceDiscoveryApi { diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index 2f448ecf..5bb8005b 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -39,7 +39,7 @@ func TestServiceDiscoveryClientCacheGetNamespace_Found(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() sdc.CacheNamespace(test.GetTestHttpNamespace()) - ns, found := sdc.GetNamespace(test.NsName) + ns, found := sdc.GetNamespace(test.HttpNsName) assert.True(t, found) assert.Equal(t, test.GetTestHttpNamespace(), ns) } @@ -47,16 +47,16 @@ func TestServiceDiscoveryClientCacheGetNamespace_Found(t *testing.T) { func TestServiceDiscoveryClientCacheGetNamespace_NotFound(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - ns, found := sdc.GetNamespace(test.NsName) + ns, found := sdc.GetNamespace(test.HttpNsName) assert.False(t, found) assert.Nil(t, ns) } func TestServiceDiscoveryClientCacheGetNamespace_Nil(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheNilNamespace(test.NsName) + sdc.CacheNilNamespace(test.HttpNsName) - ns, found := sdc.GetNamespace(test.NsName) + ns, found := sdc.GetNamespace(test.HttpNsName) assert.True(t, found) assert.Nil(t, ns) } @@ -66,18 +66,18 @@ func TestServiceDiscoveryClientCacheGetNamespace_Corrupt(t *testing.T) { if !ok { t.Fatalf("failed to create cache") } - sdc.cache.Add(sdc.buildNsKey(test.NsName), &model.Resource{}, time.Minute) + sdc.cache.Add(sdc.buildNsKey(test.HttpNsName), &model.Resource{}, time.Minute) - ns, found := sdc.GetNamespace(test.NsName) + ns, found := sdc.GetNamespace(test.HttpNsName) assert.False(t, found) assert.Nil(t, ns) } func TestServiceDiscoveryClientCacheGetServiceId_Found(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheServiceId(test.NsName, test.SvcName, test.SvcId) + sdc.CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) - svcId, found := sdc.GetServiceId(test.NsName, test.SvcName) + svcId, found := sdc.GetServiceId(test.HttpNsName, test.SvcName) assert.True(t, found) assert.Equal(t, test.SvcId, svcId) } @@ -85,7 +85,7 @@ func TestServiceDiscoveryClientCacheGetServiceId_Found(t *testing.T) { func TestServiceDiscoveryClientCacheGetServiceId_NotFound(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - svcId, found := sdc.GetServiceId(test.NsName, test.SvcName) + svcId, found := sdc.GetServiceId(test.HttpNsName, test.SvcName) assert.False(t, found) assert.Empty(t, svcId) } @@ -95,17 +95,17 @@ func TestServiceDiscoveryClientCacheGetServiceId_Corrupt(t *testing.T) { if !ok { t.Fatalf("failed to create cache") } - sdc.cache.Add(sdc.buildSvcKey(test.NsName, test.SvcName), &model.Resource{}, time.Minute) - svcId, found := sdc.GetServiceId(test.NsName, test.SvcName) + sdc.cache.Add(sdc.buildSvcKey(test.HttpNsName, test.SvcName), &model.Resource{}, time.Minute) + svcId, found := sdc.GetServiceId(test.HttpNsName, test.SvcName) assert.False(t, found) assert.Empty(t, svcId) } func TestServiceDiscoveryClientCacheGetEndpoints_Found(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) + sdc.CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) - endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) + endpts, found := sdc.GetEndpoints(test.HttpNsName, test.SvcName) assert.True(t, found) assert.Equal(t, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, endpts) } @@ -113,7 +113,7 @@ func TestServiceDiscoveryClientCacheGetEndpoints_Found(t *testing.T) { func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) + endpts, found := sdc.GetEndpoints(test.HttpNsName, test.SvcName) assert.False(t, found) assert.Nil(t, endpts) } @@ -124,18 +124,18 @@ func TestServiceDiscoveryClientCacheGetEndpoints_Corrupt(t *testing.T) { t.Fatalf("failed to create cache") } - sdc.cache.Add(sdc.buildEndptsKey(test.NsName, test.SvcName), &model.Resource{}, time.Minute) - endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) + sdc.cache.Add(sdc.buildEndptsKey(test.HttpNsName, test.SvcName), &model.Resource{}, time.Minute) + endpts, found := sdc.GetEndpoints(test.HttpNsName, test.SvcName) assert.False(t, found) assert.Nil(t, endpts) } func TestServiceDiscoveryClientEvictEndpoints(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheEndpoints(test.NsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) - sdc.EvictEndpoints(test.NsName, test.SvcName) + sdc.CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) + sdc.EvictEndpoints(test.HttpNsName, test.SvcName) - endpts, found := sdc.GetEndpoints(test.NsName, test.SvcName) + endpts, found := sdc.GetEndpoints(test.HttpNsName, test.SvcName) assert.False(t, found) assert.Nil(t, endpts) } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index a8802a94..0ccc7859 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -32,22 +32,22 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) - tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) - tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return(nil, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). Return(testHttpInstanceSummary(), nil) - tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, + tc.mockCache.EXPECT().CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) - svcs, err := tc.client.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) assert.Nil(t, err, "No error for happy case") } @@ -56,16 +56,16 @@ func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) - tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) - tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName). + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName). Return([]*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, true) - svcs, err := tc.client.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) assert.Nil(t, err, "No error for happy case") } @@ -75,11 +75,11 @@ func TestServiceDiscoveryClient_ListServices_NamespaceError(t *testing.T) { defer tc.close() nsErr := errors.New("error listing namespaces") - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return(nil, nsErr) - svcs, err := tc.client.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Equal(t, nsErr, err) assert.Empty(t, svcs) } @@ -88,13 +88,13 @@ func TestServiceDiscoveryClient_ListServices_ServiceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) svcErr := errors.New("error listing services") - tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). Return([]*model.Resource{}, svcErr) - svcs, err := tc.client.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Equal(t, svcErr, err) assert.Empty(t, svcs) } @@ -103,18 +103,18 @@ func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) - tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) endptErr := errors.New("error listing endpoints") - tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return(nil, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). Return([]types.HttpInstanceSummary{}, endptErr) - svcs, err := tc.client.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Equal(t, endptErr, err) assert.Empty(t, svcs) } @@ -123,12 +123,12 @@ func TestServiceDiscoveryClient_ListServices_NamespaceNotFound(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) + tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) - svcs, err := tc.client.ListServices(context.TODO(), test.NsName) + svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Empty(t, svcs) assert.Nil(t, err, "No error for namespace not found") } @@ -137,13 +137,13 @@ func TestServiceDiscoveryClient_CreateService_HappyCase(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestHttpNamespace(), true) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) - tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) - err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err, "No error for happy case") } @@ -151,13 +151,13 @@ func TestServiceDiscoveryClient_CreateService_HappyCaseForDNSNamespace(t *testin tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestDnsNamespace(), true) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestDnsNamespace(), true) tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). Return(test.SvcId, nil) - tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) - err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err, "No error for happy case") } @@ -166,11 +166,11 @@ func TestServiceDiscoveryClient_CreateService_NamespaceError(t *testing.T) { defer tc.close() nsErr := errors.New("error listing namespaces") - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nsErr) - err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Equal(t, nsErr, err) } @@ -178,13 +178,13 @@ func TestServiceDiscoveryClient_CreateService_CreateServiceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(test.GetTestDnsNamespace(), true) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestDnsNamespace(), true) svcErr := errors.New("error creating service") tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). Return("", svcErr) - err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Equal(t, err, svcErr) } @@ -192,22 +192,22 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_HappyCase(t *test tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) + tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) - tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). Return(test.OpId1, nil) tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). - Return(test.NsId, nil) + Return(test.HttpNsId, nil) tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) - tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) - err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err, "No error for happy case") } @@ -215,18 +215,18 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *test tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) + tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) pollErr := errors.New("polling error") - tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). Return(test.OpId1, nil) tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). Return("", pollErr) - err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Equal(t, pollErr, err) } @@ -234,16 +234,16 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_CreateNsError(t * tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.NsName) + tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) nsErr := errors.New("create namespace error") - tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.NsName). + tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). Return("", nsErr) - err := tc.client.CreateService(context.TODO(), test.NsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Equal(t, nsErr, err) } @@ -251,26 +251,26 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return([]*model.Endpoint{}, false) + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return([]*model.Endpoint{}, false) - tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName) + tc.mockCache.EXPECT().GetServiceId(test.HttpNsName, test.SvcName) - tc.mockCache.EXPECT().GetNamespace(test.NsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) tc.mockApi.EXPECT().ListNamespaces(context.TODO()). Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) - tc.mockApi.EXPECT().ListServices(context.TODO(), test.NsId). + tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.NsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) - tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName).Return([]*model.Endpoint{}, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.NsName, test.SvcName). + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return([]*model.Endpoint{}, false) + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). Return(testHttpInstanceSummary(), nil) - tc.mockCache.EXPECT().CacheEndpoints(test.NsName, test.SvcName, + tc.mockCache.EXPECT().CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) - svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) + svc, err := tc.client.GetService(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err) assert.Equal(t, test.GetTestService(), svc) } @@ -279,10 +279,10 @@ func TestServiceDiscoveryClient_GetService_CachedValues(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetEndpoints(test.NsName, test.SvcName). + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName). Return([]*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, true) - svc, err := tc.client.GetService(context.TODO(), test.NsName, test.SvcName) + svc, err := tc.client.GetService(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err) assert.Equal(t, test.GetTestService(), svc) } @@ -291,7 +291,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName).Return(test.SvcId, true) + tc.mockCache.EXPECT().GetServiceId(test.HttpNsName, test.SvcName).Return(test.SvcId, true) attrs1 := map[string]string{ model.EndpointIpv4Attr: test.EndptIp1, @@ -323,9 +323,9 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { test.OpId1: types.OperationStatusSuccess, test.OpId2: types.OperationStatusSuccess}, nil) - tc.mockCache.EXPECT().EvictEndpoints(test.NsName, test.SvcName) + tc.mockCache.EXPECT().EvictEndpoints(test.HttpNsName, test.SvcName) - err := tc.client.RegisterEndpoints(context.TODO(), test.NsName, test.SvcName, + err := tc.client.RegisterEndpoints(context.TODO(), test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) assert.Nil(t, err) @@ -335,7 +335,7 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetServiceId(test.NsName, test.SvcName).Return(test.SvcId, true) + tc.mockCache.EXPECT().GetServiceId(test.HttpNsName, test.SvcName).Return(test.SvcId, true) tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).Return(test.OpId1, nil) tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2).Return(test.OpId2, nil) @@ -344,9 +344,9 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { test.OpId1: types.OperationStatusSuccess, test.OpId2: types.OperationStatusSuccess}, nil) - tc.mockCache.EXPECT().EvictEndpoints(test.NsName, test.SvcName) + tc.mockCache.EXPECT().EvictEndpoints(test.HttpNsName, test.SvcName) - err := tc.client.DeleteEndpoints(context.TODO(), test.NsName, test.SvcName, + err := tc.client.DeleteEndpoints(context.TODO(), test.HttpNsName, test.SvcName, []*model.Endpoint{{Id: test.EndptId1}, {Id: test.EndptId2}}) assert.Nil(t, err) diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index 3e244a63..4f944995 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -37,7 +37,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { mockSDClient := cloudmapMock.NewMockServiceDiscoveryClient(mockController) // The service model in the Cloudmap - mockSDClient.EXPECT().ListServices(context.TODO(), test.NsName). + mockSDClient.EXPECT().ListServices(context.TODO(), test.HttpNsName). Return([]*model.Service{test.GetTestServiceWithEndpoint([]*model.Endpoint{test.GetTestEndpoint1()})}, nil) reconciler := getReconciler(t, mockSDClient, fakeClient) @@ -49,13 +49,13 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { // assert service import object serviceImport := &v1alpha1.ServiceImport{} - err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceImport) + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceImport) assert.NoError(t, err) assert.Equal(t, test.SvcName, serviceImport.Name, "Service imported") // assert derived service is successfully created derivedServiceList := &v1.ServiceList{} - err = fakeClient.List(context.TODO(), derivedServiceList, client.InNamespace(test.NsName)) + err = fakeClient.List(context.TODO(), derivedServiceList, client.InNamespace(test.HttpNsName)) assert.NoError(t, err) derivedService := derivedServiceList.Items[0] assert.True(t, strings.Contains(derivedService.Name, "imported-"), "Derived service created", "service", derivedService.Name) @@ -64,7 +64,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { // assert endpoint slices are created endpointSliceList := &v1beta1.EndpointSliceList{} - err = fakeClient.List(context.TODO(), endpointSliceList, client.InNamespace(test.NsName)) + err = fakeClient.List(context.TODO(), endpointSliceList, client.InNamespace(test.HttpNsName)) assert.NoError(t, err) endpointSlice := endpointSliceList.Items[0] assert.Equal(t, test.SvcName, endpointSlice.Labels["multicluster.kubernetes.io/service-name"], "Endpoint slice is created") diff --git a/pkg/controllers/controllers_common_test.go b/pkg/controllers/controllers_common_test.go index b002e06c..90a6c81e 100644 --- a/pkg/controllers/controllers_common_test.go +++ b/pkg/controllers/controllers_common_test.go @@ -15,8 +15,8 @@ import ( func k8sNamespaceForTest() *v1.Namespace { return &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: test.NsName, - Namespace: test.NsName, + Name: test.HttpNsName, + Namespace: test.HttpNsName, }, } } @@ -26,7 +26,7 @@ func k8sServiceForTest() *v1.Service { TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Name: test.SvcName, - Namespace: test.NsName, + Namespace: test.HttpNsName, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{{ @@ -44,7 +44,7 @@ func serviceExportForTest() *v1alpha1.ServiceExport { return &v1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ Name: test.SvcName, - Namespace: test.NsName, + Namespace: test.HttpNsName, }, } } @@ -54,7 +54,7 @@ func endpointSliceForTest() *discovery.EndpointSlice { protocol := v1.ProtocolTCP return &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Namespace: test.NsName, + Namespace: test.HttpNsName, Name: test.SvcName + "-slice", Labels: map[string]string{discovery.LabelServiceName: test.SvcName}, }, diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index 901f4af1..5a30dd8a 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -41,19 +41,19 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { // expected interactions with the Cloud Map client // The first get call is expected to return nil, then second call after the creation of service is // supposed to return the value - first := mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName).Return(nil, nil) - second := mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). - Return(&model.Service{Namespace: test.NsName, Name: test.SvcName}, nil) + first := mock.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName).Return(nil, nil) + second := mock.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName). + Return(&model.Service{Namespace: test.HttpNsName, Name: test.SvcName}, nil) gomock.InOrder(first, second) - mock.EXPECT().CreateService(gomock.Any(), test.NsName, test.SvcName).Return(nil).Times(1) - mock.EXPECT().RegisterEndpoints(gomock.Any(), test.NsName, test.SvcName, + mock.EXPECT().CreateService(gomock.Any(), test.HttpNsName, test.SvcName).Return(nil).Times(1) + mock.EXPECT().RegisterEndpoints(gomock.Any(), test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1()}).Return(nil).Times(1) reconciler := getServiceExportReconciler(t, mock, fakeClient) request := ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: test.NsName, + Namespace: test.HttpNsName, Name: test.SvcName, }, } @@ -66,7 +66,7 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { assert.Equal(t, ctrl.Result{}, got, "Result should be empty") serviceExport := &v1alpha1.ServiceExport{} - err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceExport) + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceExport) assert.NoError(t, err) assert.Contains(t, serviceExport.Finalizers, ServiceExportFinalizer, "Finalizer added to the service export") } @@ -87,15 +87,15 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) // GetService from Cloudmap returns endpoint1 and endpoint2 - mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). + mock.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName). Return(test.GetTestService(), nil) // call to delete the endpoint not present in the k8s cluster - mock.EXPECT().DeleteEndpoints(gomock.Any(), test.NsName, test.SvcName, + mock.EXPECT().DeleteEndpoints(gomock.Any(), test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint2()}).Return(nil).Times(1) request := ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: test.NsName, + Namespace: test.HttpNsName, Name: test.SvcName, }, } @@ -110,7 +110,7 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { assert.Equal(t, ctrl.Result{}, got, "Result should be empty") serviceExport := &v1alpha1.ServiceExport{} - err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceExport) + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceExport) assert.NoError(t, err) assert.Contains(t, serviceExport.Finalizers, ServiceExportFinalizer, "Finalizer added to the service export") } @@ -134,15 +134,15 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) // GetService from Cloudmap returns endpoint1 and endpoint2 - mock.EXPECT().GetService(gomock.Any(), test.NsName, test.SvcName). + mock.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName). Return(test.GetTestService(), nil) // call to delete the endpoint in the cloudmap - mock.EXPECT().DeleteEndpoints(gomock.Any(), test.NsName, test.SvcName, + mock.EXPECT().DeleteEndpoints(gomock.Any(), test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}).Return(nil).Times(1) request := ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: test.NsName, + Namespace: test.HttpNsName, Name: test.SvcName, }, } @@ -154,7 +154,7 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { assert.Equal(t, ctrl.Result{}, got, "Result should be empty") serviceExport := &v1alpha1.ServiceExport{} - err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.NsName, Name: test.SvcName}, serviceExport) + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceExport) assert.NoError(t, err) assert.Empty(t, serviceExport.Finalizers, "Finalizer removed from the service export") } diff --git a/pkg/controllers/utils_test.go b/pkg/controllers/utils_test.go index b06d4f37..b4e1bda6 100644 --- a/pkg/controllers/utils_test.go +++ b/pkg/controllers/utils_test.go @@ -466,9 +466,9 @@ func TestCreateServiceImportStruct(t *testing.T) { }, want: v1alpha1.ServiceImport{ ObjectMeta: metav1.ObjectMeta{ - Namespace: test.NsName, + Namespace: test.HttpNsName, Name: test.SvcName, - Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.NsName, test.SvcName)}, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName)}, }, Spec: v1alpha1.ServiceImportSpec{ IPs: []string{}, @@ -483,7 +483,7 @@ func TestCreateServiceImportStruct(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := CreateServiceImportStruct(test.NsName, test.SvcName, tt.args.servicePorts); !reflect.DeepEqual(*got, tt.want) { + if got := CreateServiceImportStruct(test.HttpNsName, test.SvcName, tt.args.servicePorts); !reflect.DeepEqual(*got, tt.want) { t.Errorf("CreateServiceImportStruct() = %v, want %v", got, tt.want) } }) diff --git a/test/test-constants.go b/test/test-constants.go index c51a2bae..58c5672c 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -7,8 +7,10 @@ import ( ) const ( - NsName = "ns-name" - NsId = "ns-id" + HttpNsName = "http-ns-name" + DnsNsName = "dns-ns-name" + HttpNsId = "http-ns-id" + DnsNsId = "dns-ns-id" SvcName = "svc-name" SvcId = "svc-id" EndptId1 = "tcp-192_168_0_1-1" @@ -34,23 +36,23 @@ const ( func GetTestHttpNamespace() *model.Namespace { return &model.Namespace{ - Id: NsId, - Name: NsName, + Id: HttpNsId, + Name: HttpNsName, Type: model.HttpNamespaceType, } } func GetTestDnsNamespace() *model.Namespace { return &model.Namespace{ - Id: NsId, - Name: NsName, + Id: DnsNsId, + Name: DnsNsName, Type: model.DnsPrivateNamespaceType, } } func GetTestService() *model.Service { return &model.Service{ - Namespace: NsName, + Namespace: HttpNsName, Name: SvcName, Endpoints: []*model.Endpoint{GetTestEndpoint1(), GetTestEndpoint2()}, } @@ -58,7 +60,7 @@ func GetTestService() *model.Service { func GetTestServiceWithEndpoint(endpoints []*model.Endpoint) *model.Service { return &model.Service{ - Namespace: NsName, + Namespace: HttpNsName, Name: SvcName, Endpoints: endpoints, } From 97072a6ae7fb8cb7af2a463ee1e4ae9a44ccf6ee Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Mon, 13 Dec 2021 15:18:19 -0800 Subject: [PATCH 062/163] Cache namespace and service maps (#124) --- integration/janitor/janitor.go | 28 ++--- integration/janitor/janitor_test.go | 12 +-- pkg/cloudmap/api.go | 51 +++++----- pkg/cloudmap/api_test.go | 27 ++--- pkg/cloudmap/cache.go | 85 ++++++++-------- pkg/cloudmap/cache_test.go | 103 +++++++++++-------- pkg/cloudmap/client.go | 116 +++++++++------------ pkg/cloudmap/client_test.go | 136 ++++++++++++------------- pkg/controllers/cloudmap_controller.go | 4 +- pkg/model/types.go | 6 -- test/test-constants.go | 2 +- 11 files changed, 279 insertions(+), 291 deletions(-) diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index 0ac9a416..24127bdf 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -39,37 +39,31 @@ func NewDefaultJanitor() CloudMapJanitor { func (j *cloudMapJanitor) Cleanup(ctx context.Context, nsName string) { fmt.Printf("Cleaning up all test resources in Cloud Map for namespace : %s\n", nsName) - nsList, err := j.sdApi.ListNamespaces(ctx) + nsMap, err := j.sdApi.GetNamespaceMap(ctx) j.checkOrFail(err, "", "could not find namespace to clean") - var nsId string - for _, ns := range nsList { - if ns.Name == nsName { - nsId = ns.Id - } - } - - if nsId == "" { + ns, found := nsMap[nsName] + if !found { fmt.Println("namespace does not exist in account, nothing to clean") return } - fmt.Printf("found namespace to clean: %s\n", nsId) + fmt.Printf("found namespace to clean: %s\n", ns.Id) - svcs, err := j.sdApi.ListServices(ctx, nsId) + svcIdMap, err := j.sdApi.GetServiceIdMap(ctx, ns.Id) j.checkOrFail(err, - fmt.Sprintf("namespace has %d services to clean", len(svcs)), + fmt.Sprintf("namespace has %d services to clean", len(svcIdMap)), "could not find services to clean") - for _, svc := range svcs { - fmt.Printf("found service to clean: %s\n", svc.Id) - j.deregisterInstances(ctx, nsName, svc.Name, svc.Id) + for svcName, svcId := range svcIdMap { + fmt.Printf("found service to clean: %s\n", svcId) + j.deregisterInstances(ctx, nsName, svcName, svcId) - delSvcErr := j.sdApi.DeleteService(ctx, svc.Id) + delSvcErr := j.sdApi.DeleteService(ctx, svcId) j.checkOrFail(delSvcErr, "service deleted", "could not cleanup service") } - opId, err := j.sdApi.DeleteNamespace(ctx, nsId) + opId, err := j.sdApi.DeleteNamespace(ctx, ns.Id) if err == nil { fmt.Println("namespace delete in progress") _, err = j.sdApi.PollNamespaceOperation(ctx, opId) diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index 38c0c7a4..196125e4 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -28,10 +28,10 @@ func TestCleanupHappyCase(t *testing.T) { tj := getTestJanitor(t) defer tj.close() - tj.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{{Id: test.HttpNsId, Name: test.HttpNsName}}, nil) - tj.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). - Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) + tj.mockApi.EXPECT().GetNamespaceMap(context.TODO()). + Return(map[string]*model.Namespace{test.HttpNsName: test.GetTestHttpNamespace()}, nil) + tj.mockApi.EXPECT().GetServiceIdMap(context.TODO(), test.HttpNsId). + Return(map[string]string{test.SvcName: test.SvcId}, nil) tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). Return([]types.HttpInstanceSummary{{InstanceId: aws.String(test.EndptId1)}}, nil) @@ -54,8 +54,8 @@ func TestCleanupNothingToClean(t *testing.T) { tj := getTestJanitor(t) defer tj.close() - tj.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{}, nil) + tj.mockApi.EXPECT().GetNamespaceMap(context.TODO()). + Return(map[string]*model.Namespace{}, nil) tj.janitor.Cleanup(context.TODO(), test.HttpNsName) assert.False(t, *tj.failed) diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index f2caaf46..38bad30f 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -20,11 +20,11 @@ const ( // ServiceDiscoveryApi handles the AWS Cloud Map API request and response processing logic, and converts results to // internal data structures. It manages all interactions with the AWS SDK. type ServiceDiscoveryApi interface { - // ListNamespaces returns a list of all namespaces. - ListNamespaces(ctx context.Context) (namespaces []*model.Namespace, err error) + // GetNamespaceMap returns a map of all namespaces in the Cloud Map account indexed by namespace name. + GetNamespaceMap(ctx context.Context) (namespaces map[string]*model.Namespace, err error) - // ListServices returns a list of services for a given namespace. - ListServices(ctx context.Context, namespaceId string) (services []*model.Resource, err error) + // GetServiceIdMap returns a map of all service IDs for a given namespace indexed by service name. + GetServiceIdMap(ctx context.Context, namespaceId string) (serviceIdMap map[string]string, err error) // DiscoverInstances returns a list of service instances registered to a given service. DiscoverInstances(ctx context.Context, nsName string, svcName string) (insts []types.HttpInstanceSummary, err error) @@ -64,52 +64,53 @@ func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { } } -func (sdApi *serviceDiscoveryApi) ListNamespaces(ctx context.Context) (namespaces []*model.Namespace, err error) { - pages := sd.NewListNamespacesPaginator(sdApi.awsFacade, &sd.ListNamespacesInput{}) +func (sdApi *serviceDiscoveryApi) GetNamespaceMap(ctx context.Context) (map[string]*model.Namespace, error) { + namespaceMap := make(map[string]*model.Namespace) + pages := sd.NewListNamespacesPaginator(sdApi.awsFacade, &sd.ListNamespacesInput{}) for pages.HasMorePages() { output, err := pages.NextPage(ctx) if err != nil { - return namespaces, err + return nil, err } for _, ns := range output.Namespaces { - if namespaceType := model.ConvertNamespaceType(ns.Type); !namespaceType.IsUnsupported() { - namespaces = append(namespaces, &model.Namespace{ - Id: aws.ToString(ns.Id), - Name: aws.ToString(ns.Name), - Type: namespaceType, - }) + namespaceType := model.ConvertNamespaceType(ns.Type) + if namespaceType.IsUnsupported() { + continue + } + namespaceMap[aws.ToString(ns.Name)] = &model.Namespace{ + Id: aws.ToString(ns.Id), + Name: aws.ToString(ns.Name), + Type: namespaceType, } } } - return namespaces, nil + return namespaceMap, nil } -func (sdApi *serviceDiscoveryApi) ListServices(ctx context.Context, nsId string) (svcs []*model.Resource, err error) { +func (sdApi *serviceDiscoveryApi) GetServiceIdMap(ctx context.Context, nsId string) (map[string]string, error) { + serviceIdMap := make(map[string]string) + filter := types.ServiceFilter{ Name: types.ServiceFilterNameNamespaceId, Values: []string{nsId}, } pages := sd.NewListServicesPaginator(sdApi.awsFacade, &sd.ListServicesInput{Filters: []types.ServiceFilter{filter}}) - for pages.HasMorePages() { output, err := pages.NextPage(ctx) if err != nil { - return svcs, err + return nil, err } for _, svc := range output.Services { - svcs = append(svcs, &model.Resource{ - Id: aws.ToString(svc.Id), - Name: aws.ToString(svc.Name), - }) + serviceIdMap[aws.ToString(svc.Name)] = aws.ToString(svc.Id) } } - return svcs, nil + return serviceIdMap, nil } func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string) (insts []types.HttpInstanceSummary, err error) { @@ -127,8 +128,8 @@ func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName return out.Instances, nil } -func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters []types.OperationFilter) (opStatusMap map[string]types.OperationStatus, err error) { - opStatusMap = make(map[string]types.OperationStatus) +func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters []types.OperationFilter) (map[string]types.OperationStatus, error) { + opStatusMap := make(map[string]types.OperationStatus) pages := sd.NewListOperationsPaginator(sdApi.awsFacade, &sd.ListOperationsInput{ Filters: opFilters, @@ -190,7 +191,7 @@ func (sdApi *serviceDiscoveryApi) CreateService(ctx context.Context, namespace m } svcId = aws.ToString(output.Service.Id) - sdApi.log.Info("service created", "svcId", svcId) + sdApi.log.Info("service created", "namespace", namespace.Name, "name", svcName, "id", svcId) return svcId, nil } diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 28a4d6f1..6207dfb9 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -8,7 +8,6 @@ import ( cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" @@ -23,7 +22,7 @@ func TestNewServiceDiscoveryApi(t *testing.T) { assert.NotNil(t, sdc) } -func TestServiceDiscoveryApi_ListNamespaces_HappyCase(t *testing.T) { +func TestServiceDiscoveryApi_GetNamespaceMap_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() @@ -39,12 +38,13 @@ func TestServiceDiscoveryApi_ListNamespaces_HappyCase(t *testing.T) { awsFacade.EXPECT().ListNamespaces(context.TODO(), &sd.ListNamespacesInput{}). Return(&sd.ListNamespacesOutput{Namespaces: []types.NamespaceSummary{ns}}, nil) - namespaces, _ := sdApi.ListNamespaces(context.TODO()) + namespaces, err := sdApi.GetNamespaceMap(context.TODO()) + assert.Nil(t, err, "No error for happy case") assert.True(t, len(namespaces) == 1) - assert.Equal(t, test.GetTestDnsNamespace(), namespaces[0], "No error for happy case") + assert.Equal(t, test.GetTestDnsNamespace(), namespaces[test.DnsNsName]) } -func TestServiceDiscoveryApi_ListNamespaces_SkipPublicDNSNotSupported(t *testing.T) { +func TestServiceDiscoveryApi_GetNamespaceMap_SkipPublicDNSNotSupported(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() @@ -60,11 +60,12 @@ func TestServiceDiscoveryApi_ListNamespaces_SkipPublicDNSNotSupported(t *testing awsFacade.EXPECT().ListNamespaces(context.TODO(), &sd.ListNamespacesInput{}). Return(&sd.ListNamespacesOutput{Namespaces: []types.NamespaceSummary{ns}}, nil) - namespaces, _ := sdApi.ListNamespaces(context.TODO()) - assert.True(t, len(namespaces) == 0, "Successfully skipped DNS_PUBLIC from the output") + namespaces, err := sdApi.GetNamespaceMap(context.TODO()) + assert.Nil(t, err, "No error for happy case") + assert.Empty(t, namespaces, "Successfully skipped DNS_PUBLIC from the output") } -func TestServiceDiscoveryApi_ListServices_HappyCase(t *testing.T) { +func TestServiceDiscoveryApi_GetServiceIdMap_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() @@ -81,10 +82,10 @@ func TestServiceDiscoveryApi_ListServices_HappyCase(t *testing.T) { {Id: aws.String(test.SvcId), Name: aws.String(test.SvcName)}, }}, nil) - svcs, err := sdApi.ListServices(context.TODO(), test.HttpNsId) + svcs, err := sdApi.GetServiceIdMap(context.TODO(), test.HttpNsId) assert.Nil(t, err, "No error for happy case") assert.True(t, len(svcs) == 1) - assert.Equal(t, svcs[0], &model.Resource{Id: test.SvcId, Name: test.SvcName}) + assert.Equal(t, svcs[test.SvcName], test.SvcId) } func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { @@ -184,7 +185,8 @@ func TestServiceDiscoveryApi_CreateService_CreateForHttpNamespace(t *testing.T) }, }, nil) - retSvcId, _ := sdApi.CreateService(context.TODO(), *test.GetTestHttpNamespace(), svcName) + retSvcId, err := sdApi.CreateService(context.TODO(), *test.GetTestHttpNamespace(), svcName) + assert.Nil(t, err) assert.Equal(t, svcId, retSvcId, "Successfully created service") } @@ -212,7 +214,8 @@ func TestServiceDiscoveryApi_CreateService_CreateForDnsNamespace(t *testing.T) { }, }, nil) - retSvcId, _ := sdApi.CreateService(context.TODO(), *test.GetTestDnsNamespace(), svcName) + retSvcId, err := sdApi.CreateService(context.TODO(), *test.GetTestDnsNamespace(), svcName) + assert.Nil(t, err) assert.Equal(t, svcId, retSvcId, "Successfully created service") } diff --git a/pkg/cloudmap/cache.go b/pkg/cloudmap/cache.go index ccad3691..a808938a 100644 --- a/pkg/cloudmap/cache.go +++ b/pkg/cloudmap/cache.go @@ -11,22 +11,23 @@ import ( ) const ( - nsKeyPrefix = "ns" - svcKeyPrefix = "svc" - endptKeyPrefix = "endpt" + nsKey = "ns-map" + svcKeyPrefix = "svc-map" + endptsKeyPrefix = "endpts" defaultCacheSize = 1024 - defaultNsTTL = 2 * time.Minute - defaultSvcTTL = 2 * time.Minute + defaultNsTTL = 10 * time.Second + defaultSvcTTL = 10 * time.Second defaultEndptTTL = 5 * time.Second ) type ServiceDiscoveryClientCache interface { - GetNamespace(namespaceName string) (namespace *model.Namespace, found bool) - CacheNamespace(namespace *model.Namespace) - CacheNilNamespace(namespaceName string) - GetServiceId(namespaceName string, serviceName string) (serviceId string, found bool) - CacheServiceId(namespaceName string, serviceName string, serviceId string) + GetNamespaceMap() (namespaces map[string]*model.Namespace, found bool) + CacheNamespaceMap(namespaces map[string]*model.Namespace) + EvictNamespaceMap() + GetServiceIdMap(namespaceName string) (serviceIdMap map[string]string, found bool) + CacheServiceIdMap(namespaceName string, serviceIdMap map[string]string) + EvictServiceIdMap(namespaceName string) GetEndpoints(namespaceName string, serviceName string) (endpoints []*model.Endpoint, found bool) CacheEndpoints(namespaceName string, serviceName string, endpoints []*model.Endpoint) EvictEndpoints(namespaceName string, serviceName string) @@ -61,58 +62,56 @@ func NewDefaultServiceDiscoveryClientCache() ServiceDiscoveryClientCache { }) } -func (sdCache *sdCache) GetNamespace(nsName string) (ns *model.Namespace, found bool) { - key := sdCache.buildNsKey(nsName) - entry, exists := sdCache.cache.Get(key) +func (sdCache *sdCache) GetNamespaceMap() (namespaceMap map[string]*model.Namespace, found bool) { + entry, exists := sdCache.cache.Get(nsKey) if !exists { return nil, false } - if entry == nil { - return nil, true - } - - nsEntry, ok := entry.(model.Namespace) + namespaceMap, ok := entry.(map[string]*model.Namespace) if !ok { - sdCache.log.Error(errors.New("failed to retrieve namespace from cache"), "", "nsName", nsName) - sdCache.cache.Remove(key) + sdCache.log.Error(errors.New("failed to retrieve namespaceMap from cache"), "") + sdCache.cache.Remove(nsKey) return nil, false } - return &nsEntry, true + return namespaceMap, true } -func (sdCache *sdCache) CacheNamespace(namespace *model.Namespace) { - key := sdCache.buildNsKey(namespace.Name) - sdCache.cache.Add(key, *namespace, sdCache.config.NsTTL) +func (sdCache *sdCache) CacheNamespaceMap(namespaces map[string]*model.Namespace) { + sdCache.cache.Add(nsKey, namespaces, sdCache.config.NsTTL) } -func (sdCache *sdCache) CacheNilNamespace(nsName string) { - key := sdCache.buildNsKey(nsName) - sdCache.cache.Add(key, nil, sdCache.config.NsTTL) +func (sdCache *sdCache) EvictNamespaceMap() { + sdCache.cache.Remove(nsKey) } -func (sdCache *sdCache) GetServiceId(nsName string, svcName string) (svcId string, found bool) { - key := sdCache.buildSvcKey(nsName, svcName) +func (sdCache *sdCache) GetServiceIdMap(nsName string) (serviceIdMap map[string]string, found bool) { + key := sdCache.buildSvcKey(nsName) entry, exists := sdCache.cache.Get(key) if !exists { - return "", false + return nil, false } - svcId, ok := entry.(string) + serviceIdMap, ok := entry.(map[string]string) if !ok { - sdCache.log.Error(errors.New("failed to retrieve service ID from cache"), "", - "nsName", nsName, "svcName", svcName) + sdCache.log.Error(errors.New("failed to retrieve service IDs from cache"), "", + "nsName", nsName) sdCache.cache.Remove(key) - return "", false + return nil, false } - return svcId, true + return serviceIdMap, true } -func (sdCache *sdCache) CacheServiceId(nsName string, svcName string, svcId string) { - key := sdCache.buildSvcKey(nsName, svcName) - sdCache.cache.Add(key, svcId, sdCache.config.SvcTTL) +func (sdCache *sdCache) CacheServiceIdMap(nsName string, serviceIdMap map[string]string) { + key := sdCache.buildSvcKey(nsName) + sdCache.cache.Add(key, serviceIdMap, sdCache.config.SvcTTL) +} + +func (sdCache *sdCache) EvictServiceIdMap(nsName string) { + key := sdCache.buildSvcKey(nsName) + sdCache.cache.Remove(key) } func (sdCache *sdCache) GetEndpoints(nsName string, svcName string) (endpts []*model.Endpoint, found bool) { @@ -143,14 +142,10 @@ func (sdCache *sdCache) EvictEndpoints(nsName string, svcName string) { sdCache.cache.Remove(key) } -func (sdCache *sdCache) buildNsKey(nsName string) (cacheKey string) { - return fmt.Sprintf("%s:%s", nsKeyPrefix, nsName) -} - -func (sdCache *sdCache) buildSvcKey(nsName string, svcName string) (cacheKey string) { - return fmt.Sprintf("%s:%s:%s", svcKeyPrefix, nsName, svcName) +func (sdCache *sdCache) buildSvcKey(nsName string) (cacheKey string) { + return fmt.Sprintf("%s:%s", svcKeyPrefix, nsName) } func (sdCache *sdCache) buildEndptsKey(nsName string, svcName string) string { - return fmt.Sprintf("%s:%s:%s", endptKeyPrefix, nsName, svcName) + return fmt.Sprintf("%s:%s:%s", endptsKeyPrefix, nsName, svcName) } diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index 5bb8005b..11c65ada 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -4,9 +4,12 @@ import ( "testing" "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + testing2 "github.com/go-logr/logr/testing" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/cache" ) func TestNewServiceDiscoveryClientCache(t *testing.T) { @@ -35,70 +38,84 @@ func TestNewDefaultServiceDiscoveryClientCache(t *testing.T) { assert.Equal(t, defaultEndptTTL, sdc.config.EndptTTL) } -func TestServiceDiscoveryClientCacheGetNamespace_Found(t *testing.T) { +func TestServiceDiscoveryClientCacheGetNamespaceMap_Found(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheNamespace(test.GetTestHttpNamespace()) + sdc.CacheNamespaceMap(map[string]*model.Namespace{ + test.HttpNsName: test.GetTestHttpNamespace(), + }) - ns, found := sdc.GetNamespace(test.HttpNsName) + nsMap, found := sdc.GetNamespaceMap() assert.True(t, found) - assert.Equal(t, test.GetTestHttpNamespace(), ns) + assert.Equal(t, test.GetTestHttpNamespace(), nsMap[test.HttpNsName]) } -func TestServiceDiscoveryClientCacheGetNamespace_NotFound(t *testing.T) { +func TestServiceDiscoveryClientCacheGetNamespaceMap_NotFound(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - ns, found := sdc.GetNamespace(test.HttpNsName) + nsMap, found := sdc.GetNamespaceMap() assert.False(t, found) - assert.Nil(t, ns) + assert.Nil(t, nsMap) } -func TestServiceDiscoveryClientCacheGetNamespace_Nil(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheNilNamespace(test.HttpNsName) +func TestServiceDiscoveryClientCacheGetNamespaceMap_Corrupt(t *testing.T) { + sdc := getCacheImpl(t) + sdc.cache.Add(nsKey, &model.Plan{}, time.Minute) - ns, found := sdc.GetNamespace(test.HttpNsName) - assert.True(t, found) - assert.Nil(t, ns) + nsMap, found := sdc.GetNamespaceMap() + assert.False(t, found) + assert.Nil(t, nsMap) } -func TestServiceDiscoveryClientCacheGetNamespace_Corrupt(t *testing.T) { - sdc, ok := NewDefaultServiceDiscoveryClientCache().(*sdCache) - if !ok { - t.Fatalf("failed to create cache") - } - sdc.cache.Add(sdc.buildNsKey(test.HttpNsName), &model.Resource{}, time.Minute) +func TestServiceDiscoveryClientEvictNamespaceMap(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + sdc.CacheNamespaceMap(map[string]*model.Namespace{ + test.HttpNsName: test.GetTestHttpNamespace(), + }) + sdc.EvictNamespaceMap() - ns, found := sdc.GetNamespace(test.HttpNsName) + nsMap, found := sdc.GetNamespaceMap() assert.False(t, found) - assert.Nil(t, ns) + assert.Nil(t, nsMap) } -func TestServiceDiscoveryClientCacheGetServiceId_Found(t *testing.T) { +func TestServiceDiscoveryClientCacheGetServiceIdMap_Found(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - sdc.CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + sdc.CacheServiceIdMap(test.HttpNsName, map[string]string{ + test.SvcName: test.SvcId, + }) - svcId, found := sdc.GetServiceId(test.HttpNsName, test.SvcName) + svcIdMap, found := sdc.GetServiceIdMap(test.HttpNsName) assert.True(t, found) - assert.Equal(t, test.SvcId, svcId) + assert.Equal(t, test.SvcId, svcIdMap[test.SvcName]) } -func TestServiceDiscoveryClientCacheGetServiceId_NotFound(t *testing.T) { +func TestServiceDiscoveryClientCacheGetServiceIdMap_NotFound(t *testing.T) { sdc := NewDefaultServiceDiscoveryClientCache() - svcId, found := sdc.GetServiceId(test.HttpNsName, test.SvcName) + svcIdMap, found := sdc.GetServiceIdMap(test.HttpNsName) assert.False(t, found) - assert.Empty(t, svcId) + assert.Empty(t, svcIdMap) } -func TestServiceDiscoveryClientCacheGetServiceId_Corrupt(t *testing.T) { - sdc, ok := NewDefaultServiceDiscoveryClientCache().(*sdCache) - if !ok { - t.Fatalf("failed to create cache") - } - sdc.cache.Add(sdc.buildSvcKey(test.HttpNsName, test.SvcName), &model.Resource{}, time.Minute) - svcId, found := sdc.GetServiceId(test.HttpNsName, test.SvcName) +func TestServiceDiscoveryClientCacheGetServiceIdMap_Corrupt(t *testing.T) { + sdc := getCacheImpl(t) + sdc.cache.Add(sdc.buildSvcKey(test.HttpNsName), &model.Plan{}, time.Minute) + + svcIdMap, found := sdc.GetServiceIdMap(test.HttpNsName) assert.False(t, found) - assert.Empty(t, svcId) + assert.Empty(t, svcIdMap) +} + +func TestServiceDiscoveryClientEvictServiceIdMap(t *testing.T) { + sdc := NewDefaultServiceDiscoveryClientCache() + sdc.CacheServiceIdMap(test.HttpNsName, map[string]string{ + test.SvcName: test.SvcId, + }) + sdc.EvictServiceIdMap(test.HttpNsName) + + svcIdMap, found := sdc.GetServiceIdMap(test.HttpNsName) + assert.False(t, found) + assert.Empty(t, svcIdMap) } func TestServiceDiscoveryClientCacheGetEndpoints_Found(t *testing.T) { @@ -119,12 +136,9 @@ func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { } func TestServiceDiscoveryClientCacheGetEndpoints_Corrupt(t *testing.T) { - sdc, ok := NewDefaultServiceDiscoveryClientCache().(*sdCache) - if !ok { - t.Fatalf("failed to create cache") - } + sdc := getCacheImpl(t) + sdc.cache.Add(sdc.buildEndptsKey(test.HttpNsName, test.SvcName), &model.Plan{}, time.Minute) - sdc.cache.Add(sdc.buildEndptsKey(test.HttpNsName, test.SvcName), &model.Resource{}, time.Minute) endpts, found := sdc.GetEndpoints(test.HttpNsName, test.SvcName) assert.False(t, found) assert.Nil(t, endpts) @@ -139,3 +153,10 @@ func TestServiceDiscoveryClientEvictEndpoints(t *testing.T) { assert.False(t, found) assert.Nil(t, endpts) } + +func getCacheImpl(t *testing.T) sdCache { + return sdCache{ + log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), + cache: cache.NewLRUExpireCache(defaultCacheSize), + } +} diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index e10b891c..66eae577 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -2,6 +2,7 @@ package cloudmap import ( "context" + "errors" "fmt" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" @@ -53,28 +54,20 @@ func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCa } func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName string) (svcs []*model.Service, err error) { - namespace, err := sdc.getNamespace(ctx, nsName) - if err != nil || namespace == nil { - return svcs, err - } - - // TODO: Cache list - svcSums, err := sdc.sdApi.ListServices(ctx, namespace.Id) + svcIdMap, err := sdc.getServiceIds(ctx, nsName) if err != nil { return svcs, err } - for _, svcSum := range svcSums { - sdc.cache.CacheServiceId(nsName, svcSum.Name, svcSum.Id) - - endpts, endptsErr := sdc.listEndpoints(ctx, nsName, svcSum.Name) + for svcName := range svcIdMap { + endpts, endptsErr := sdc.getEndpoints(ctx, nsName, svcName) if endptsErr != nil { return svcs, endptsErr } svcs = append(svcs, &model.Service{ Namespace: nsName, - Name: svcSum.Name, + Name: svcName, Endpoints: endpts, }) } @@ -82,28 +75,29 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri return svcs, nil } -func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName string, svcName string) (err error) { +func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName string, svcName string) error { sdc.log.Info("creating a new service", "namespace", nsName, "name", svcName) - namespace, err := sdc.getNamespace(ctx, nsName) + nsMap, err := sdc.getNamespaces(ctx) if err != nil { return err } + namespace := nsMap[nsName] if namespace == nil { - // Create HttpNamespace if the namespace is not present in the CloudMap + // Create HttpNamespace if the namespace is not present in CloudMap namespace, err = sdc.createNamespace(ctx, nsName) if err != nil { return err } } - svcId, err := sdc.sdApi.CreateService(ctx, *namespace, svcName) + _, err = sdc.sdApi.CreateService(ctx, *namespace, svcName) if err != nil { return err } - sdc.cache.CacheServiceId(nsName, svcName, svcId) + sdc.cache.EvictServiceIdMap(nsName) return nil } @@ -120,17 +114,16 @@ func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string }, nil } - svcId, err := sdc.getServiceId(ctx, nsName, svcName) - + svcIdMap, err := sdc.getServiceIds(ctx, nsName) if err != nil { return nil, err } - - if svcId == "" { + _, found := svcIdMap[svcName] + if !found { return nil, nil } - endpts, err = sdc.listEndpoints(ctx, nsName, svcName) + endpts, err = sdc.getEndpoints(ctx, nsName, svcName) if err != nil { return nil, err @@ -151,10 +144,14 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName sdc.log.Info("registering endpoints", "namespaceName", nsName, "serviceName", svcName, "endpoints", endpts) - svcId, err := sdc.getServiceId(ctx, nsName, svcName) + svcIdMap, err := sdc.getServiceIds(ctx, nsName) if err != nil { return err } + svcId, found := svcIdMap[svcName] + if !found { + return fmt.Errorf("service not found in Cloud Map: %s", svcName) + } opCollector := NewOperationCollector() @@ -176,7 +173,7 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName } if !opCollector.IsAllOperationsCreated() { - return fmt.Errorf("failure while registering endpoints") + return errors.New("failure while registering endpoints") } return nil @@ -191,10 +188,14 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s sdc.log.Info("deleting endpoints", "namespaceName", nsName, "serviceName", svcName, "endpoints", endpts) - svcId, err := sdc.getServiceId(ctx, nsName, svcName) + svcIdMap, err := sdc.getServiceIds(ctx, nsName) if err != nil { return err } + svcId, found := svcIdMap[svcName] + if !found { + return fmt.Errorf("service not found in Cloud Map: %s", svcName) + } opCollector := NewOperationCollector() @@ -214,15 +215,15 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s } if !opCollector.IsAllOperationsCreated() { - return fmt.Errorf("failure while de-registering endpoints") + return errors.New("failure while de-registering endpoints") } return nil } -func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, nsName string, svcName string) (endpts []*model.Endpoint, err error) { - endpts, found := sdc.cache.GetEndpoints(nsName, svcName) - if found { +func (sdc *serviceDiscoveryClient) getEndpoints(ctx context.Context, nsName string, svcName string) (endpts []*model.Endpoint, err error) { + endpts, cacheHit := sdc.cache.GetEndpoints(nsName, svcName) + if cacheHit { return endpts, nil } @@ -239,65 +240,46 @@ func (sdc *serviceDiscoveryClient) listEndpoints(ctx context.Context, nsName str } endpts = append(endpts, endpt) } - sdc.cache.CacheEndpoints(nsName, svcName, endpts) return endpts, nil } -func (sdc *serviceDiscoveryClient) getNamespace(ctx context.Context, nsName string) (namespace *model.Namespace, err error) { +func (sdc *serviceDiscoveryClient) getNamespaces(ctx context.Context) (namespace map[string]*model.Namespace, err error) { // We are assuming a unique namespace name per account - namespace, exists := sdc.cache.GetNamespace(nsName) - if exists { - return namespace, nil + namespaces, cacheHit := sdc.cache.GetNamespaceMap() + if cacheHit { + return namespaces, nil } - namespaces, err := sdc.sdApi.ListNamespaces(ctx) + namespaces, err = sdc.sdApi.GetNamespaceMap(ctx) if err != nil { return nil, err } + sdc.cache.CacheNamespaceMap(namespaces) - for _, ns := range namespaces { - sdc.cache.CacheNamespace(ns) - // Set the return namespace - if nsName == ns.Name { - namespace = ns - } - } - - if namespace == nil { - // This will cache empty namespace for namespaces not in Cloud Map - // This is so that we can avoid ListNamespaces call - sdc.cache.CacheNilNamespace(nsName) - } - - return namespace, nil + return namespaces, nil } -func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName string, svcName string) (svcId string, err error) { - svcId, found := sdc.cache.GetServiceId(nsName, svcName) - if found { - return svcId, nil +func (sdc *serviceDiscoveryClient) getServiceIds(ctx context.Context, nsName string) (map[string]string, error) { + serviceIdMap, cacheHit := sdc.cache.GetServiceIdMap(nsName) + if cacheHit { + return serviceIdMap, nil } - namespace, err := sdc.getNamespace(ctx, nsName) + nsMap, err := sdc.getNamespaces(ctx) + namespace := nsMap[nsName] if err != nil || namespace == nil { - return "", err + return nil, err } - services, err := sdc.sdApi.ListServices(ctx, namespace.Id) + serviceIdMap, err = sdc.sdApi.GetServiceIdMap(ctx, namespace.Id) if err != nil { - return "", err - } - - for _, svc := range services { - sdc.cache.CacheServiceId(nsName, svcName, svc.Id) - if svc.Name == svcName { - svcId = svc.Id - } + return nil, err } + sdc.cache.CacheServiceIdMap(nsName, serviceIdMap) - return svcId, nil + return serviceIdMap, nil } func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName string) (namespace *model.Namespace, err error) { @@ -321,6 +303,6 @@ func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName s Type: model.HttpNamespaceType, } - sdc.cache.CacheNamespace(namespace) + sdc.cache.EvictNamespaceMap() return namespace, nil } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 0ccc7859..9a21dcd9 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -2,7 +2,6 @@ package cloudmap import ( "context" - "errors" "testing" cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" @@ -13,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" testing2 "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) @@ -32,18 +32,18 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) - tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). - Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, false) + tc.mockApi.EXPECT().GetNamespaceMap(context.TODO()).Return(getNamespaceMapForTest(), nil) + tc.mockCache.EXPECT().CacheNamespaceMap(getNamespaceMapForTest()) + + tc.mockApi.EXPECT().GetServiceIdMap(context.TODO(), test.HttpNsId).Return(getServiceIdMapForTest(), nil) + tc.mockCache.EXPECT().CacheServiceIdMap(test.HttpNsName, getServiceIdMapForTest()) tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). - Return(testHttpInstanceSummary(), nil) + Return(getHttpInstanceSummaryForTest(), nil) tc.mockCache.EXPECT().CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) @@ -56,17 +56,16 @@ func TestServiceDiscoveryClient_ListServices_HappyCaseCachedResults(t *testing.T tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) + dnsService := test.GetTestService() + dnsService.Namespace = test.DnsNsName - tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). - Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().GetServiceIdMap(test.DnsNsName).Return(getServiceIdMapForTest(), true) - tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName). + tc.mockCache.EXPECT().GetEndpoints(test.DnsNsName, test.SvcName). Return([]*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, true) - svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) - assert.Equal(t, []*model.Service{test.GetTestService()}, svcs) + svcs, err := tc.client.ListServices(context.TODO(), test.DnsNsName) + assert.Equal(t, []*model.Service{dnsService}, svcs) assert.Nil(t, err, "No error for happy case") } @@ -74,10 +73,11 @@ func TestServiceDiscoveryClient_ListServices_NamespaceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(nil, false) + nsErr := errors.New("error listing namespaces") - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return(nil, nsErr) + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, false) + tc.mockApi.EXPECT().GetNamespaceMap(context.TODO()).Return(nil, nsErr) svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Equal(t, nsErr, err) @@ -88,11 +88,13 @@ func TestServiceDiscoveryClient_ListServices_ServiceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(nil, false) + + tc.mockCache.EXPECT().GetNamespaceMap().Return(getNamespaceMapForTest(), true) svcErr := errors.New("error listing services") - tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). - Return([]*model.Resource{}, svcErr) + tc.mockApi.EXPECT().GetServiceIdMap(context.TODO(), test.HttpNsId). + Return(nil, svcErr) svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Equal(t, svcErr, err) @@ -103,11 +105,7 @@ func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) - - tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). - Return([]*model.Resource{{Name: test.SvcName, Id: test.SvcId}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) endptErr := errors.New("error listing endpoints") tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) @@ -123,10 +121,8 @@ func TestServiceDiscoveryClient_ListServices_NamespaceNotFound(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(nil, false) + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, true) svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) assert.Empty(t, svcs) @@ -137,11 +133,11 @@ func TestServiceDiscoveryClient_CreateService_HappyCase(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestHttpNamespace(), true) + tc.mockCache.EXPECT().GetNamespaceMap().Return(getNamespaceMapForTest(), true) tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) - tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().EvictServiceIdMap(test.HttpNsName) err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err, "No error for happy case") @@ -151,13 +147,13 @@ func TestServiceDiscoveryClient_CreateService_HappyCaseForDNSNamespace(t *testin tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestDnsNamespace(), true) + tc.mockCache.EXPECT().GetNamespaceMap().Return(getNamespaceMapForTest(), true) tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). Return(test.SvcId, nil) - tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().EvictServiceIdMap(test.DnsNsName) - err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.DnsNsName, test.SvcName) assert.Nil(t, err, "No error for happy case") } @@ -166,9 +162,9 @@ func TestServiceDiscoveryClient_CreateService_NamespaceError(t *testing.T) { defer tc.close() nsErr := errors.New("error listing namespaces") - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{}, nsErr) + + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, false) + tc.mockApi.EXPECT().GetNamespaceMap(context.TODO()).Return(nil, nsErr) err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Equal(t, nsErr, err) @@ -178,13 +174,13 @@ func TestServiceDiscoveryClient_CreateService_CreateServiceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(test.GetTestDnsNamespace(), true) + tc.mockCache.EXPECT().GetNamespaceMap().Return(getNamespaceMapForTest(), true) svcErr := errors.New("error creating service") tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestDnsNamespace(), test.SvcName). Return("", svcErr) - err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) + err := tc.client.CreateService(context.TODO(), test.DnsNsName, test.SvcName) assert.Equal(t, err, svcErr) } @@ -192,20 +188,19 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_HappyCase(t *test tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) + tc.mockCache.EXPECT().GetNamespaceMap().Return(map[string]*model.Namespace{ + test.DnsNsName: test.GetTestDnsNamespace(), + }, true) tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). Return(test.OpId1, nil) tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). Return(test.HttpNsId, nil) - tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) + tc.mockCache.EXPECT().EvictNamespaceMap() tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). Return(test.SvcId, nil) - tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + tc.mockCache.EXPECT().EvictServiceIdMap(test.HttpNsName) err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Nil(t, err, "No error for happy case") @@ -215,10 +210,7 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *test tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, true) pollErr := errors.New("polling error") tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). @@ -234,10 +226,7 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_CreateNsError(t * tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{}, nil) - tc.mockCache.EXPECT().CacheNilNamespace(test.HttpNsName) + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, true) nsErr := errors.New("create namespace error") tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). @@ -251,22 +240,22 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return([]*model.Endpoint{}, false) + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) - tc.mockCache.EXPECT().GetServiceId(test.HttpNsName, test.SvcName) + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(nil, false) - tc.mockCache.EXPECT().GetNamespace(test.HttpNsName).Return(nil, false) - tc.mockApi.EXPECT().ListNamespaces(context.TODO()). - Return([]*model.Namespace{test.GetTestHttpNamespace()}, nil) - tc.mockCache.EXPECT().CacheNamespace(test.GetTestHttpNamespace()) + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, false) + tc.mockApi.EXPECT().GetNamespaceMap(context.TODO()). + Return(getNamespaceMapForTest(), nil) + tc.mockCache.EXPECT().CacheNamespaceMap(getNamespaceMapForTest()) - tc.mockApi.EXPECT().ListServices(context.TODO(), test.HttpNsId). - Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil) - tc.mockCache.EXPECT().CacheServiceId(test.HttpNsName, test.SvcName, test.SvcId) + tc.mockApi.EXPECT().GetServiceIdMap(context.TODO(), test.HttpNsId). + Return(map[string]string{test.SvcName: test.SvcId}, nil) + tc.mockCache.EXPECT().CacheServiceIdMap(test.HttpNsName, getServiceIdMapForTest()) tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return([]*model.Endpoint{}, false) tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). - Return(testHttpInstanceSummary(), nil) + Return(getHttpInstanceSummaryForTest(), nil) tc.mockCache.EXPECT().CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) @@ -291,7 +280,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetServiceId(test.HttpNsName, test.SvcName).Return(test.SvcId, true) + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) attrs1 := map[string]string{ model.EndpointIpv4Attr: test.EndptIp1, @@ -335,7 +324,7 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { tc := getTestSdClient(t) defer tc.close() - tc.mockCache.EXPECT().GetServiceId(test.HttpNsName, test.SvcName).Return(test.SvcId, true) + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).Return(test.OpId1, nil) tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2).Return(test.OpId2, nil) @@ -368,7 +357,7 @@ func getTestSdClient(t *testing.T) *testSdClient { } } -func testHttpInstanceSummary() []types.HttpInstanceSummary { +func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { return []types.HttpInstanceSummary{ { InstanceId: aws.String(test.EndptId1), @@ -398,3 +387,14 @@ func testHttpInstanceSummary() []types.HttpInstanceSummary { }, } } + +func getNamespaceMapForTest() map[string]*model.Namespace { + return map[string]*model.Namespace{ + test.HttpNsName: test.GetTestHttpNamespace(), + test.DnsNsName: test.GetTestDnsNamespace(), + } +} + +func getServiceIdMapForTest() map[string]string { + return map[string]string{test.SvcName: test.SvcId} +} diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/cloudmap_controller.go index 4235b40f..2cd53655 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/cloudmap_controller.go @@ -55,12 +55,10 @@ func (r *CloudMapReconciler) Start(ctx context.Context) error { func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { namespaces := v1.NamespaceList{} if err := r.Client.List(ctx, &namespaces); err != nil { - r.Log.Error(err, "unable to list namespaces") + r.Log.Error(err, "unable to list cluster namespaces") return err } - //TODO: Fetch list of namespaces from Cloudmap and only reconcile the intersection - for _, ns := range namespaces.Items { if err := r.reconcileNamespace(ctx, ns.Name); err != nil { return err diff --git a/pkg/model/types.go b/pkg/model/types.go index 04c761a0..edbca8ff 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -10,12 +10,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" ) -// Resource encapsulates a ID/name pair. -type Resource struct { - Id string - Name string -} - const ( HttpNamespaceType NamespaceType = "HTTP" DnsPrivateNamespaceType NamespaceType = "DNS_PRIVATE" diff --git a/test/test-constants.go b/test/test-constants.go index 58c5672c..efd10116 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -25,7 +25,7 @@ const ( ServicePortStr1 = "11" Port2 = 2 PortStr2 = "2" - PortName2 = "http" + PortName2 = "https" Protocol2 = "UDP" ServicePort2 = 22 ServicePortStr2 = "22" From 6f90059fe537a0f10c8833e2376c45c5d57daf97 Mon Sep 17 00:00:00 2001 From: Cameron Senese <36317955+cameronsenese@users.noreply.github.com> Date: Tue, 15 Feb 2022 04:30:30 +1100 Subject: [PATCH 063/163] Update deployment samples and README.md (#127) * Update deployment samples and README.md * modify updates to deployment samples and README.md * include example- config files in /samples * update README.md - standardise kubectl command format and minor grammar/wording changes --- README.md | 62 ++++++++++---- samples/coredns-clusterrole.yaml | 58 +++++++++++++ samples/coredns-configmap.yaml | 26 ++++++ samples/coredns-deployment.yaml | 137 +++++++++++++++++++++++++++++++ 4 files changed, 267 insertions(+), 16 deletions(-) create mode 100644 samples/coredns-clusterrole.yaml create mode 100644 samples/coredns-configmap.yaml create mode 100644 samples/coredns-deployment.yaml diff --git a/README.md b/README.md index 2bf882c8..411d4668 100644 --- a/README.md +++ b/README.md @@ -13,20 +13,45 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/aws/aws-cloud-map-mcs-controller-for-k8s)](https://goreportcard.com/report/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) ## Introduction -AWS Cloud Map multi-cluster service discovery for Kubernetes (K8s) is a controller that implements existing multi-cluster services API that allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. +The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [multi-cluster services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) specification, which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. See the demo from AWS Container Day x KubeCon! [![Watch the video](https://img.youtube.com/vi/3f0Tv7IiQQw/0.jpg)](https://youtu.be/3f0Tv7IiQQw?t=24458) -## Usage -> ⚠ **There must exist network connectivity (i.e. VPC peering, security group rules, ACLs, etc.) between clusters**: Undefined behavior may occur if controller is set up without network connectivity between clusters. +## Installation + +Perform the following installation steps on each participating cluster. + +- For multi-cluster service discovery and consumption, the controller should be installed on a minimum of 2 EKS clusters. +- Participating clusters should be provisioned into a single AWS account, within a single AWS region. + +### Dependencies + +#### Network + +> ⚠ **The AWS Cloud Map MCS Controller for K8s provides service discovery and communication across multiple clusters, therefore implementations depend on end-end network connectivity between workloads provisioned within each participating cluster.** + +- In deployment scenarios where participating clusters are provisioned into separate VPCs, connectivity will depend on correctly configured [VPC Peering](https://docs.aws.amazon.com/vpc/latest/peering/create-vpc-peering-connection.html), [inter-VPC routing](https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-routing.html), and Security Group configuration. The [VPC Reachability Analyzer](https://docs.aws.amazon.com/vpc/latest/reachability/getting-started.html) can be used to test and validate end-end connectivity between worker nodes within each cluster. +- Undefined behavior may occur if controllers are deployed without the required network connectivity between clusters. + +#### Configure CoreDNS -### Setup clusters +Install the The CoreDNS multicluster plugin into each participating cluster. The multicluster plugin enables CoreDNS to lifecycle manage DNS records for `ServiceImport` objects. -First, install the controller with latest release on at least 2 AWS EKS clusters. Nodes must have sufficient IAM permissions to perform CloudMap operations. +To install the plugin, run the following commands. -> **_NOTE:_** AWS region environment variable can be _optionaly_ set like `export AWS_REGION=us-west-2` Otherwise controller will infer region in the order `AWS_REGION` environment variable, ~/.aws/config file, then EC2 metadata (for EKS environment) +```bash +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-clusterrole.yaml +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-configmap.yaml +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-deployment.yaml +``` + +### Install Controller + +To install the latest release of the controller, run the following commands. + +> **_NOTE:_** AWS region environment variable can be _optionaly_ set like `export AWS_REGION=us-west-2` Otherwise the controller will infer region in the order `AWS_REGION` environment variable, ~/.aws/config file, then EC2 metadata (for EKS environment) ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release" @@ -34,6 +59,10 @@ kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/con > 📌 See [Releases](#Releases) section for details on how to install other versions. +The controller must have sufficient IAM permissions to perform required Cloud Map operations. Grant IAM access rights `AWSCloudMapFullAccess` to the controller Service Account to enable the controller to manage Cloud Map resources. + +## Usage + ### Export services Then assuming you already have a Service installed, apply a `ServiceExport` yaml to the cluster in which you want to export a service. This can be done for each service you want to export. @@ -55,17 +84,18 @@ metadata: name: my-amazing-service ``` -*See the `samples` directory for a set of example yaml files to set up a service and export it. To apply the sample files run* +*See the `samples` directory for a set of example yaml files to set up a service and export it. To apply the sample files run the following commands.* + ```sh kubectl create namespace example -kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-deployment.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-service.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/aws-cloud-map-mcs-controller-for-k8s/main/samples/example-serviceexport.yaml +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/example-deployment.yaml +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/example-service.yaml +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/example-serviceexport.yaml ``` ### Import services -In your other cluster, the controller will automatically sync services registered in AWS Cloud Map by applying the appropriate `ServiceImport`. To list them all, run +In your other cluster, the controller will automatically sync services registered in AWS Cloud Map by applying the appropriate `ServiceImport`. To list them all, run the following command. ```sh kubectl get ServiceImport -A ``` @@ -76,24 +106,24 @@ AWS Cloud Map MCS Controller for K8s adheres to the [SemVer](https://semver.org/ > **_NOTE:_** AWS region environment variable can be _optionally_ set like `export AWS_REGION=us-west-2` Otherwise controller will infer region in the order `AWS_REGION` environment variable, ~/.aws/config file, then EC2 metadata (for EKS environment) -To install from a release run +The following command format is used to install from a particular release. ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release[?ref=*git version tag*]" ``` -Example to install latest release +Run the following command to install the latest release. ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release" ``` -Example to install v0.1.0 +The following example will install release v0.1.0. ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release?ref=v0.1.0" ``` We also maintain a `latest` tag, which is updated to stay in line with the `main` branch. We **do not** recommend installing this on any production cluster, as any new major versions updated on the `main` branch will introduce breaking changes. -To install from `latest` tag run +To install from `latest` tag run the following command. ```sh kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" ``` @@ -109,4 +139,4 @@ Join the channel with this [invite](https://join.slack.com/t/awsappmesh/shared_i This project is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0), -see [LICENSE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/LICENSE) and [NOTICE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/NOTICE) for more information. +see [LICENSE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/LICENSE) and [NOTICE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/NOTICE) for more information. \ No newline at end of file diff --git a/samples/coredns-clusterrole.yaml b/samples/coredns-clusterrole.yaml new file mode 100644 index 00000000..242212fa --- /dev/null +++ b/samples/coredns-clusterrole.yaml @@ -0,0 +1,58 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - multicluster.x-k8s.io + resources: + - serviceimports + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - multicluster.x-k8s.io + resources: + - serviceexports + verbs: + - create + - get + - list + - patch + - update + - watch \ No newline at end of file diff --git a/samples/coredns-configmap.yaml b/samples/coredns-configmap.yaml new file mode 100644 index 00000000..4eb3ea24 --- /dev/null +++ b/samples/coredns-configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +data: + Corefile: | + .:53 { + errors + health + multicluster clusterset.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + annotations: + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + name: coredns + namespace: kube-system \ No newline at end of file diff --git a/samples/coredns-deployment.yaml b/samples/coredns-deployment.yaml new file mode 100644 index 00000000..5039a0f8 --- /dev/null +++ b/samples/coredns-deployment.yaml @@ -0,0 +1,137 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + progressDeadlineSeconds: 600 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + eks.amazonaws.com/compute-type: ec2 + creationTimestamp: null + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.4 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + - mountPath: /tmp + name: tmp + dnsPolicy: Default + priorityClassName: system-cluster-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: coredns + serviceAccountName: coredns + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + volumes: + - emptyDir: {} + name: tmp + - configMap: + defaultMode: 420 + items: + - key: Corefile + path: Corefile + name: coredns + name: config-volume \ No newline at end of file From 48ed001bdf83524c1daba6f7051770a8769f16aa Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Tue, 19 Apr 2022 19:10:37 -0700 Subject: [PATCH 064/163] Patch release v0.2.3 (#129) Co-authored-by: Dustin Hemmerling --- .github/workflows/build.yml | 12 +++++++----- Makefile | 2 +- config/controller_install_release/kustomization.yaml | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72f04bad..7dbed56b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,25 +11,27 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.17 + - name: checkout + uses: actions/checkout@v3 + - name: Unit tests run: make test - name: Upload code coverage - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: files: cover.out - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: - version: v1.43 + version: v1.45.2 # Optional: if set to true then the action don't cache or restore ~/go/pkg. skip-pkg-cache: true # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. diff --git a/Makefile b/Makefile index 214de487..d8b097a5 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ tidy: golangci-lint: ## Download golangci-lint @mkdir -p $(shell pwd)/bin - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.43.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.45.2 .PHONY: lint lint: golangci-lint ## Run linter diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index 6c3c71b9..ff9558c1 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -4,4 +4,4 @@ bases: images: - name: controller newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s - newTag: v0.2.2 + newTag: v0.2.3 From 060ac2d8e554c2737f9a0e4a6f3c0bd2dd77c344 Mon Sep 17 00:00:00 2001 From: Fred Wang <58385135+fredjywang@users.noreply.github.com> Date: Tue, 17 May 2022 14:28:09 -0700 Subject: [PATCH 065/163] Fix build for multiple Go versions (#132) --- Makefile | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d8b097a5..8fd0c884 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,16 @@ else GOBIN=$(shell go env GOBIN) endif +# Find current user version of Go, set golangci-lint version accordingly +GOLANGCI_VER= 1.43.0 +GO_VER = $(shell go version | awk '{ print $$3 }' | awk -F '.' '{ print $$2 }') + +ifeq ($(shell expr $(GO_VER) \> 17), 1) +GOLANGCI_VER = 1.45.2 +else +GOLANGCI_VER = 1.43.0 +endif + # Setting SHELL to bash allows bash commands to be executed by recipes. # This is a requirement for 'setup-envtest.sh' in the test target. # Options are set to exit when a recipe line exits non-zero or a piped command fails. @@ -62,7 +72,7 @@ tidy: golangci-lint: ## Download golangci-lint @mkdir -p $(shell pwd)/bin - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.45.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v$(GOLANGCI_VER) .PHONY: lint lint: golangci-lint ## Run linter @@ -150,7 +160,7 @@ controller-gen: ## Download controller-gen locally if necessary. KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. - $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.4) MOCKGEN = $(shell pwd)/bin/mockgen mockgen: ## Download mockgen @@ -160,6 +170,12 @@ KIND = $(shell pwd)/bin/kind kind: ## Download kind $(call go-get-tool,$(KIND),sigs.k8s.io/kind@v0.11.1) + +GOGET_CMD = "install" +ifeq ($(shell expr $(GO_VER) \< 16), 1) +GOGET_CMD = "get" +endif + # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) define go-get-tool @@ -169,7 +185,7 @@ TMP_DIR=$$(mktemp -d) ;\ cd $$TMP_DIR ;\ go mod init tmp ;\ echo "Downloading $(2)" ;\ -GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ +GOBIN=$(PROJECT_DIR)/bin go $(GOGET_CMD) $(2) ;\ rm -rf $$TMP_DIR ;\ } endef From 4cbe722517cba3f378ee24c0736238af823b2dfe Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 19 May 2022 10:59:37 -0700 Subject: [PATCH 066/163] Update dependencies and move to go1.17 (#133) --- Makefile | 14 +- .../multicluster.x-k8s.io_serviceexports.yaml | 2 +- .../multicluster.x-k8s.io_serviceimports.yaml | 2 +- go.mod | 104 ++- go.sum | 626 +++++++++++++----- integration/scripts/common.sh | 2 +- pkg/cloudmap/api_test.go | 4 +- pkg/cloudmap/cache_test.go | 4 +- pkg/cloudmap/client_test.go | 4 +- pkg/cloudmap/operation_poller_test.go | 4 +- pkg/controllers/cloudmap_controller_test.go | 4 +- .../serviceexport_controller_test.go | 4 +- 12 files changed, 558 insertions(+), 216 deletions(-) diff --git a/Makefile b/Makefile index 8fd0c884..fd1c76cf 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ test: manifests generate generate-mocks fmt vet test-setup ## Run tests. test-setup: ## setup test environment mkdir -p ${ENVTEST_ASSETS_DIR} - test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh + test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR) integration-suite: ## Provision and run integration tests with cleanup @@ -156,7 +156,7 @@ generate-mocks: mockgen CONTROLLER_GEN = $(shell pwd)/bin/controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.2) KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. @@ -168,13 +168,7 @@ mockgen: ## Download mockgen KIND = $(shell pwd)/bin/kind kind: ## Download kind - $(call go-get-tool,$(KIND),sigs.k8s.io/kind@v0.11.1) - - -GOGET_CMD = "install" -ifeq ($(shell expr $(GO_VER) \< 16), 1) -GOGET_CMD = "get" -endif + $(call go-get-tool,$(KIND),sigs.k8s.io/kind@v0.13.0) # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) @@ -185,7 +179,7 @@ TMP_DIR=$$(mktemp -d) ;\ cd $$TMP_DIR ;\ go mod init tmp ;\ echo "Downloading $(2)" ;\ -GOBIN=$(PROJECT_DIR)/bin go $(GOGET_CMD) $(2) ;\ +GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ rm -rf $$TMP_DIR ;\ } endef diff --git a/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml b/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml index b0303f67..3dd6dd26 100644 --- a/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: serviceexports.multicluster.x-k8s.io spec: diff --git a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml index ec3fafb9..4e48acb4 100644 --- a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: serviceimports.multicluster.x-k8s.io spec: diff --git a/go.mod b/go.mod index ff8a208b..ab4cc45b 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,98 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s -go 1.15 +go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.8.1 - github.com/aws/aws-sdk-go-v2/config v1.6.1 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.7.3 - github.com/go-logr/logr v0.3.0 + github.com/aws/aws-sdk-go-v2 v1.16.4 + github.com/aws/aws-sdk-go-v2/config v1.15.7 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5 + github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 - github.com/onsi/ginkgo v1.14.1 - github.com/onsi/gomega v1.10.2 - github.com/stretchr/testify v1.6.1 - k8s.io/api v0.20.2 - k8s.io/apimachinery v0.20.2 - k8s.io/client-go v0.20.2 - sigs.k8s.io/controller-runtime v0.8.3 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.18.1 + github.com/pkg/errors v0.9.1 + github.com/stretchr/testify v1.7.0 + k8s.io/api v0.24.0 + k8s.io/apimachinery v0.24.0 + k8s.io/client-go v0.24.0 + sigs.k8s.io/controller-runtime v0.12.1 ) -replace github.com/spf13/viper v1.4.0 => github.com/spf13/viper v1.8.0 +require ( + cloud.google.com/go v0.81.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 // indirect + github.com/aws/smithy-go v1.11.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/zapr v1.2.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-cmp v0.5.7 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.12.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.1 // indirect + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/apiextensions-apiserver v0.24.0 // indirect + k8s.io/component-base v0.24.0 // indirect + k8s.io/klog/v2 v2.60.1 // indirect + k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/go.sum b/go.sum index 01799e08..98a0080f 100644 --- a/go.sum +++ b/go.sum @@ -8,163 +8,219 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1 h1:eVvIXUKiTgv++6YnWb42DUA1YL7qDugnKP0HljexdnQ= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag= -github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= -github.com/aws/aws-sdk-go-v2/config v1.6.1 h1:qrZINaORyr78syO1zfD4l7r4tZjy0Z1l0sy4jiysyOM= -github.com/aws/aws-sdk-go-v2/config v1.6.1/go.mod h1:t/y3UPu0XEDy0cEw6mvygaBQaPzWiYAxfP2SzgtvclA= -github.com/aws/aws-sdk-go-v2/credentials v1.3.3 h1:A13QPatmUl41SqUfnuT3V0E3XiNGL6qNTOINbE8cZL4= -github.com/aws/aws-sdk-go-v2/credentials v1.3.3/go.mod h1:oVieKMT3m9BSfqhOfuQ+E0j/yN84ZAJ7Qv8Sfume/ak= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1 h1:rc+fRGvlKbeSd9IFhFS1KWBs0XjTkq0CfK5xqyLgIp0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1/go.mod h1:+GTydg3uHmVlQdkRoetz6VHKbOMEYof70m19IpMLifc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1 h1:IkqRRUZTKaS16P2vpX+FNc2jq3JWa3c478gykQp4ow4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1/go.mod h1:Pv3WenDjI0v2Jl7UaMFIIbPOBbhn33RmmAmGgkXDoqY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3 h1:VxFCgxsqWe7OThOwJ5IpFX3xrObtuIH9Hg/NW7oot1Y= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3/go.mod h1:7gcsONBmFoCcKrAqrm95trrMd2+C/ReYKP7Vfu8yHHA= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.7.3 h1:jdoRhOcuqrCbvifZT//qCb+DhCzjVEy6f2NH+ppKP3I= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.7.3/go.mod h1:aukzhWNlyrzDQ2cjZeDj2vFgY2VYN5eMXrQUZwF58go= -github.com/aws/aws-sdk-go-v2/service/sso v1.3.3 h1:K2gCnGvAASpz+jqP9iyr+F/KNjmTYf8aWOtTQzhmZ5w= -github.com/aws/aws-sdk-go-v2/service/sso v1.3.3/go.mod h1:Jgw5O+SK7MZ2Yi9Yvzb4PggAPYaFSliiQuWR0hNjexk= -github.com/aws/aws-sdk-go-v2/service/sts v1.6.2 h1:l504GWCoQi1Pk68vSUFGLmDIEMzRfVGNgLakDK+Uj58= -github.com/aws/aws-sdk-go-v2/service/sts v1.6.2/go.mod h1:RBhoMJB8yFToaCnbe0jNq5Dcdy0jp6LhHqg55rjClkM= -github.com/aws/smithy-go v1.7.0 h1:+cLHMRrDZvQ4wk+KuQ9yH6eEg6KZEJ9RI2IkDqnygCg= -github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/aws-sdk-go-v2 v1.16.4 h1:swQTEQUyJF/UkEA94/Ga55miiKFoXmm/Zd67XHgmjSg= +github.com/aws/aws-sdk-go-v2 v1.16.4/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= +github.com/aws/aws-sdk-go-v2/config v1.15.7 h1:PrzhYjDpWnGSpjedmEapldQKPW4x8cCNzUI8XOho1CM= +github.com/aws/aws-sdk-go-v2/config v1.15.7/go.mod h1:exERlvqx1OoUHrxQpMgrmfSW0H6B1+r3xziZD3bBXRg= +github.com/aws/aws-sdk-go-v2/credentials v1.12.2 h1:tX4EHQFU4+O9at5QjnwIKb/Qgv7MbgbUNtqTRF0Vu2M= +github.com/aws/aws-sdk-go-v2/credentials v1.12.2/go.mod h1:/XWqDVuzclEKvzileqtD7/t+wIhOogv//6JFlKEe0Wc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5 h1:YPxclBeE07HsLQE8vtjC8T2emcTjM9nzqsnDi2fv5UM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5/go.mod h1:WAPnuhG5IQ/i6DETFl5NmX3kKqCzw7aau9NHAGcm4QE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11 h1:gsqHplNh1DaQunEKZISK56wlpbCg0yKxNVvGWCFuF1k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11/go.mod h1:tmUB6jakq5DFNcXsXOA/ZQ7/C8VnSKYkx58OI7Fh79g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5 h1:PLFj+M2PgIDHG//hw3T0O0KLI4itVtAjtxrZx4AHPLg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5/go.mod h1:fV1AaS2gFc1tM0RCb015FJ0pvWVUfJZANzjwoO4YakM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12 h1:j0VqrjtgsY1Bx27tD0ysay36/K4kFMWRp9K3ieO9nLU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12/go.mod h1:00c7+ALdPh4YeEUPXJzyU0Yy01nPGOq2+9rUaz05z9g= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 h1:gRW1ZisKc93EWEORNJRvy/ZydF3o6xLSveJHdi1Oa0U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5/go.mod h1:ZbkttHXaVn3bBo/wpJbQGiiIWR90eTBUVBrEHUEQlho= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5 h1:rwZ+vd0xq8jUj+oXRXOkTvf+wWJtCCmzz1M3UceF5vM= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5/go.mod h1:wue0pRTZvvY6ntvvDdK/bcDAw+pgto9HzBHs+kdun6Q= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.5 h1:TfJ/zuOYvHnxkvohSwAF3Ppn9KT/SrGZuOZHTPy8Guw= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.5/go.mod h1:TFVe6Rr2joVLsYQ1ABACXgOC6lXip/qpX2x5jWg/A9w= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 h1:aYToU0/iazkMY67/BYLt3r6/LT/mUtarLAF5mGof1Kg= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.6/go.mod h1:rP1rEOKAGZoXp4iGDxSXFvODAtXpm34Egf0lL0eshaQ= +github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE= +github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= -github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= -github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -172,6 +228,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -179,47 +236,63 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -235,8 +308,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -244,42 +315,50 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -292,33 +371,44 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= -github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= -github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -326,6 +416,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -333,8 +424,10 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -343,36 +436,43 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -380,53 +480,81 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -448,8 +576,9 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -458,8 +587,10 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -476,8 +607,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -486,22 +617,54 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -517,13 +680,12 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -535,28 +697,61 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -570,15 +765,12 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -595,18 +787,38 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -616,14 +828,26 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -641,16 +865,57 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -660,14 +925,17 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -681,58 +949,60 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= -k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8txgVQ= -k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= -k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= -k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= -k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= +k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= +k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= +k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= +k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= +k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= +k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= +k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= +k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= +k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= -k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= -sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= +sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= +sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/integration/scripts/common.sh b/integration/scripts/common.sh index 3c94c44d..226fee70 100755 --- a/integration/scripts/common.sh +++ b/integration/scripts/common.sh @@ -11,6 +11,6 @@ ENDPT_PORT=80 SERVICE_PORT=8080 KIND_SHORT='cloud-map-e2e' CLUSTER='kind-cloud-map-e2e' -IMAGE='kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729' +IMAGE='kindest/node:v1.19.16@sha256:dec41184d10deca01a08ea548197b77dc99eeacb56ff3e371af3193c86ca99f4' EXPECTED_ENDPOINT_COUNT=5 UPDATED_ENDPOINT_COUNT=6 diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 6207dfb9..8d70d9c0 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -12,7 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - testingLogger "github.com/go-logr/logr/testing" + "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" ) @@ -322,7 +322,7 @@ func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmapMock.MockAwsFacade) ServiceDiscoveryApi { return &serviceDiscoveryApi{ - log: common.NewLoggerWithLogr(testingLogger.TestLogger{T: t}), + log: common.NewLoggerWithLogr(testr.New(t)), awsFacade: awsFacade, } } diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index 11c65ada..89968f42 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" - testing2 "github.com/go-logr/logr/testing" + "github.com/go-logr/logr/testr" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/cache" ) @@ -156,7 +156,7 @@ func TestServiceDiscoveryClientEvictEndpoints(t *testing.T) { func getCacheImpl(t *testing.T) sdCache { return sdCache{ - log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), + log: common.NewLoggerWithLogr(testr.New(t)), cache: cache.NewLRUExpireCache(defaultCacheSize), } } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 9a21dcd9..c22b580e 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -10,7 +10,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - testing2 "github.com/go-logr/logr/testing" + "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -347,7 +347,7 @@ func getTestSdClient(t *testing.T) *testSdClient { mockApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) return &testSdClient{ client: &serviceDiscoveryClient{ - log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), + log: common.NewLoggerWithLogr(testr.New(t)), sdApi: mockApi, cache: mockCache, }, diff --git a/pkg/cloudmap/operation_poller_test.go b/pkg/cloudmap/operation_poller_test.go index 2e81c9bc..08bcc7c6 100644 --- a/pkg/cloudmap/operation_poller_test.go +++ b/pkg/cloudmap/operation_poller_test.go @@ -11,7 +11,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - testing2 "github.com/go-logr/logr/testing" + "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" ) @@ -187,7 +187,7 @@ func TestOperationPoller_PollTimeout(t *testing.T) { sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) p := operationPoller{ - log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), + log: common.NewLoggerWithLogr(testr.New(t)), sdApi: sdApi, timeout: 2 * time.Millisecond, opIds: []string{test.OpId1, test.OpId2}, diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index 4f944995..023bd7db 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -10,7 +10,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" - testingLogger "github.com/go-logr/logr/testing" + "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -76,6 +76,6 @@ func getReconciler(t *testing.T, mockSDClient *cloudmapMock.MockServiceDiscovery return &CloudMapReconciler{ Client: client, Cloudmap: mockSDClient, - Log: common.NewLoggerWithLogr(testingLogger.TestLogger{T: t}), + Log: common.NewLoggerWithLogr(testr.New(t)), } } diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/serviceexport_controller_test.go index 5a30dd8a..d158c421 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/serviceexport_controller_test.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" - testing2 "github.com/go-logr/logr/testing" + "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "testing" @@ -170,7 +170,7 @@ func getServiceExportScheme() *runtime.Scheme { func getServiceExportReconciler(t *testing.T, mockClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *ServiceExportReconciler { return &ServiceExportReconciler{ Client: client, - Log: common.NewLoggerWithLogr(testing2.TestLogger{T: t}), + Log: common.NewLoggerWithLogr(testr.New(t)), Scheme: client.Scheme(), CloudMap: mockClient, } From 3ec2adb817ecfe639b78902503dfbc27406c8eca Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 19 May 2022 14:45:18 -0700 Subject: [PATCH 067/163] Update CodeQL to v2 (#136) --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fad209fb..802ce6b5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From 661806ad8e775d7db990947d8649fb3e2b7a2323 Mon Sep 17 00:00:00 2001 From: Fred Wang <58385135+fredjywang@users.noreply.github.com> Date: Thu, 19 May 2022 14:54:07 -0700 Subject: [PATCH 068/163] Revert fix build for older Go versions (#137) --- Makefile | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Makefile b/Makefile index fd1c76cf..1640fba9 100644 --- a/Makefile +++ b/Makefile @@ -16,16 +16,6 @@ else GOBIN=$(shell go env GOBIN) endif -# Find current user version of Go, set golangci-lint version accordingly -GOLANGCI_VER= 1.43.0 -GO_VER = $(shell go version | awk '{ print $$3 }' | awk -F '.' '{ print $$2 }') - -ifeq ($(shell expr $(GO_VER) \> 17), 1) -GOLANGCI_VER = 1.45.2 -else -GOLANGCI_VER = 1.43.0 -endif - # Setting SHELL to bash allows bash commands to be executed by recipes. # This is a requirement for 'setup-envtest.sh' in the test target. # Options are set to exit when a recipe line exits non-zero or a piped command fails. @@ -72,7 +62,7 @@ tidy: golangci-lint: ## Download golangci-lint @mkdir -p $(shell pwd)/bin - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v$(GOLANGCI_VER) + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.45.2 .PHONY: lint lint: golangci-lint ## Run linter From d2df88b2ec4cc39a3761512c63695a5145e41fd4 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Tue, 24 May 2022 14:18:09 -0700 Subject: [PATCH 069/163] Upgrade controller-gen to v0.8 (#138) --- Makefile | 6 ++---- .../multicluster.x-k8s.io_serviceexports.yaml | 16 +++++++--------- .../multicluster.x-k8s.io_serviceimports.yaml | 3 +-- config/rbac/role.yaml | 1 - 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 1640fba9..7765a9d0 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,6 @@ PKG:=github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version # Image URL to use all building/pushing image targets IMG ?= controller:latest -# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) -CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" # AWS Region AWS_REGION ?= us-east-1 @@ -43,7 +41,7 @@ help: ## Display this help. ##@ Development manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) crd rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." @@ -146,7 +144,7 @@ generate-mocks: mockgen CONTROLLER_GEN = $(shell pwd)/bin/controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.2) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0) KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. diff --git a/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml b/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml index 3dd6dd26..0703a5fe 100644 --- a/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.6.2 + controller-gen.kubebuilder.io/version: v0.8.0 creationTimestamp: null name: serviceexports.multicluster.x-k8s.io spec: @@ -45,13 +44,12 @@ spec: description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, - type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: - \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type - \ // +patchStrategy=merge // +listType=map // +listMapKey=type - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` - \n // other fields }" + type FooStatus struct{ // Represents the observations of a foo's + current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" properties: lastTransitionTime: description: lastTransitionTime is the last time the condition diff --git a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml index 4e48acb4..17bac395 100644 --- a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.6.2 + controller-gen.kubebuilder.io/version: v0.8.0 creationTimestamp: null name: serviceimports.multicluster.x-k8s.io spec: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 921e22ba..307f5ea8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,4 +1,3 @@ - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole From 80d6271fa6481e259ef07742af0de28247a45464 Mon Sep 17 00:00:00 2001 From: Matthew Goodman Date: Tue, 14 Jun 2022 10:58:04 -0700 Subject: [PATCH 070/163] Added managed by label - issue 110 (#139) * Added managed by label - issue 110 * Update utils.go * rename constants * Update cloudmap_controller_test.go --- pkg/controllers/cloudmap_controller_test.go | 1 + pkg/controllers/utils.go | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/cloudmap_controller_test.go index 023bd7db..f4c97e7e 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/cloudmap_controller_test.go @@ -68,6 +68,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { assert.NoError(t, err) endpointSlice := endpointSliceList.Items[0] assert.Equal(t, test.SvcName, endpointSlice.Labels["multicluster.kubernetes.io/service-name"], "Endpoint slice is created") + assert.Contains(t, endpointSlice.Labels, LabelEndpointSliceManagedBy, "Managed by label is added") assert.Equal(t, int32(test.Port1), *endpointSlice.Ports[0].Port) assert.Equal(t, test.EndptIp1, endpointSlice.Endpoints[0].Addresses[0]) } diff --git a/pkg/controllers/utils.go b/pkg/controllers/utils.go index 517a8f92..5465a3c6 100644 --- a/pkg/controllers/utils.go +++ b/pkg/controllers/utils.go @@ -20,6 +20,12 @@ const ( // LabelServiceImportName indicates the name of the multi-cluster service that an EndpointSlice belongs to. LabelServiceImportName = "multicluster.kubernetes.io/service-name" + + // LabelEndpointSliceManagedBy indicates the name of the entity that manages the EndpointSlice. + LabelEndpointSliceManagedBy = "endpointslice.kubernetes.io/managed-by" + + // ValueEndpointSliceManagedBy indicates the name of the entity that manages the EndpointSlice. + ValueEndpointSliceManagedBy = "aws-cloud-map-mcs-controller-for-k8s" ) // ServicePortToPort converts a k8s service port to internal model port @@ -203,8 +209,12 @@ func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string) *discovery return &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - discovery.LabelServiceName: svc.Name, // derived Service name - LabelServiceImportName: svcImportName, // original ServiceImport name + // derived Service name + discovery.LabelServiceName: svc.Name, + // original ServiceImport name + LabelServiceImportName: svcImportName, + // 'managed-by' label set to controller + LabelEndpointSliceManagedBy: ValueEndpointSliceManagedBy, }, GenerateName: svc.Name + "-", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(svc, schema.GroupVersionKind{ From 5be8d91ab6079d198ff2ed11becbe0a52a6782ad Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Tue, 21 Jun 2022 15:47:23 -0700 Subject: [PATCH 071/163] Create dependabot.yml (#142) --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..b444581e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" From 5edbd7543023a716de003f563ef9ad98d706b66a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:12:45 -0700 Subject: [PATCH 072/163] Bump k8s.io/client-go from 0.24.0 to 0.24.2 (#146) Bumps [k8s.io/client-go](https://github.com/kubernetes/client-go) from 0.24.0 to 0.24.2. - [Release notes](https://github.com/kubernetes/client-go/releases) - [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md) - [Commits](https://github.com/kubernetes/client-go/compare/v0.24.0...v0.24.2) --- updated-dependencies: - dependency-name: k8s.io/client-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ab4cc45b..8e34bd00 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,9 @@ require ( github.com/onsi/gomega v1.18.1 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.0 - k8s.io/api v0.24.0 - k8s.io/apimachinery v0.24.0 - k8s.io/client-go v0.24.0 + k8s.io/api v0.24.2 + k8s.io/apimachinery v0.24.2 + k8s.io/client-go v0.24.2 sigs.k8s.io/controller-runtime v0.12.1 ) diff --git a/go.sum b/go.sum index 98a0080f..e45cad02 100644 --- a/go.sum +++ b/go.sum @@ -969,15 +969,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= +k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= +k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= -k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= +k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= -k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= +k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= +k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= From 074c9938b86ef8cf390e003f88af9470e595c028 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:13:10 -0700 Subject: [PATCH 073/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.15.7 to 1.15.11 (#147) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.15.7 to 1.15.11. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.15.7...config/v1.15.11) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 24 ++++++++++++------------ go.sum | 44 ++++++++++++++++++++++++-------------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 8e34bd00..f08b68fe 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.4 - github.com/aws/aws-sdk-go-v2/config v1.15.7 + github.com/aws/aws-sdk-go-v2 v1.16.5 + github.com/aws/aws-sdk-go-v2/config v1.15.11 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,15 +28,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 // indirect - github.com/aws/smithy-go v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 // indirect + github.com/aws/smithy-go v1.11.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -52,7 +52,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.7 // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect diff --git a/go.sum b/go.sum index e45cad02..a2118a02 100644 --- a/go.sum +++ b/go.sum @@ -74,30 +74,34 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.4 h1:swQTEQUyJF/UkEA94/Ga55miiKFoXmm/Zd67XHgmjSg= github.com/aws/aws-sdk-go-v2 v1.16.4/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2/config v1.15.7 h1:PrzhYjDpWnGSpjedmEapldQKPW4x8cCNzUI8XOho1CM= -github.com/aws/aws-sdk-go-v2/config v1.15.7/go.mod h1:exERlvqx1OoUHrxQpMgrmfSW0H6B1+r3xziZD3bBXRg= -github.com/aws/aws-sdk-go-v2/credentials v1.12.2 h1:tX4EHQFU4+O9at5QjnwIKb/Qgv7MbgbUNtqTRF0Vu2M= -github.com/aws/aws-sdk-go-v2/credentials v1.12.2/go.mod h1:/XWqDVuzclEKvzileqtD7/t+wIhOogv//6JFlKEe0Wc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5 h1:YPxclBeE07HsLQE8vtjC8T2emcTjM9nzqsnDi2fv5UM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5/go.mod h1:WAPnuhG5IQ/i6DETFl5NmX3kKqCzw7aau9NHAGcm4QE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11 h1:gsqHplNh1DaQunEKZISK56wlpbCg0yKxNVvGWCFuF1k= +github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= +github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= +github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= +github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI= +github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+xxqXOYmoObm33ws= +github.com/aws/aws-sdk-go-v2/credentials v1.12.6/go.mod h1:mQgnRmBPF2S/M01W4T4Obp3ZaZB6o1s/R8cOUda9vtI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 h1:+NZzDh/RpcQTpo9xMFUgkseIam6PC+YJbdhbQp1NOXI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6/go.mod h1:ClLMcuQA/wcHPmOIfNzNI4Y1Q0oDbmEkbYhMFOzHDh8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11/go.mod h1:tmUB6jakq5DFNcXsXOA/ZQ7/C8VnSKYkx58OI7Fh79g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5 h1:PLFj+M2PgIDHG//hw3T0O0KLI4itVtAjtxrZx4AHPLg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5/go.mod h1:fV1AaS2gFc1tM0RCb015FJ0pvWVUfJZANzjwoO4YakM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12 h1:j0VqrjtgsY1Bx27tD0ysay36/K4kFMWRp9K3ieO9nLU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12/go.mod h1:00c7+ALdPh4YeEUPXJzyU0Yy01nPGOq2+9rUaz05z9g= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 h1:gRW1ZisKc93EWEORNJRvy/ZydF3o6xLSveJHdi1Oa0U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5/go.mod h1:ZbkttHXaVn3bBo/wpJbQGiiIWR90eTBUVBrEHUEQlho= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 h1:L/l0WbIpIadRO7i44jZh1/XeXpNDX0sokFppb4ZnXUI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13/go.mod h1:hiM/y1XPp3DoEPhoVEYc/CZcS58dP6RKJRDFp99wdX0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 h1:0ZxYAZ1cn7Swi/US55VKciCE6RhRHIwCKIWaMLdT6pg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6/go.mod h1:DxAPjquoEHf3rUHh1b9+47RAaXB8/7cB6jkzCt/GOEI= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5 h1:rwZ+vd0xq8jUj+oXRXOkTvf+wWJtCCmzz1M3UceF5vM= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5/go.mod h1:wue0pRTZvvY6ntvvDdK/bcDAw+pgto9HzBHs+kdun6Q= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.5 h1:TfJ/zuOYvHnxkvohSwAF3Ppn9KT/SrGZuOZHTPy8Guw= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.5/go.mod h1:TFVe6Rr2joVLsYQ1ABACXgOC6lXip/qpX2x5jWg/A9w= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 h1:aYToU0/iazkMY67/BYLt3r6/LT/mUtarLAF5mGof1Kg= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.6/go.mod h1:rP1rEOKAGZoXp4iGDxSXFvODAtXpm34Egf0lL0eshaQ= -github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WGE1PT2BYDwcYhJg8= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= +github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -259,8 +263,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -815,7 +820,6 @@ golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= From 4067da49d9cb0a99f900ea29ece4a752ace55c92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:21:51 -0700 Subject: [PATCH 074/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#145) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.5 to 1.17.6. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.5...service/ecr/v1.17.6) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f08b68fe..a8836716 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.5 github.com/aws/aws-sdk-go-v2/config v1.15.11 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/onsi/ginkgo v1.16.5 diff --git a/go.sum b/go.sum index a2118a02..33b840e3 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.4/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= @@ -83,23 +82,20 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+ github.com/aws/aws-sdk-go-v2/credentials v1.12.6/go.mod h1:mQgnRmBPF2S/M01W4T4Obp3ZaZB6o1s/R8cOUda9vtI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 h1:+NZzDh/RpcQTpo9xMFUgkseIam6PC+YJbdhbQp1NOXI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6/go.mod h1:ClLMcuQA/wcHPmOIfNzNI4Y1Q0oDbmEkbYhMFOzHDh8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11/go.mod h1:tmUB6jakq5DFNcXsXOA/ZQ7/C8VnSKYkx58OI7Fh79g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5/go.mod h1:fV1AaS2gFc1tM0RCb015FJ0pvWVUfJZANzjwoO4YakM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 h1:L/l0WbIpIadRO7i44jZh1/XeXpNDX0sokFppb4ZnXUI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13/go.mod h1:hiM/y1XPp3DoEPhoVEYc/CZcS58dP6RKJRDFp99wdX0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 h1:0ZxYAZ1cn7Swi/US55VKciCE6RhRHIwCKIWaMLdT6pg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6/go.mod h1:DxAPjquoEHf3rUHh1b9+47RAaXB8/7cB6jkzCt/GOEI= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5 h1:rwZ+vd0xq8jUj+oXRXOkTvf+wWJtCCmzz1M3UceF5vM= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.5/go.mod h1:wue0pRTZvvY6ntvvDdK/bcDAw+pgto9HzBHs+kdun6Q= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6 h1:M6dN81eNXH7xVR9N43UWObDt8FWI49iUH/FI3bG1bHc= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6/go.mod h1:WwRCfkmgBKYIqqpSlSH3fVGPqgUyjSVCQez4RWekxJE= github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WGE1PT2BYDwcYhJg8= github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= -github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -263,7 +259,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= From 513a79df6b070032644683239c84de751f426701 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jun 2022 07:59:19 -0700 Subject: [PATCH 075/163] Bump github.com/stretchr/testify from 1.7.0 to 1.7.4 (#148) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.0 to 1.7.4. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.7.0...v1.7.4) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a8836716..b34e0d02 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.18.1 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.4 k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 @@ -86,7 +86,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.24.0 // indirect k8s.io/component-base v0.24.0 // indirect k8s.io/klog/v2 v2.60.1 // indirect diff --git a/go.sum b/go.sum index 33b840e3..067bae31 100644 --- a/go.sum +++ b/go.sum @@ -480,13 +480,16 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.4 h1:wZRexSlwd7ZXfKINDLsO4r7WBt3gTKONc6K/VesHvHM= +github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -957,8 +960,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 9c0e4c9822734ebf6771b77e6182e7b2a4ecccc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jun 2022 08:08:54 -0700 Subject: [PATCH 076/163] Bump github.com/onsi/gomega from 1.18.1 to 1.19.0 (#149) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.18.1 to 1.19.0. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.18.1...v1.19.0) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index b34e0d02..79d73e38 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.18.1 + github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.4 k8s.io/api v0.24.2 @@ -74,7 +74,7 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect diff --git a/go.sum b/go.sum index 067bae31..53b3bec6 100644 --- a/go.sum +++ b/go.sum @@ -400,14 +400,16 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -642,8 +644,9 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From 5220cb586070943b9c6e3134e62e89a92da43149 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 13:08:23 -0700 Subject: [PATCH 077/163] Bump github.com/stretchr/testify from 1.7.4 to 1.7.5 (#151) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.4 to 1.7.5. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.7.4...v1.7.5) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 79d73e38..553ad4a2 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.7.4 + github.com/stretchr/testify v1.7.5 k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 diff --git a/go.sum b/go.sum index 53b3bec6..220e967b 100644 --- a/go.sum +++ b/go.sum @@ -490,8 +490,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.4 h1:wZRexSlwd7ZXfKINDLsO4r7WBt3gTKONc6K/VesHvHM= -github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= From 902d8642decfcfb3d2949bb85c4a6ca4ec3c6217 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 13:09:29 -0700 Subject: [PATCH 078/163] Bump sigs.k8s.io/controller-runtime from 0.12.1 to 0.12.2 (#152) Bumps [sigs.k8s.io/controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) from 0.12.1 to 0.12.2. - [Release notes](https://github.com/kubernetes-sigs/controller-runtime/releases) - [Changelog](https://github.com/kubernetes-sigs/controller-runtime/blob/master/RELEASE.md) - [Commits](https://github.com/kubernetes-sigs/controller-runtime/compare/v0.12.1...v0.12.2) --- updated-dependencies: - dependency-name: sigs.k8s.io/controller-runtime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 19 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 553ad4a2..7a681e33 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 - sigs.k8s.io/controller-runtime v0.12.1 + sigs.k8s.io/controller-runtime v0.12.2 ) require ( @@ -87,8 +87,8 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.24.0 // indirect - k8s.io/component-base v0.24.0 // indirect + k8s.io/apiextensions-apiserver v0.24.2 // indirect + k8s.io/component-base v0.24.2 // indirect k8s.io/klog/v2 v2.60.1 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect diff --git a/go.sum b/go.sum index 220e967b..1ee146f4 100644 --- a/go.sum +++ b/go.sum @@ -975,21 +975,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= -k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= -k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= +k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= -k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= +k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= -k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= -k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= +k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= +k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -1005,8 +1002,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= -sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= +sigs.k8s.io/controller-runtime v0.12.2 h1:nqV02cvhbAj7tbt21bpPpTByrXGn2INHRsi39lXy9sE= +sigs.k8s.io/controller-runtime v0.12.2/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= From e06be9a1f95414d489ad7c48638501e519a2f3e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Jul 2022 10:25:14 -0700 Subject: [PATCH 079/163] Bump github.com/aws/aws-sdk-go-v2 from 1.16.5 to 1.16.6 (#157) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.16.5 to 1.16.6. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.16.5...v1.16.6) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7a681e33..82d2d8f8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.5 + github.com/aws/aws-sdk-go-v2 v1.16.6 github.com/aws/aws-sdk-go-v2/config v1.15.11 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6 github.com/go-logr/logr v1.2.3 @@ -36,7 +36,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 // indirect - github.com/aws/smithy-go v1.11.3 // indirect + github.com/aws/smithy-go v1.12.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 1ee146f4..77a8057d 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,9 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= +github.com/aws/aws-sdk-go-v2 v1.16.6 h1:kzafGZYwkwVgLZ2zEX7P+vTwLli6uIMXF8aGjunN6UI= +github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI= github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+xxqXOYmoObm33ws= @@ -96,8 +97,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WG github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= -github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0= +github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 98e57b3f2acd335f26d724d1669513537b295765 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Jul 2022 10:39:33 -0700 Subject: [PATCH 080/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.15.11 to 1.15.12 (#156) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.15.11 to 1.15.12. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.15.11...config/v1.15.12) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 34 ++++++++++++++++++---------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 82d2d8f8..98f07e27 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.6 - github.com/aws/aws-sdk-go-v2/config v1.15.11 + github.com/aws/aws-sdk-go-v2/config v1.15.12 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,14 +28,14 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 // indirect github.com/aws/smithy-go v1.12.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 77a8057d..9625ce55 100644 --- a/go.sum +++ b/go.sum @@ -77,26 +77,28 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= github.com/aws/aws-sdk-go-v2 v1.16.6 h1:kzafGZYwkwVgLZ2zEX7P+vTwLli6uIMXF8aGjunN6UI= github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= -github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= -github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI= -github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+xxqXOYmoObm33ws= -github.com/aws/aws-sdk-go-v2/credentials v1.12.6/go.mod h1:mQgnRmBPF2S/M01W4T4Obp3ZaZB6o1s/R8cOUda9vtI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 h1:+NZzDh/RpcQTpo9xMFUgkseIam6PC+YJbdhbQp1NOXI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6/go.mod h1:ClLMcuQA/wcHPmOIfNzNI4Y1Q0oDbmEkbYhMFOzHDh8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= +github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA= +github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+vd2p9BkpJ72BdyzA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 h1:8yi2ORCwXpXEPnj0vP3DjYhejwDQD/5klgBoxXcKOxY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 h1:WuQ1yGs3TMJgxpGVLspcsU/5q1omSA0SG6Cu0yZ4jkM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 h1:L/l0WbIpIadRO7i44jZh1/XeXpNDX0sokFppb4ZnXUI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13/go.mod h1:hiM/y1XPp3DoEPhoVEYc/CZcS58dP6RKJRDFp99wdX0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 h1:0ZxYAZ1cn7Swi/US55VKciCE6RhRHIwCKIWaMLdT6pg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6/go.mod h1:DxAPjquoEHf3rUHh1b9+47RAaXB8/7cB6jkzCt/GOEI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 h1:mCeDDYeDXp3loo/xKi7nkx34eeh7q3n1mUBtzptsj8c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 h1:bJv4Y9QOiW0GZPStgLgpGrpdfRDSR3XM4V4M3YCQRZo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 h1:M7/BzQNsu0XXiJRe3gUn8UA8tExF6kLMAfvo5PT/KJY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6 h1:M6dN81eNXH7xVR9N43UWObDt8FWI49iUH/FI3bG1bHc= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6/go.mod h1:WwRCfkmgBKYIqqpSlSH3fVGPqgUyjSVCQez4RWekxJE= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WGE1PT2BYDwcYhJg8= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 h1:icon5WWg9Yg5nkB0pJF6bfKw6M0xozukeGKSNKtnqzw= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 h1:GLGfpqX+1bmjNvUJkwB1ZaDpNFXQwJ3z9RkQDA58OBY= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0= github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= From c61174ba64613eae03c169da5aeb4838b607aaff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Jul 2022 10:54:30 -0700 Subject: [PATCH 081/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#158) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.6 to 1.17.7. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.6...service/ecr/v1.17.7) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 98f07e27..16a213ef 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.6 github.com/aws/aws-sdk-go-v2/config v1.15.12 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/onsi/ginkgo v1.16.5 diff --git a/go.sum b/go.sum index 9625ce55..0b16d937 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= github.com/aws/aws-sdk-go-v2 v1.16.6 h1:kzafGZYwkwVgLZ2zEX7P+vTwLli6uIMXF8aGjunN6UI= github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA= @@ -83,23 +82,20 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+v github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 h1:8yi2ORCwXpXEPnj0vP3DjYhejwDQD/5klgBoxXcKOxY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 h1:WuQ1yGs3TMJgxpGVLspcsU/5q1omSA0SG6Cu0yZ4jkM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 h1:mCeDDYeDXp3loo/xKi7nkx34eeh7q3n1mUBtzptsj8c= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 h1:bJv4Y9QOiW0GZPStgLgpGrpdfRDSR3XM4V4M3YCQRZo= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 h1:M7/BzQNsu0XXiJRe3gUn8UA8tExF6kLMAfvo5PT/KJY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6 h1:M6dN81eNXH7xVR9N43UWObDt8FWI49iUH/FI3bG1bHc= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.6/go.mod h1:WwRCfkmgBKYIqqpSlSH3fVGPqgUyjSVCQez4RWekxJE= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7 h1:oBA2kHlC24mLutk2cpJv7sb2qrDLrWAiBNLQ2kQGp5g= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7/go.mod h1:NWl7ZPKL3eHMeJTEjOnF/iWhM8HKJdb8z+tFuawoGhk= github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 h1:icon5WWg9Yg5nkB0pJF6bfKw6M0xozukeGKSNKtnqzw= github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94= github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 h1:GLGfpqX+1bmjNvUJkwB1ZaDpNFXQwJ3z9RkQDA58OBY= github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY= -github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0= github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From ff9b6ff39a721705cb7aa813619e753c01f67c58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Jul 2022 11:15:32 -0700 Subject: [PATCH 082/163] Bump github.com/stretchr/testify from 1.7.5 to 1.8.0 (#155) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.5 to 1.8.0. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.7.5...v1.8.0) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 16a213ef..60237f06 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.7.5 + github.com/stretchr/testify v1.8.0 k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 diff --git a/go.sum b/go.sum index 0b16d937..8ecca462 100644 --- a/go.sum +++ b/go.sum @@ -490,8 +490,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= From 5b248a195f496973e95f7735ab691e554cb98566 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 12:31:32 -0700 Subject: [PATCH 083/163] Bump github.com/aws/aws-sdk-go-v2 from 1.16.6 to 1.16.7 (#159) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.16.6 to 1.16.7. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.16.6...v1.16.7) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 60237f06..5a1b0bbf 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.6 + github.com/aws/aws-sdk-go-v2 v1.16.7 github.com/aws/aws-sdk-go-v2/config v1.15.12 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7 github.com/go-logr/logr v1.2.3 diff --git a/go.sum b/go.sum index 8ecca462..b1f22b0d 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,9 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.6 h1:kzafGZYwkwVgLZ2zEX7P+vTwLli6uIMXF8aGjunN6UI= github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= +github.com/aws/aws-sdk-go-v2 v1.16.7 h1:zfBwXus3u14OszRxGcqCDS4MfMCv10e8SMJ2r8Xm0Ns= +github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA= github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA= github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+vd2p9BkpJ72BdyzA= From 13dc38b3e429c81a03eeeca4eabf8383c8443984 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 13:08:14 -0700 Subject: [PATCH 084/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.15.12 to 1.15.13 (#160) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.15.12 to 1.15.13. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.15.12...config/v1.15.13) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 34 ++++++++++++++++++---------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 5a1b0bbf..016825ca 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.7 - github.com/aws/aws-sdk-go-v2/config v1.15.12 + github.com/aws/aws-sdk-go-v2/config v1.15.13 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,14 +28,14 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.9 // indirect github.com/aws/smithy-go v1.12.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index b1f22b0d..c0befe13 100644 --- a/go.sum +++ b/go.sum @@ -77,26 +77,28 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2 v1.16.7 h1:zfBwXus3u14OszRxGcqCDS4MfMCv10e8SMJ2r8Xm0Ns= github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= -github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA= -github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA= -github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+vd2p9BkpJ72BdyzA= -github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 h1:8yi2ORCwXpXEPnj0vP3DjYhejwDQD/5klgBoxXcKOxY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 h1:WuQ1yGs3TMJgxpGVLspcsU/5q1omSA0SG6Cu0yZ4jkM= +github.com/aws/aws-sdk-go-v2/config v1.15.13 h1:CJH9zn/Enst7lDiGpoguVt0lZr5HcpNVlRJWbJ6qreo= +github.com/aws/aws-sdk-go-v2/config v1.15.13/go.mod h1:AcMu50uhV6wMBUlURnEXhr9b3fX6FLSTlEV89krTEGk= +github.com/aws/aws-sdk-go-v2/credentials v1.12.8 h1:niTa7zc7uyOP2ufri0jPESBt1h9yP3Zc0q+xzih3h8o= +github.com/aws/aws-sdk-go-v2/credentials v1.12.8/go.mod h1:P2Hd4Sy7mXRxPNcQMPBmqszSJoDXexX8XEDaT6lucO0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 h1:VfBdn2AxwMbFyJN/lF/xuT3SakomJ86PZu3rCxb5K0s= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8/go.mod h1:oL1Q3KuCq1D4NykQnIvtRiBGLUXhcpY5pl6QZB2XEPU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 h1:mCeDDYeDXp3loo/xKi7nkx34eeh7q3n1mUBtzptsj8c= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 h1:2C0pYHcUBmdzPj+EKNC4qj97oK6yjrUhc1KoSodglvk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14/go.mod h1:kdjrMwHwrC3+FsKhNcCMJ7tUVj/8uSD5CZXeQ4wV6fM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 h1:bJv4Y9QOiW0GZPStgLgpGrpdfRDSR3XM4V4M3YCQRZo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 h1:M7/BzQNsu0XXiJRe3gUn8UA8tExF6kLMAfvo5PT/KJY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8 h1:2J+jdlBJWEmTyAwC82Ym68xCykIvnSnIN18b8xHGlcc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8/go.mod h1:ZIV8GYoC6WLBW5KGs+o4rsc65/ozd+eQ0L31XF5VDwk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 h1:QquxR7NH3ULBsKC+NoTpilzbKKS+5AELfNREInbhvas= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15/go.mod h1:Tkrthp/0sNBShQQsamR7j/zY4p19tVTAs+nnqhH6R3c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 h1:oKnAXxSF2FUvfgw8uzU/v9OTYorJJZ8eBmWhr9TWVVQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8/go.mod h1:rDVhIMAX9N2r8nWxDUlbubvvaFMnfsm+3jAV7q+rpM4= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7 h1:oBA2kHlC24mLutk2cpJv7sb2qrDLrWAiBNLQ2kQGp5g= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7/go.mod h1:NWl7ZPKL3eHMeJTEjOnF/iWhM8HKJdb8z+tFuawoGhk= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 h1:icon5WWg9Yg5nkB0pJF6bfKw6M0xozukeGKSNKtnqzw= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 h1:GLGfpqX+1bmjNvUJkwB1ZaDpNFXQwJ3z9RkQDA58OBY= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.11 h1:XOJWXNFXJyapJqQuCIPfftsOf0XZZioM0kK6OPRt9MY= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.11/go.mod h1:MO4qguFjs3wPGcCSpQ7kOFTwRvb+eu+fn+1vKleGHUk= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.9 h1:yOfILxyjmtr2ubRkRJldlHDFBhf5vw4CzhbwWIBmimQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.9/go.mod h1:O1IvkYxr+39hRf960Us6j0x1P8pDqhTX+oXM5kQNl/Y= github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0= github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From fcb26dbe63adbbe58d0b7d682d29a1e7283e9c2d Mon Sep 17 00:00:00 2001 From: Matthew Goodman Date: Thu, 7 Jul 2022 18:09:19 -0700 Subject: [PATCH 085/163] Single Group to Multi-Group Project Structure Migration (#153) * Added managed by label - issue 110 * migration to multigroup structure * updated paths in PROJECT * updated paths in .golangci & .codecov * update v1alpha1 & controllers to include group prefix --- .github/.codecov.yml | 2 +- .golangci.yaml | 2 +- PROJECT | 5 +++-- integration/scenarios/export_service.go | 4 ++-- main.go | 8 ++++---- .../v1alpha1/groupversion_info.go | 0 .../v1alpha1/serviceexport_types.go | 0 .../v1alpha1/serviceimport_types.go | 0 .../v1alpha1/zz_generated.deepcopy.go | 0 .../{ => multicluster}/cloudmap_controller.go | 20 +++++++++---------- .../cloudmap_controller_test.go | 6 +++--- .../controllers_common_test.go | 6 +++--- .../{ => multicluster}/endpointslice_plan.go | 0 .../endpointslice_plan_test.go | 0 .../serviceexport_controller.go | 12 +++++------ .../serviceexport_controller_test.go | 10 +++++----- .../{ => multicluster}/suite_test.go | 4 ++-- pkg/controllers/{ => multicluster}/utils.go | 20 +++++++++---------- .../{ => multicluster}/utils_test.go | 20 +++++++++---------- 19 files changed, 60 insertions(+), 59 deletions(-) rename pkg/{api => apis/multicluster}/v1alpha1/groupversion_info.go (100%) rename pkg/{api => apis/multicluster}/v1alpha1/serviceexport_types.go (100%) rename pkg/{api => apis/multicluster}/v1alpha1/serviceimport_types.go (100%) rename pkg/{api => apis/multicluster}/v1alpha1/zz_generated.deepcopy.go (100%) rename pkg/controllers/{ => multicluster}/cloudmap_controller.go (92%) rename pkg/controllers/{ => multicluster}/cloudmap_controller_test.go (91%) rename pkg/controllers/{ => multicluster}/controllers_common_test.go (90%) rename pkg/controllers/{ => multicluster}/endpointslice_plan.go (100%) rename pkg/controllers/{ => multicluster}/endpointslice_plan_test.go (100%) rename pkg/controllers/{ => multicluster}/serviceexport_controller.go (96%) rename pkg/controllers/{ => multicluster}/serviceexport_controller_test.go (94%) rename pkg/controllers/{ => multicluster}/suite_test.go (92%) rename pkg/controllers/{ => multicluster}/utils.go (90%) rename pkg/controllers/{ => multicluster}/utils_test.go (95%) diff --git a/.github/.codecov.yml b/.github/.codecov.yml index 4e479a10..1ab3b5b2 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -22,7 +22,7 @@ comment: ignore: - "config/**/*" - - "pkg/api/**/*" + - "pkg/apis/**/*" - "mocks/**/*" - "integration/scenarios/**/*" - "pkg/common/logger.go" diff --git a/.golangci.yaml b/.golangci.yaml index 376f5fee..aae8ba43 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -32,4 +32,4 @@ run: skip: - .*_mock.go - mocks/ - - pkg/api/ + - pkg/apis/ diff --git a/PROJECT b/PROJECT index 60ca5e0f..83e31c52 100644 --- a/PROJECT +++ b/PROJECT @@ -3,6 +3,7 @@ layout: - go.kubebuilder.io/v3 projectName: aws-cloud-map-mcs-controller-for-k8s repo: github.com/aws/aws-cloud-map-mcs-controller-for-k8s +multigroup: true resources: - api: crdVersion: v1 @@ -11,7 +12,7 @@ resources: domain: x-k8s.io group: multicluster kind: ServiceExport - path: github.com/aws/aws-cloud-map-mcs-controller-for-k8s/api/v1alpha1 + path: github.com/aws/aws-cloud-map-mcs-controller-for-k8s/apis/multicluster/v1alpha1 version: v1alpha1 - api: crdVersion: v1 @@ -20,6 +21,6 @@ resources: domain: x-k8s.io group: multicluster kind: ServiceImport - path: github.com/aws/aws-cloud-map-mcs-controller-for-k8s/api/v1alpha1 + path: github.com/aws/aws-cloud-map-mcs-controller-for-k8s/apis/multicluster/v1alpha1 version: v1alpha1 version: "3" diff --git a/integration/scenarios/export_service.go b/integration/scenarios/export_service.go index 51826cf1..45b0b864 100644 --- a/integration/scenarios/export_service.go +++ b/integration/scenarios/export_service.go @@ -8,7 +8,7 @@ import ( "time" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers" + multiclustercontrollers "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers/multicluster" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" @@ -107,7 +107,7 @@ func (e *exportServiceScenario) compareEndpoints(cmEndpoints []*model.Endpoint) match := false for _, actual := range cmEndpoints { // Ignore K8S instance attribute for the purpose of this test. - delete(actual.Attributes, controllers.K8sVersionAttr) + delete(actual.Attributes, multiclustercontrollers.K8sVersionAttr) if expected.Equals(actual) { match = true break diff --git a/main.go b/main.go index 757383f7..23483879 100644 --- a/main.go +++ b/main.go @@ -22,8 +22,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" + multiclustercontrollers "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers/multicluster" // +kubebuilder:scaffold:imports ) @@ -85,7 +85,7 @@ func main() { log.Info("Running with AWS region", "AWS_REGION", awsCfg.Region) serviceDiscoveryClient := cloudmap.NewDefaultServiceDiscoveryClient(&awsCfg) - if err = (&controllers.ServiceExportReconciler{ + if err = (&multiclustercontrollers.ServiceExportReconciler{ Client: mgr.GetClient(), Log: common.NewLogger("controllers", "ServiceExport"), Scheme: mgr.GetScheme(), @@ -95,7 +95,7 @@ func main() { os.Exit(1) } - cloudMapReconciler := &controllers.CloudMapReconciler{ + cloudMapReconciler := &multiclustercontrollers.CloudMapReconciler{ Client: mgr.GetClient(), Cloudmap: serviceDiscoveryClient, Log: common.NewLogger("controllers", "Cloudmap"), diff --git a/pkg/api/v1alpha1/groupversion_info.go b/pkg/apis/multicluster/v1alpha1/groupversion_info.go similarity index 100% rename from pkg/api/v1alpha1/groupversion_info.go rename to pkg/apis/multicluster/v1alpha1/groupversion_info.go diff --git a/pkg/api/v1alpha1/serviceexport_types.go b/pkg/apis/multicluster/v1alpha1/serviceexport_types.go similarity index 100% rename from pkg/api/v1alpha1/serviceexport_types.go rename to pkg/apis/multicluster/v1alpha1/serviceexport_types.go diff --git a/pkg/api/v1alpha1/serviceimport_types.go b/pkg/apis/multicluster/v1alpha1/serviceimport_types.go similarity index 100% rename from pkg/api/v1alpha1/serviceimport_types.go rename to pkg/apis/multicluster/v1alpha1/serviceimport_types.go diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go similarity index 100% rename from pkg/api/v1alpha1/zz_generated.deepcopy.go rename to pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go diff --git a/pkg/controllers/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go similarity index 92% rename from pkg/controllers/cloudmap_controller.go rename to pkg/controllers/multicluster/cloudmap_controller.go index 2cd53655..a9fe22fd 100644 --- a/pkg/controllers/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -77,13 +77,13 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa return err } - serviceImports := v1alpha1.ServiceImportList{} + serviceImports := multiclusterv1alpha1.ServiceImportList{} if err = r.Client.List(ctx, &serviceImports, client.InNamespace(namespaceName)); err != nil { r.Log.Error(err, "failed to reconcile namespace", "namespace", namespaceName) return nil } - existingImportsMap := make(map[string]v1alpha1.ServiceImport) + existingImportsMap := make(map[string]multiclusterv1alpha1.ServiceImport) for _, svc := range serviceImports.Items { existingImportsMap[svc.Namespace+"/"+svc.Name] = svc } @@ -154,13 +154,13 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se return r.updateEndpointSlices(ctx, svcImport, svc.Endpoints, derivedService) } -func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace string, name string) (*v1alpha1.ServiceImport, error) { - existingServiceImport := &v1alpha1.ServiceImport{} +func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace string, name string) (*multiclusterv1alpha1.ServiceImport, error) { + existingServiceImport := &multiclusterv1alpha1.ServiceImport{} err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existingServiceImport) return existingServiceImport, err } -func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, namespace string, name string, servicePorts []*model.Port) (*v1alpha1.ServiceImport, error) { +func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, namespace string, name string, servicePorts []*model.Port) (*multiclusterv1alpha1.ServiceImport, error) { toCreate := CreateServiceImportStruct(namespace, name, servicePorts) if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err @@ -176,7 +176,7 @@ func (r *CloudMapReconciler) getDerivedService(ctx context.Context, namespace st return existingService, err } -func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svcImport *v1alpha1.ServiceImport, svcPorts []*model.Port) (*v1.Service, error) { +func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, svcPorts []*model.Port) (*v1.Service, error) { toCreate := CreateDerivedServiceStruct(svcImport, svcPorts) if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err @@ -186,7 +186,7 @@ func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svc return r.getDerivedService(ctx, toCreate.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) } -func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *v1alpha1.ServiceImport, desiredEndpoints []*model.Endpoint, svc *v1.Service) error { +func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, desiredEndpoints []*model.Endpoint, svc *v1.Service) error { existingSlicesList := discovery.EndpointSliceList{} if err := r.Client.List(ctx, &existingSlicesList, client.InNamespace(svc.Namespace), client.MatchingLabels{discovery.LabelServiceName: svc.Name}); err != nil { @@ -231,7 +231,7 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport return nil } -func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport *v1alpha1.ServiceImport, svc *v1.Service, importedSvcPorts []*model.Port) error { +func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, svc *v1.Service, importedSvcPorts []*model.Port) error { updateRequired := false if len(svcImport.Spec.IPs) != 1 || svcImport.Spec.IPs[0] != svc.Spec.ClusterIP { r.Log.Debug("ServiceImport IP need update", "ServiceImport IPs", svcImport.Spec.IPs, "cluster IP", svc.Spec.ClusterIP) @@ -257,7 +257,7 @@ func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport if !PortsEqualIgnoreOrder(svcImportPorts, simplifiedSvcPorts) { r.Log.Debug("ServiceImport ports need update", "ServiceImport Ports", svcImport.Spec.Ports, "imported ports", importedSvcPorts) - serviceImportPorts := make([]v1alpha1.ServicePort, 0) + serviceImportPorts := make([]multiclusterv1alpha1.ServicePort, 0) for _, port := range importedSvcPorts { serviceImportPorts = append(serviceImportPorts, PortToServiceImportPort(*port)) } diff --git a/pkg/controllers/cloudmap_controller_test.go b/pkg/controllers/multicluster/cloudmap_controller_test.go similarity index 91% rename from pkg/controllers/cloudmap_controller_test.go rename to pkg/controllers/multicluster/cloudmap_controller_test.go index f4c97e7e..d1c42bcd 100644 --- a/pkg/controllers/cloudmap_controller_test.go +++ b/pkg/controllers/multicluster/cloudmap_controller_test.go @@ -6,7 +6,7 @@ import ( "testing" cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -27,7 +27,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { objs := []runtime.Object{k8sNamespaceForTest()} s := scheme.Scheme - s.AddKnownTypes(v1alpha1.GroupVersion, &v1alpha1.ServiceImportList{}, &v1alpha1.ServiceImport{}) + s.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceImportList{}, &multiclusterv1alpha1.ServiceImport{}) fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() @@ -48,7 +48,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { } // assert service import object - serviceImport := &v1alpha1.ServiceImport{} + serviceImport := &multiclusterv1alpha1.ServiceImport{} err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceImport) assert.NoError(t, err) assert.Equal(t, test.SvcName, serviceImport.Name, "Service imported") diff --git a/pkg/controllers/controllers_common_test.go b/pkg/controllers/multicluster/controllers_common_test.go similarity index 90% rename from pkg/controllers/controllers_common_test.go rename to pkg/controllers/multicluster/controllers_common_test.go index 90a6c81e..2917376a 100644 --- a/pkg/controllers/controllers_common_test.go +++ b/pkg/controllers/multicluster/controllers_common_test.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" @@ -40,8 +40,8 @@ func k8sServiceForTest() *v1.Service { } } -func serviceExportForTest() *v1alpha1.ServiceExport { - return &v1alpha1.ServiceExport{ +func serviceExportForTest() *multiclusterv1alpha1.ServiceExport { + return &multiclusterv1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ Name: test.SvcName, Namespace: test.HttpNsName, diff --git a/pkg/controllers/endpointslice_plan.go b/pkg/controllers/multicluster/endpointslice_plan.go similarity index 100% rename from pkg/controllers/endpointslice_plan.go rename to pkg/controllers/multicluster/endpointslice_plan.go diff --git a/pkg/controllers/endpointslice_plan_test.go b/pkg/controllers/multicluster/endpointslice_plan_test.go similarity index 100% rename from pkg/controllers/endpointslice_plan_test.go rename to pkg/controllers/multicluster/endpointslice_plan_test.go diff --git a/pkg/controllers/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go similarity index 96% rename from pkg/controllers/serviceexport_controller.go rename to pkg/controllers/multicluster/serviceexport_controller.go index 0d77d265..c6a31685 100644 --- a/pkg/controllers/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -24,7 +24,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" ) const ( @@ -54,7 +54,7 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques name := req.NamespacedName r.Log.Debug("reconciling ServiceExport", "Namespace", namespace, "Name", name) - serviceExport := v1alpha1.ServiceExport{} + serviceExport := multiclusterv1alpha1.ServiceExport{} if err := r.Client.Get(ctx, name, &serviceExport); err != nil { if errors.IsNotFound(err) { r.Log.Debug("no ServiceExport found", @@ -92,7 +92,7 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques return r.handleUpdate(ctx, &serviceExport, &service) } -func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExport *v1alpha1.ServiceExport, service *v1.Service) (ctrl.Result, error) { +func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExport *multiclusterv1alpha1.ServiceExport, service *v1.Service) (ctrl.Result, error) { // Add the finalizer to the service export if not present, ensures the ServiceExport won't be deleted if !controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { controllerutil.AddFinalizer(serviceExport, ServiceExportFinalizer) @@ -185,7 +185,7 @@ func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context return cmService, nil } -func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExport *v1alpha1.ServiceExport) (ctrl.Result, error) { +func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExport *multiclusterv1alpha1.ServiceExport) (ctrl.Result, error) { if controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { r.Log.Info("removing service export", "namespace", serviceExport.Namespace, "name", serviceExport.Name) @@ -261,7 +261,7 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. func (r *ServiceExportReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.ServiceExport{}). + For(&multiclusterv1alpha1.ServiceExport{}). // Watch for the changes to the EndpointSlice object. This object is bound to be // updated when Service or Deployment are updated. There is also a filtering logic // to enqueue those EndpointSlice event which have corresponding ServiceExport @@ -310,7 +310,7 @@ func (r *ServiceExportReconciler) doesEndpointSliceHaveServiceExport(object clie Name: serviceName, Namespace: object.GetNamespace(), } - svcExport := v1alpha1.ServiceExport{} + svcExport := multiclusterv1alpha1.ServiceExport{} if err := r.Client.Get(context.TODO(), ns, &svcExport); err != nil { return false } diff --git a/pkg/controllers/serviceexport_controller_test.go b/pkg/controllers/multicluster/serviceexport_controller_test.go similarity index 94% rename from pkg/controllers/serviceexport_controller_test.go rename to pkg/controllers/multicluster/serviceexport_controller_test.go index d158c421..327c9693 100644 --- a/pkg/controllers/serviceexport_controller_test.go +++ b/pkg/controllers/multicluster/serviceexport_controller_test.go @@ -4,7 +4,7 @@ import ( "context" cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -65,7 +65,7 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { } assert.Equal(t, ctrl.Result{}, got, "Result should be empty") - serviceExport := &v1alpha1.ServiceExport{} + serviceExport := &multiclusterv1alpha1.ServiceExport{} err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceExport) assert.NoError(t, err) assert.Contains(t, serviceExport.Finalizers, ServiceExportFinalizer, "Finalizer added to the service export") @@ -109,7 +109,7 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { } assert.Equal(t, ctrl.Result{}, got, "Result should be empty") - serviceExport := &v1alpha1.ServiceExport{} + serviceExport := &multiclusterv1alpha1.ServiceExport{} err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceExport) assert.NoError(t, err) assert.Contains(t, serviceExport.Finalizers, ServiceExportFinalizer, "Finalizer added to the service export") @@ -153,7 +153,7 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { assert.NoError(t, err) assert.Equal(t, ctrl.Result{}, got, "Result should be empty") - serviceExport := &v1alpha1.ServiceExport{} + serviceExport := &multiclusterv1alpha1.ServiceExport{} err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, serviceExport) assert.NoError(t, err) assert.Empty(t, serviceExport.Finalizers, "Finalizer removed from the service export") @@ -161,7 +161,7 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { func getServiceExportScheme() *runtime.Scheme { scheme := runtime.NewScheme() - scheme.AddKnownTypes(v1alpha1.GroupVersion, &v1alpha1.ServiceExport{}) + scheme.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceExport{}) scheme.AddKnownTypes(v1.SchemeGroupVersion, &v1.Service{}) scheme.AddKnownTypes(discovery.SchemeGroupVersion, &discovery.EndpointSlice{}, &discovery.EndpointSliceList{}) return scheme diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/multicluster/suite_test.go similarity index 92% rename from pkg/controllers/suite_test.go rename to pkg/controllers/multicluster/suite_test.go index 56672340..c37817c3 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/multicluster/suite_test.go @@ -14,7 +14,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -38,7 +38,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, } diff --git a/pkg/controllers/utils.go b/pkg/controllers/multicluster/utils.go similarity index 90% rename from pkg/controllers/utils.go rename to pkg/controllers/multicluster/utils.go index 5465a3c6..75acbb94 100644 --- a/pkg/controllers/utils.go +++ b/pkg/controllers/multicluster/utils.go @@ -5,7 +5,7 @@ import ( "encoding/base32" "strings" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" @@ -39,7 +39,7 @@ func ServicePortToPort(svcPort v1.ServicePort) model.Port { } // ServiceImportPortToPort converts a service import port to an internal model port -func ServiceImportPortToPort(svcPort v1alpha1.ServicePort) model.Port { +func ServiceImportPortToPort(svcPort multiclusterv1alpha1.ServicePort) model.Port { return model.Port{ Name: svcPort.Name, Port: svcPort.Port, @@ -67,8 +67,8 @@ func PortToServicePort(port model.Port) v1.ServicePort { } // PortToServiceImportPort converts an internal model port to a service import port -func PortToServiceImportPort(port model.Port) v1alpha1.ServicePort { - return v1alpha1.ServicePort{ +func PortToServiceImportPort(port model.Port) multiclusterv1alpha1.ServicePort { + return multiclusterv1alpha1.ServicePort{ Name: port.Name, Protocol: v1.Protocol(port.Protocol), Port: port.Port, @@ -142,28 +142,28 @@ func DerivedName(namespace string, name string) string { } // CreateServiceImportStruct creates struct representation of a ServiceImport -func CreateServiceImportStruct(namespace string, name string, servicePorts []*model.Port) *v1alpha1.ServiceImport { - serviceImportPorts := make([]v1alpha1.ServicePort, 0) +func CreateServiceImportStruct(namespace string, name string, servicePorts []*model.Port) *multiclusterv1alpha1.ServiceImport { + serviceImportPorts := make([]multiclusterv1alpha1.ServicePort, 0) for _, port := range servicePorts { serviceImportPorts = append(serviceImportPorts, PortToServiceImportPort(*port)) } - return &v1alpha1.ServiceImport{ + return &multiclusterv1alpha1.ServiceImport{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: name, Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(namespace, name)}, }, - Spec: v1alpha1.ServiceImportSpec{ + Spec: multiclusterv1alpha1.ServiceImportSpec{ IPs: []string{}, - Type: v1alpha1.ClusterSetIP, + Type: multiclusterv1alpha1.ClusterSetIP, Ports: serviceImportPorts, }, } } // CreateDerivedServiceStruct creates struct representation of a derived service -func CreateDerivedServiceStruct(svcImport *v1alpha1.ServiceImport, importedSvcPorts []*model.Port) *v1.Service { +func CreateDerivedServiceStruct(svcImport *multiclusterv1alpha1.ServiceImport, importedSvcPorts []*model.Port) *v1.Service { ownerRef := metav1.NewControllerRef(svcImport, schema.GroupVersionKind{ Version: svcImport.TypeMeta.APIVersion, Kind: svcImport.TypeMeta.Kind, diff --git a/pkg/controllers/utils_test.go b/pkg/controllers/multicluster/utils_test.go similarity index 95% rename from pkg/controllers/utils_test.go rename to pkg/controllers/multicluster/utils_test.go index b4e1bda6..9edc386e 100644 --- a/pkg/controllers/utils_test.go +++ b/pkg/controllers/multicluster/utils_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/api/v1alpha1" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" v1 "k8s.io/api/core/v1" @@ -54,7 +54,7 @@ func TestServicePortToPort(t *testing.T) { func TestServiceImportPortToPort(t *testing.T) { type args struct { - svcImportPort v1alpha1.ServicePort + svcImportPort multiclusterv1alpha1.ServicePort } tests := []struct { name string @@ -64,7 +64,7 @@ func TestServiceImportPortToPort(t *testing.T) { { name: "happy case", args: args{ - svcImportPort: v1alpha1.ServicePort{ + svcImportPort: multiclusterv1alpha1.ServicePort{ Name: test.PortName1, Protocol: v1.ProtocolTCP, Port: 80, @@ -190,7 +190,7 @@ func TestPortToServiceImportPort(t *testing.T) { tests := []struct { name string args args - want v1alpha1.ServicePort + want multiclusterv1alpha1.ServicePort }{ { name: "happy case", @@ -202,7 +202,7 @@ func TestPortToServiceImportPort(t *testing.T) { Protocol: test.Protocol1, }, }, - want: v1alpha1.ServicePort{ + want: multiclusterv1alpha1.ServicePort{ Name: test.PortName1, Protocol: v1.ProtocolTCP, Port: test.Port1, @@ -454,7 +454,7 @@ func TestCreateServiceImportStruct(t *testing.T) { tests := []struct { name string args args - want v1alpha1.ServiceImport + want multiclusterv1alpha1.ServiceImport }{ { name: "happy case", @@ -464,16 +464,16 @@ func TestCreateServiceImportStruct(t *testing.T) { {Name: test.PortName2, Protocol: test.Protocol1, Port: test.Port2}, }, }, - want: v1alpha1.ServiceImport{ + want: multiclusterv1alpha1.ServiceImport{ ObjectMeta: metav1.ObjectMeta{ Namespace: test.HttpNsName, Name: test.SvcName, Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName)}, }, - Spec: v1alpha1.ServiceImportSpec{ + Spec: multiclusterv1alpha1.ServiceImportSpec{ IPs: []string{}, - Type: v1alpha1.ClusterSetIP, - Ports: []v1alpha1.ServicePort{ + Type: multiclusterv1alpha1.ClusterSetIP, + Ports: []multiclusterv1alpha1.ServicePort{ {Name: test.PortName1, Protocol: v1.ProtocolTCP, Port: test.Port1}, {Name: test.PortName2, Protocol: v1.ProtocolTCP, Port: test.Port2}, }, From 71f5d9132dbed7b79207e1d6742e0ccec8d0c76f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jul 2022 10:39:33 -0700 Subject: [PATCH 086/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#161) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.7 to 1.17.8. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.7...service/ecr/v1.17.8) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 016825ca..17137fb5 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.7 github.com/aws/aws-sdk-go-v2/config v1.15.13 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/onsi/ginkgo v1.16.5 diff --git a/go.sum b/go.sum index c0befe13..c69552d1 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2 v1.16.7 h1:zfBwXus3u14OszRxGcqCDS4MfMCv10e8SMJ2r8Xm0Ns= github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2/config v1.15.13 h1:CJH9zn/Enst7lDiGpoguVt0lZr5HcpNVlRJWbJ6qreo= @@ -83,18 +82,16 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.8 h1:niTa7zc7uyOP2ufri0jPESBt1h9y github.com/aws/aws-sdk-go-v2/credentials v1.12.8/go.mod h1:P2Hd4Sy7mXRxPNcQMPBmqszSJoDXexX8XEDaT6lucO0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 h1:VfBdn2AxwMbFyJN/lF/xuT3SakomJ86PZu3rCxb5K0s= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8/go.mod h1:oL1Q3KuCq1D4NykQnIvtRiBGLUXhcpY5pl6QZB2XEPU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 h1:2C0pYHcUBmdzPj+EKNC4qj97oK6yjrUhc1KoSodglvk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14/go.mod h1:kdjrMwHwrC3+FsKhNcCMJ7tUVj/8uSD5CZXeQ4wV6fM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8 h1:2J+jdlBJWEmTyAwC82Ym68xCykIvnSnIN18b8xHGlcc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8/go.mod h1:ZIV8GYoC6WLBW5KGs+o4rsc65/ozd+eQ0L31XF5VDwk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 h1:QquxR7NH3ULBsKC+NoTpilzbKKS+5AELfNREInbhvas= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15/go.mod h1:Tkrthp/0sNBShQQsamR7j/zY4p19tVTAs+nnqhH6R3c= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 h1:oKnAXxSF2FUvfgw8uzU/v9OTYorJJZ8eBmWhr9TWVVQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8/go.mod h1:rDVhIMAX9N2r8nWxDUlbubvvaFMnfsm+3jAV7q+rpM4= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7 h1:oBA2kHlC24mLutk2cpJv7sb2qrDLrWAiBNLQ2kQGp5g= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.7/go.mod h1:NWl7ZPKL3eHMeJTEjOnF/iWhM8HKJdb8z+tFuawoGhk= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 h1:jv1gOqDBXIFKrxay5gbZ0Ii0wZeXPQ8LHdwjzU6A5To= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8/go.mod h1:G5MzJj6NHYmeI8cN2PRd+QkGqbFT3jPM+Q0h0GL9lZQ= github.com/aws/aws-sdk-go-v2/service/sso v1.11.11 h1:XOJWXNFXJyapJqQuCIPfftsOf0XZZioM0kK6OPRt9MY= github.com/aws/aws-sdk-go-v2/service/sso v1.11.11/go.mod h1:MO4qguFjs3wPGcCSpQ7kOFTwRvb+eu+fn+1vKleGHUk= github.com/aws/aws-sdk-go-v2/service/sts v1.16.9 h1:yOfILxyjmtr2ubRkRJldlHDFBhf5vw4CzhbwWIBmimQ= From 24c96f34872fcb5bc9052248e080cf1e65e0bbbf Mon Sep 17 00:00:00 2001 From: Fred Wang <58385135+fredjywang@users.noreply.github.com> Date: Mon, 11 Jul 2022 16:05:08 -0700 Subject: [PATCH 087/163] Implement end-to-end EKS integration test (#154) --- .github/.codecov.yml | 2 +- .github/workflows/integration-test.yml | 8 +- CONTRIBUTING.md | 6 +- Makefile | 38 +++-- .../eks-test/configs/client-hello.yaml | 12 ++ .../eks-test/configs/coredns-clusterrole.yaml | 58 ++++++++ .../eks-test/configs/coredns-configmap.yaml | 26 ++++ .../eks-test/configs/coredns-deployment.yaml | 137 ++++++++++++++++++ .../eks-test/configs/eksctl-cluster.yaml | 18 +++ .../eks-test/configs/nginx-deployment.yaml | 22 +++ .../eks-test/configs/nginx-service.yaml | 10 ++ .../eks-test/configs/nginx-serviceexport.yaml | 5 + integration/eks-test/scripts/eks-DNS-test.sh | 25 ++++ integration/eks-test/scripts/eks-cleanup.sh | 43 ++++++ integration/eks-test/scripts/eks-common.sh | 17 +++ integration/eks-test/scripts/eks-run-tests.sh | 74 ++++++++++ .../eks-test/scripts/eks-setup-helper.sh | 17 +++ integration/eks-test/scripts/eks-setup.sh | 26 ++++ .../configs/e2e-deployment.yaml | 0 .../{ => kind-test}/configs/e2e-export.yaml | 0 .../{ => kind-test}/configs/e2e-service.yaml | 0 .../{ => kind-test}/scripts/cleanup-kind.sh | 4 +- integration/kind-test/scripts/common.sh | 16 ++ .../{ => kind-test}/scripts/ensure-jq.sh | 0 .../{ => kind-test}/scripts/run-tests.sh | 10 +- .../{ => kind-test}/scripts/setup-kind.sh | 4 +- integration/scripts/common.sh | 16 -- .../{ => shared}/scenarios/export_service.go | 0 .../{ => shared}/scenarios/runner/main.go | 2 +- .../{ => shared}/scripts/cleanup-cloudmap.sh | 1 - .../{ => shared}/scripts/poll-endpoints.sh | 8 +- .../{ => shared}/scripts/test-import.sh | 7 - 32 files changed, 556 insertions(+), 56 deletions(-) create mode 100644 integration/eks-test/configs/client-hello.yaml create mode 100644 integration/eks-test/configs/coredns-clusterrole.yaml create mode 100644 integration/eks-test/configs/coredns-configmap.yaml create mode 100644 integration/eks-test/configs/coredns-deployment.yaml create mode 100644 integration/eks-test/configs/eksctl-cluster.yaml create mode 100644 integration/eks-test/configs/nginx-deployment.yaml create mode 100644 integration/eks-test/configs/nginx-service.yaml create mode 100644 integration/eks-test/configs/nginx-serviceexport.yaml create mode 100755 integration/eks-test/scripts/eks-DNS-test.sh create mode 100755 integration/eks-test/scripts/eks-cleanup.sh create mode 100755 integration/eks-test/scripts/eks-common.sh create mode 100755 integration/eks-test/scripts/eks-run-tests.sh create mode 100755 integration/eks-test/scripts/eks-setup-helper.sh create mode 100755 integration/eks-test/scripts/eks-setup.sh rename integration/{ => kind-test}/configs/e2e-deployment.yaml (100%) rename integration/{ => kind-test}/configs/e2e-export.yaml (100%) rename integration/{ => kind-test}/configs/e2e-service.yaml (100%) rename integration/{ => kind-test}/scripts/cleanup-kind.sh (57%) create mode 100755 integration/kind-test/scripts/common.sh rename integration/{ => kind-test}/scripts/ensure-jq.sh (100%) rename integration/{ => kind-test}/scripts/run-tests.sh (73%) rename integration/{ => kind-test}/scripts/setup-kind.sh (80%) delete mode 100755 integration/scripts/common.sh rename integration/{ => shared}/scenarios/export_service.go (100%) rename integration/{ => shared}/scenarios/runner/main.go (98%) rename integration/{ => shared}/scripts/cleanup-cloudmap.sh (80%) rename integration/{ => shared}/scripts/poll-endpoints.sh (57%) rename integration/{ => shared}/scripts/test-import.sh (84%) diff --git a/.github/.codecov.yml b/.github/.codecov.yml index 1ab3b5b2..fbd4614c 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -24,5 +24,5 @@ ignore: - "config/**/*" - "pkg/apis/**/*" - "mocks/**/*" - - "integration/scenarios/**/*" + - "integration/shared/scenarios/**/*" - "pkg/common/logger.go" diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 512635e2..80054e88 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -26,10 +26,10 @@ jobs: - name: Set up env run: source ~/.bashrc - name: Start clean - run: make integration-cleanup + run: make kind-integration-cleanup - name: Set up cluster - run: make integration-setup + run: make kind-integration-setup - name: Run tests - run: make integration-run + run: make kind-integration-run - name: Clean up clusters - run: make integration-cleanup + run: make kind-integration-cleanup diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2b078c6d..68a94ae7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -173,15 +173,15 @@ You must first push a Docker image containing the changes to a Docker repository If you are deploying to cluster using kustomize templates from the `config` directory, you will need to override the image URI away from `ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s` in order to use your own docker images. -## Integration testing +## Local integration testing The end-to-end integration test suite can be run locally to validate controller core functionality. This will provision a local Kind cluster and build and run the AWS Cloud Map MCS Controller for K8s. The test will verify service endpoints sync with AWS Cloud Map. If successful, the suite will then de-provision the local test cluster and delete AWS Cloud Map namespace `aws-cloud-map-mcs-e2e` along with test service and service instance resources: ```sh -make integration-suite +make kind-integration-suite ``` If integration test suite fails for some reason, you can perform a cleanup: ```sh -make integration-cleanup +make kind-integration-cleanup ``` ## Build and push docker image to ECR diff --git a/Makefile b/Makefile index 7765a9d0..fc0fe59c 100644 --- a/Makefile +++ b/Makefile @@ -79,20 +79,36 @@ test-setup: ## setup test environment test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR) -integration-suite: ## Provision and run integration tests with cleanup - make integration-setup && \ - make integration-run && \ - make integration-cleanup +kind-integration-suite: ## Provision and run integration tests with cleanup + make kind-integration-setup && \ + make kind-integration-run && \ + make kind-integration-cleanup -integration-setup: build kind test-setup ## Setup the integration test using kind clusters - @./integration/scripts/setup-kind.sh +kind-integration-setup: build kind test-setup ## Setup the integration test using kind clusters + @./integration/kind-test/scripts/setup-kind.sh -integration-run: ## Run the integration test controller - @./integration/scripts/run-tests.sh +kind-integration-run: ## Run the integration test controller + @./integration/kind-test/scripts/run-tests.sh -integration-cleanup: kind ## Cleanup integration test resources in Cloud Map and local kind cluster - @./integration/scripts/cleanup-cloudmap.sh - @./integration/scripts/cleanup-kind.sh +kind-integration-cleanup: kind ## Cleanup integration test resources in Cloud Map and local kind cluster + @./integration/kind-test/scripts/cleanup-kind.sh + +eks-integration-suite: ## Provision and run EKS integration tests with cleanup + make eks-integration-setup && \ + make eks-integration-run && \ + make eks-integration-cleanup + +eks-integration-setup: build test-setup ## Setup the integration test using EKS clusters + @./integration/eks-test/scripts/eks-setup.sh + +eks-integration-run: ## Run the integration test controller + @./integration/eks-test/scripts/eks-run-tests.sh + +eks-integration-cleanup: ## Cleanup integration test resources in Cloud Map and EKS cluster + @./integration/eks-test/scripts/eks-cleanup.sh + +eks-test: + @./integration/eks-test/scripts/eks-test.sh ##@ Build diff --git a/integration/eks-test/configs/client-hello.yaml b/integration/eks-test/configs/client-hello.yaml new file mode 100644 index 00000000..6c9c8d1d --- /dev/null +++ b/integration/eks-test/configs/client-hello.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: client-hello + namespace: aws-cloud-map-mcs-eks-e2e +spec: + containers: + - command: + - sleep + - "1d" + image: alpine + name: client-hello diff --git a/integration/eks-test/configs/coredns-clusterrole.yaml b/integration/eks-test/configs/coredns-clusterrole.yaml new file mode 100644 index 00000000..242212fa --- /dev/null +++ b/integration/eks-test/configs/coredns-clusterrole.yaml @@ -0,0 +1,58 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - multicluster.x-k8s.io + resources: + - serviceimports + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - multicluster.x-k8s.io + resources: + - serviceexports + verbs: + - create + - get + - list + - patch + - update + - watch \ No newline at end of file diff --git a/integration/eks-test/configs/coredns-configmap.yaml b/integration/eks-test/configs/coredns-configmap.yaml new file mode 100644 index 00000000..dfdf9c7c --- /dev/null +++ b/integration/eks-test/configs/coredns-configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +data: + Corefile: | + .:53 { + errors + health + multicluster clusterset.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + annotations: + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + name: coredns + namespace: kube-system diff --git a/integration/eks-test/configs/coredns-deployment.yaml b/integration/eks-test/configs/coredns-deployment.yaml new file mode 100644 index 00000000..3061042c --- /dev/null +++ b/integration/eks-test/configs/coredns-deployment.yaml @@ -0,0 +1,137 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + progressDeadlineSeconds: 600 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + eks.amazonaws.com/compute-type: ec2 + creationTimestamp: null + labels: + eks.amazonaws.com/component: coredns + k8s-app: kube-dns + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.6 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + - mountPath: /tmp + name: tmp + dnsPolicy: Default + priorityClassName: system-cluster-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: coredns + serviceAccountName: coredns + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + volumes: + - emptyDir: {} + name: tmp + - configMap: + defaultMode: 420 + items: + - key: Corefile + path: Corefile + name: coredns + name: config-volume \ No newline at end of file diff --git a/integration/eks-test/configs/eksctl-cluster.yaml b/integration/eks-test/configs/eksctl-cluster.yaml new file mode 100644 index 00000000..f47d4217 --- /dev/null +++ b/integration/eks-test/configs/eksctl-cluster.yaml @@ -0,0 +1,18 @@ +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: $CLUSTER_NAME + region: $AWS_REGION + version: "1.22" +vpc: + cidr: $VPC_CIDR + autoAllocateIPv6: false + clusterEndpoints: + publicAccess: true + privateAccess: true +managedNodeGroups: +- name: $NODEGROUP_NAME + instanceType: t3.small + minSize: 1 + maxSize: 10 + desiredCapacity: 1 diff --git a/integration/eks-test/configs/nginx-deployment.yaml b/integration/eks-test/configs/nginx-deployment.yaml new file mode 100644 index 00000000..b60facdd --- /dev/null +++ b/integration/eks-test/configs/nginx-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: aws-cloud-map-mcs-eks-e2e + name: nginx-demo + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginxdemos/hello:plain-text + ports: + - containerPort: 80 diff --git a/integration/eks-test/configs/nginx-service.yaml b/integration/eks-test/configs/nginx-service.yaml new file mode 100644 index 00000000..ad15cd94 --- /dev/null +++ b/integration/eks-test/configs/nginx-service.yaml @@ -0,0 +1,10 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: aws-cloud-map-mcs-eks-e2e + name: nginx-hello +spec: + selector: + app: nginx + ports: + - port: 80 diff --git a/integration/eks-test/configs/nginx-serviceexport.yaml b/integration/eks-test/configs/nginx-serviceexport.yaml new file mode 100644 index 00000000..cbb57568 --- /dev/null +++ b/integration/eks-test/configs/nginx-serviceexport.yaml @@ -0,0 +1,5 @@ +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: aws-cloud-map-mcs-eks-e2e + name: nginx-hello \ No newline at end of file diff --git a/integration/eks-test/scripts/eks-DNS-test.sh b/integration/eks-test/scripts/eks-DNS-test.sh new file mode 100755 index 00000000..8bdf7578 --- /dev/null +++ b/integration/eks-test/scripts/eks-DNS-test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Testing service consumption with client-hello pod + +echo "verifying cross-cluster service consumption..." + +$KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- curl --version &>/dev/null +exit_code=$? + +# Install curl if not installed +if [ "$exit_code" -eq 126 ]; then + $KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- apk add curl +fi + +# Call to DNS server, if unable to reach, importing cluster is not able to properly consume service +$KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- curl -s $SERVICE.$NAMESPACE.svc.clusterset.local +exit_code=$? + +if [ "$exit_code" -ne 0 ]; then + exit $exit_code +fi + +echo "confirmed service consumption" +exit 0 + diff --git a/integration/eks-test/scripts/eks-cleanup.sh b/integration/eks-test/scripts/eks-cleanup.sh new file mode 100755 index 00000000..203f8cb4 --- /dev/null +++ b/integration/eks-test/scripts/eks-cleanup.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Cleanup EKS cluster used for integration test. + +source ./integration/eks-test/scripts/eks-common.sh + +# Delete service and namespace from export and import cluster +$KUBECTL_BIN config use-context $EXPORT_CLS +$KUBECTL_BIN delete svc $SERVICE -n $NAMESPACE + +# Verfication to check if there are hanging ServiceExport or ServiceImport CRDs and clears the finalizers to allow cleanup process to continue +for CRD in $($KUBECTL_BIN get crd -n $NAMESPACE | grep multicluster | cut -d " " -f 1 | xargs); do + $KUBECTL_BIN patch crd -n $NAMESPACE $CRD --type merge -p '{"metadata":{"finalizers": [null]}}'; + $KUBECTL_BIN delete crd $CRD -n $NAMESPACE # CRD needs to be explictly deleted in order to ensure zero resources are hanging for future tests +done + +$KUBECTL_BIN delete namespaces $NAMESPACE + +# IAM Service Account needs to be explictly deleted, as not doing so creates hanging service accounts that cause permissions issues in future tests +eksctl delete iamserviceaccount \ + --name cloud-map-mcs-controller-manager \ + --namespace $MCS_NAMESPACE \ + --cluster $EXPORT_CLS \ + --wait + +$KUBECTL_BIN config use-context $IMPORT_CLS +$KUBECTL_BIN delete pod $CLIENT_POD -n $NAMESPACE +$KUBECTL_BIN delete namespaces $NAMESPACE +eksctl delete iamserviceaccount \ + --name cloud-map-mcs-controller-manager \ + --namespace $MCS_NAMESPACE \ + --cluster $IMPORT_CLS \ + --wait + +$KUBECTL_BIN config use-context $EXPORT_CLS +$KUBECTL_BIN delete -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" +$KUBECTL_BIN config use-context $IMPORT_CLS +$KUBECTL_BIN delete -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" + +echo "EKS clusters cleaned!" + +./integration/shared/scripts/cleanup-cloudmap.sh + diff --git a/integration/eks-test/scripts/eks-common.sh b/integration/eks-test/scripts/eks-common.sh new file mode 100755 index 00000000..c3ce0264 --- /dev/null +++ b/integration/eks-test/scripts/eks-common.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +export KIND_BIN='./bin/kind' +export KUBECTL_BIN='kubectl' +export LOGS='./integration/eks-test/testlog' +export CONFIGS='./integration/eks-test/configs' +export SCENARIOS='./integration/shared/scenarios' +export NAMESPACE='aws-cloud-map-mcs-eks-e2e' +export MCS_NAMESPACE='cloud-map-mcs-system' +export SERVICE='nginx-hello' +export CLIENT_POD='client-hello' +export ENDPT_PORT=80 +export SERVICE_PORT=80 # from nginx-service.yaml +export EXPORT_CLS='cls1' +export IMPORT_CLS='cls2' +export EXPECTED_ENDPOINT_COUNT=3 +export UPDATED_ENDPOINT_COUNT=4 \ No newline at end of file diff --git a/integration/eks-test/scripts/eks-run-tests.sh b/integration/eks-test/scripts/eks-run-tests.sh new file mode 100755 index 00000000..becb7f77 --- /dev/null +++ b/integration/eks-test/scripts/eks-run-tests.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Runs the AWS Cloud Map MCS Controller for K8s in EKS clusters and test services has been exported from one cluster and imported from the other + +source ./integration/eks-test/scripts/eks-common.sh + +# Checking expected endpoints number in exporting cluster +$KUBECTL_BIN config use-context $EXPORT_CLS +if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT"); then + exit $? +fi + +# Runner to verify expected endpoints are exported to Cloud Map +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$endpts" +exit_code=$? + +# Check imported endpoints in importing cluster +if [ "$exit_code" -eq 0 ] ; then + $KUBECTL_BIN config use-context $IMPORT_CLS + ./integration/shared/scripts/test-import.sh "$EXPECTED_ENDPOINT_COUNT" "$endpts" + exit_code=$? +fi + +# Verifying that importing cluster is properly consuming services +if [ "$exit_code" -eq 0 ] ; then + ./integration/eks-test/scripts/eks-DNS-test.sh + exit_code=$? +fi + +echo "sleeping..." +sleep 2s + +# Scaling and verifying deployment +if [ "$exit_code" -eq 0 ] ; then + $KUBECTL_BIN config use-context $EXPORT_CLS + deployment=$($KUBECTL_BIN get deployment --namespace "$NAMESPACE" -o json | jq -r '.items[0].metadata.name') + + echo "scaling the deployment $deployment to $UPDATED_ENDPOINT_COUNT" + $KUBECTL_BIN scale deployment/"$deployment" --replicas="$UPDATED_ENDPOINT_COUNT" --namespace "$NAMESPACE" + exit_code=$? +fi + +if [ "$exit_code" -eq 0 ] ; then + if ! updated_endpoints=$(./integration/shared/scripts/poll-endpoints.sh "$UPDATED_ENDPOINT_COUNT") ; then + exit $? + fi +fi + +if [ "$exit_code" -eq 0 ] ; then + go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" + exit_code=$? +fi + +if [ "$exit_code" -eq 0 ] ; then + $KUBECTL_BIN config use-context $IMPORT_CLS + ./integration/shared/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" + exit_code=$? +fi + +if [ "$exit_code" -eq 0 ] ; then + ./integration/eks-test/scripts/eks-DNS-test.sh + exit_code=$? +fi + + +# Dump logs +mkdir -p "$LOGS" +$KUBECTL_BIN config use-context $EXPORT_CLS +$KUBECTL_BIN logs -l control-plane=controller-manager -c manager --namespace $MCS_NAMESPACE &> "$LOGS/ctl-1.log" +$KUBECTL_BIN config use-context $IMPORT_CLS +$KUBECTL_BIN logs -l control-plane=controller-manager -c manager --namespace $MCS_NAMESPACE &> "$LOGS/ctl-2.log" +echo "dumped logs" + +exit $exit_code diff --git a/integration/eks-test/scripts/eks-setup-helper.sh b/integration/eks-test/scripts/eks-setup-helper.sh new file mode 100755 index 00000000..df4c53d6 --- /dev/null +++ b/integration/eks-test/scripts/eks-setup-helper.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Adding IAM service accounts +$KUBECTL_BIN config use-context $1 +$KUBECTL_BIN create namespace $MCS_NAMESPACE +eksctl create iamserviceaccount \ +--cluster $1 \ +--namespace $MCS_NAMESPACE \ +--name cloud-map-mcs-controller-manager \ +--attach-policy-arn arn:aws:iam::aws:policy/AWSCloudMapFullAccess \ +--override-existing-serviceaccounts \ +--approve + +# Installing controller +$KUBECTL_BIN config use-context $1 +$KUBECTL_BIN apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" + diff --git a/integration/eks-test/scripts/eks-setup.sh b/integration/eks-test/scripts/eks-setup.sh new file mode 100755 index 00000000..236cec2a --- /dev/null +++ b/integration/eks-test/scripts/eks-setup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +source ./integration/eks-test/scripts/eks-common.sh + +# Call helper for service account and controller installation +./integration/eks-test/scripts/eks-setup-helper.sh $EXPORT_CLS +./integration/eks-test/scripts/eks-setup-helper.sh $IMPORT_CLS + +# Installing service +$KUBECTL_BIN config use-context $EXPORT_CLS +$KUBECTL_BIN create namespace $NAMESPACE +$KUBECTL_BIN apply -f "$CONFIGS/nginx-deployment.yaml" +$KUBECTL_BIN apply -f "$CONFIGS/nginx-service.yaml" + +$KUBECTL_BIN config use-context $IMPORT_CLS +$KUBECTL_BIN create namespace $NAMESPACE + +# Creating service export +$KUBECTL_BIN config use-context $EXPORT_CLS +$KUBECTL_BIN apply -f "$CONFIGS/nginx-serviceexport.yaml" + +# Create client-hello pod +$KUBECTL_BIN config use-context $IMPORT_CLS +$KUBECTL_BIN apply -f "$CONFIGS/client-hello.yaml" +sleep 15s +$KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- apk add curl ## install curl diff --git a/integration/configs/e2e-deployment.yaml b/integration/kind-test/configs/e2e-deployment.yaml similarity index 100% rename from integration/configs/e2e-deployment.yaml rename to integration/kind-test/configs/e2e-deployment.yaml diff --git a/integration/configs/e2e-export.yaml b/integration/kind-test/configs/e2e-export.yaml similarity index 100% rename from integration/configs/e2e-export.yaml rename to integration/kind-test/configs/e2e-export.yaml diff --git a/integration/configs/e2e-service.yaml b/integration/kind-test/configs/e2e-service.yaml similarity index 100% rename from integration/configs/e2e-service.yaml rename to integration/kind-test/configs/e2e-service.yaml diff --git a/integration/scripts/cleanup-kind.sh b/integration/kind-test/scripts/cleanup-kind.sh similarity index 57% rename from integration/scripts/cleanup-kind.sh rename to integration/kind-test/scripts/cleanup-kind.sh index 4166a580..f314c0c2 100755 --- a/integration/scripts/cleanup-kind.sh +++ b/integration/kind-test/scripts/cleanup-kind.sh @@ -3,6 +3,8 @@ # Deletes Kind cluster used for integration test. set -eo pipefail -source ./integration/scripts/common.sh +source ./integration/kind-test/scripts/common.sh $KIND_BIN delete cluster --name "$KIND_SHORT" + +./integration/shared/scripts/cleanup-cloudmap.sh diff --git a/integration/kind-test/scripts/common.sh b/integration/kind-test/scripts/common.sh new file mode 100755 index 00000000..37253a25 --- /dev/null +++ b/integration/kind-test/scripts/common.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +export KIND_BIN='./bin/kind' +export KUBECTL_BIN='./testbin/bin/kubectl' +export LOGS='./integration/kind-test/testlog' +export CONFIGS='./integration/kind-test/configs' +export SCENARIOS='./integration/shared/scenarios' +export NAMESPACE='aws-cloud-map-mcs-e2e' +export SERVICE='e2e-service' +export ENDPT_PORT=80 +export SERVICE_PORT=8080 +export KIND_SHORT='cloud-map-e2e' +export CLUSTER='kind-cloud-map-e2e' +export IMAGE='kindest/node:v1.19.16@sha256:dec41184d10deca01a08ea548197b77dc99eeacb56ff3e371af3193c86ca99f4' +export EXPECTED_ENDPOINT_COUNT=5 +export UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/scripts/ensure-jq.sh b/integration/kind-test/scripts/ensure-jq.sh similarity index 100% rename from integration/scripts/ensure-jq.sh rename to integration/kind-test/scripts/ensure-jq.sh diff --git a/integration/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh similarity index 73% rename from integration/scripts/run-tests.sh rename to integration/kind-test/scripts/run-tests.sh index 8430bff4..c313b570 100755 --- a/integration/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -2,13 +2,13 @@ # Runs the AWS Cloud Map MCS Controller for K8s as a background process and tests services have been exported -source ./integration/scripts/common.sh +source ./integration/kind-test/scripts/common.sh $KUBECTL_BIN apply -f "$CONFIGS/e2e-deployment.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-service.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-export.yaml" -if ! endpts=$(./integration/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") ; then +if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") ; then exit $? fi @@ -21,7 +21,7 @@ go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT " exit_code=$? if [ "$exit_code" -eq 0 ] ; then - ./integration/scripts/test-import.sh "$EXPECTED_ENDPOINT_COUNT" "$endpts" + ./integration/shared/scripts/test-import.sh "$EXPECTED_ENDPOINT_COUNT" "$endpts" exit_code=$? fi @@ -35,7 +35,7 @@ $KUBECTL_BIN scale deployment/"$deployment" --replicas="$UPDATED_ENDPOINT_COUNT" exit_code=$? if [ "$exit_code" -eq 0 ] ; then - if ! updated_endpoints=$(./integration/scripts/poll-endpoints.sh "$UPDATED_ENDPOINT_COUNT") ; then + if ! updated_endpoints=$(./integration/shared/scripts/poll-endpoints.sh "$UPDATED_ENDPOINT_COUNT") ; then exit $? fi @@ -43,7 +43,7 @@ if [ "$exit_code" -eq 0 ] ; then exit_code=$? if [ "$exit_code" -eq 0 ] ; then - ./integration/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" + ./integration/shared/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" exit_code=$? fi fi diff --git a/integration/scripts/setup-kind.sh b/integration/kind-test/scripts/setup-kind.sh similarity index 80% rename from integration/scripts/setup-kind.sh rename to integration/kind-test/scripts/setup-kind.sh index f37ee54d..e8c9b8ee 100755 --- a/integration/scripts/setup-kind.sh +++ b/integration/kind-test/scripts/setup-kind.sh @@ -5,9 +5,9 @@ set -e -source ./integration/scripts/common.sh +source ./integration/kind-test/scripts/common.sh -./integration/scripts/ensure-jq.sh +./integration/kind-test/scripts/ensure-jq.sh $KIND_BIN create cluster --name "$KIND_SHORT" --image "$IMAGE" $KUBECTL_BIN config use-context "$CLUSTER" diff --git a/integration/scripts/common.sh b/integration/scripts/common.sh deleted file mode 100755 index 226fee70..00000000 --- a/integration/scripts/common.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -KIND_BIN='./bin/kind' -KUBECTL_BIN='./testbin/bin/kubectl' -LOGS='./integration/testlog' -CONFIGS='./integration/configs' -SCENARIOS='./integration/scenarios' -NAMESPACE='aws-cloud-map-mcs-e2e' -SERVICE='e2e-service' -ENDPT_PORT=80 -SERVICE_PORT=8080 -KIND_SHORT='cloud-map-e2e' -CLUSTER='kind-cloud-map-e2e' -IMAGE='kindest/node:v1.19.16@sha256:dec41184d10deca01a08ea548197b77dc99eeacb56ff3e371af3193c86ca99f4' -EXPECTED_ENDPOINT_COUNT=5 -UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/scenarios/export_service.go b/integration/shared/scenarios/export_service.go similarity index 100% rename from integration/scenarios/export_service.go rename to integration/shared/scenarios/export_service.go diff --git a/integration/scenarios/runner/main.go b/integration/shared/scenarios/runner/main.go similarity index 98% rename from integration/scenarios/runner/main.go rename to integration/shared/scenarios/runner/main.go index e2448773..15995c0f 100644 --- a/integration/scenarios/runner/main.go +++ b/integration/shared/scenarios/runner/main.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/scenarios" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/shared/scenarios" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" ) diff --git a/integration/scripts/cleanup-cloudmap.sh b/integration/shared/scripts/cleanup-cloudmap.sh similarity index 80% rename from integration/scripts/cleanup-cloudmap.sh rename to integration/shared/scripts/cleanup-cloudmap.sh index bb3d4173..0fbd7384 100755 --- a/integration/scripts/cleanup-cloudmap.sh +++ b/integration/shared/scripts/cleanup-cloudmap.sh @@ -3,6 +3,5 @@ # Deletes all AWS Cloud Map resources used for integration test. set -eo pipefail -source ./integration/scripts/common.sh go run ./integration/janitor/runner/main.go "$NAMESPACE" diff --git a/integration/scripts/poll-endpoints.sh b/integration/shared/scripts/poll-endpoints.sh similarity index 57% rename from integration/scripts/poll-endpoints.sh rename to integration/shared/scripts/poll-endpoints.sh index 5b47af76..2e52aca2 100755 --- a/integration/scripts/poll-endpoints.sh +++ b/integration/shared/scripts/poll-endpoints.sh @@ -3,20 +3,19 @@ # Poll for endpoints to become active set -e -source ./integration/scripts/common.sh endpt_count=0 poll_count=0 while ((endpt_count < $1)) do if ((poll_count++ > 30)) ; then - echo "timed out polling for endpoints" + echo "timed out polling for endpoints" >&2 exit 1 fi sleep 2s if ! addresses=$($KUBECTL_BIN get endpointslices -o json --namespace "$NAMESPACE" | \ - jq '.items[] | select(.metadata.ownerReferences[].name=="e2e-service") | .endpoints[].addresses[0]' 2> /dev/null) + jq --arg SERVICE "$SERVICE" '.items[] | select(.metadata.ownerReferences[].name==$SERVICE) | .endpoints[].addresses[0]' 2> /dev/null) then # no endpoints ready continue @@ -25,5 +24,6 @@ do endpt_count=$(echo "$addresses" | wc -l | xargs) done -echo "$addresses" | tr -d '"' | paste -sd "," - +echo "$addresses" | tr -d '"' | paste -sd "," - +echo "matched number of endpoints to expected count" >&2 exit 0 diff --git a/integration/scripts/test-import.sh b/integration/shared/scripts/test-import.sh similarity index 84% rename from integration/scripts/test-import.sh rename to integration/shared/scripts/test-import.sh index 8c111949..ed986331 100755 --- a/integration/scripts/test-import.sh +++ b/integration/shared/scripts/test-import.sh @@ -4,13 +4,6 @@ set -e -source ./integration/scripts/common.sh - -if [ "$#" -ne 2 ]; then - echo "test script expects \"expected number of endpoints\" and endpoints IP list as arguments" - exit 1 -fi - expected_endpoint_count=$1 endpoints=$2 echo "checking service imports..." From b91fe75d474c6d94789b207fd18ef5ef7062f1b6 Mon Sep 17 00:00:00 2001 From: Dustin Hemmerling <52467318+astaticvoid@users.noreply.github.com> Date: Thu, 14 Jul 2022 10:39:52 -0700 Subject: [PATCH 088/163] Update libs to support Apple silicon (#164) Re-use test-env, mocks, and linter --- Makefile | 43 ++++++++++++------- .../multicluster.x-k8s.io_serviceexports.yaml | 8 +--- .../multicluster.x-k8s.io_serviceimports.yaml | 8 +--- go.mod | 2 +- go.sum | 4 +- integration/eks-test/scripts/eks-run-tests.sh | 2 +- integration/eks-test/scripts/eks-setup.sh | 2 +- integration/kind-test/scripts/common.sh | 4 +- integration/kind-test/scripts/run-tests.sh | 2 +- integration/shared/scripts/poll-endpoints.sh | 2 +- 10 files changed, 39 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index fc0fe59c..3b00d575 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,6 @@ GOBIN=$(shell go env GOBIN) endif # Setting SHELL to bash allows bash commands to be executed by recipes. -# This is a requirement for 'setup-envtest.sh' in the test target. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec @@ -58,33 +57,40 @@ mod: tidy: go mod tidy +GOLANGCI_LINT=$(shell pwd)/bin/golangci-lint golangci-lint: ## Download golangci-lint +ifneq ($(shell test -f $(GOLANGCI_LINT); echo $$?), 0) + @echo Getting golangci-lint... @mkdir -p $(shell pwd)/bin - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.45.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.46.2 +endif .PHONY: lint lint: golangci-lint ## Run linter - $(shell pwd)/bin/golangci-lint run + $(GOLANGCI_LINT) run .PHONY: goimports goimports: ## run goimports updating files in place goimports -w . ENVTEST_ASSETS_DIR=$(shell pwd)/testbin -test: manifests generate generate-mocks fmt vet test-setup ## Run tests. - source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -covermode=atomic +KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)" +test: manifests generate generate-mocks fmt vet test-setup ## Run tests + KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out -covermode=atomic -test-setup: ## setup test environment +test-setup: setup-envtest ## Ensure test environment has been downloaded +ifneq ($(shell test -d $(ENVTEST_ASSETS_DIR); echo $$?), 0) + @echo Setting up K8s test environment... mkdir -p ${ENVTEST_ASSETS_DIR} - test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh - source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR) + $(ENVTEST) use 1.24.x --bin-dir $(ENVTEST_ASSETS_DIR) +endif kind-integration-suite: ## Provision and run integration tests with cleanup make kind-integration-setup && \ make kind-integration-run && \ make kind-integration-cleanup -kind-integration-setup: build kind test-setup ## Setup the integration test using kind clusters +kind-integration-setup: build kind ## Setup the integration test using kind clusters @./integration/kind-test/scripts/setup-kind.sh kind-integration-run: ## Run the integration test controller @@ -98,7 +104,7 @@ eks-integration-suite: ## Provision and run EKS integration tests with cleanup make eks-integration-run && \ make eks-integration-cleanup -eks-integration-setup: build test-setup ## Setup the integration test using EKS clusters +eks-integration-setup: build ## Setup the integration test using EKS clusters @./integration/eks-test/scripts/eks-setup.sh eks-integration-run: ## Run the integration test controller @@ -129,7 +135,8 @@ docker-push: ## Push docker image with the manager. clean: @echo Cleaning... go clean - rm -rf $(MOCKS_DESTINATION) bin/ testbin/ cover.out + if test -d $(ENVTEST_ASSETS_DIR) ; then chmod -R +w $(ENVTEST_ASSETS_DIR) ; fi + rm -rf $(MOCKS_DESTINATION)/ bin/ $(ENVTEST_ASSETS_DIR)/ cover.out ##@ Deployment @@ -148,6 +155,8 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi MOCKS_DESTINATION=mocks generate-mocks: mockgen +ifneq ($(shell test -d $(MOCKS_DESTINATION); echo $$?), 0) + @echo Generating mocks... $(MOCKGEN) --source pkg/cloudmap/client.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/client_mock.go --package cloudmap_mock $(MOCKGEN) --source pkg/cloudmap/cache.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/cache_mock.go --package cloudmap_mock $(MOCKGEN) --source pkg/cloudmap/operation_poller.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_poller_mock.go --package cloudmap_mock @@ -156,15 +165,19 @@ generate-mocks: mockgen $(MOCKGEN) --source pkg/cloudmap/aws_facade.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/aws_facade_mock.go --package cloudmap_mock $(MOCKGEN) --source integration/janitor/api.go --destination $(MOCKS_DESTINATION)/integration/janitor/api_mock.go --package janitor_mock $(MOCKGEN) --source integration/janitor/aws_facade.go --destination $(MOCKS_DESTINATION)/integration/janitor/aws_facade_mock.go --package janitor_mock - +endif CONTROLLER_GEN = $(shell pwd)/bin/controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2) KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. - $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.4) + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.5) + +ENVTEST = $(shell pwd)/bin/setup-envtest +setup-envtest: ## Download setup-envtest + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) MOCKGEN = $(shell pwd)/bin/mockgen mockgen: ## Download mockgen @@ -172,7 +185,7 @@ mockgen: ## Download mockgen KIND = $(shell pwd)/bin/kind kind: ## Download kind - $(call go-get-tool,$(KIND),sigs.k8s.io/kind@v0.13.0) + $(call go-get-tool,$(KIND),sigs.k8s.io/kind@v0.14.0) # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml b/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml index 0703a5fe..1e41af78 100644 --- a/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_serviceexports.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: serviceexports.multicluster.x-k8s.io spec: @@ -114,9 +114,3 @@ spec: type: object served: true storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml index 17bac395..075f5e44 100644 --- a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: serviceimports.multicluster.x-k8s.io spec: @@ -136,9 +136,3 @@ spec: type: object served: true storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/go.mod b/go.mod index 17137fb5..fdf98bd1 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 - sigs.k8s.io/controller-runtime v0.12.2 + sigs.k8s.io/controller-runtime v0.12.3 ) require ( diff --git a/go.sum b/go.sum index c69552d1..dfbc3f14 100644 --- a/go.sum +++ b/go.sum @@ -1002,8 +1002,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/controller-runtime v0.12.2 h1:nqV02cvhbAj7tbt21bpPpTByrXGn2INHRsi39lXy9sE= -sigs.k8s.io/controller-runtime v0.12.2/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0= +sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio= +sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/integration/eks-test/scripts/eks-run-tests.sh b/integration/eks-test/scripts/eks-run-tests.sh index becb7f77..e8ccb4ff 100755 --- a/integration/eks-test/scripts/eks-run-tests.sh +++ b/integration/eks-test/scripts/eks-run-tests.sh @@ -28,7 +28,7 @@ if [ "$exit_code" -eq 0 ] ; then fi echo "sleeping..." -sleep 2s +sleep 2 # Scaling and verifying deployment if [ "$exit_code" -eq 0 ] ; then diff --git a/integration/eks-test/scripts/eks-setup.sh b/integration/eks-test/scripts/eks-setup.sh index 236cec2a..d605930b 100755 --- a/integration/eks-test/scripts/eks-setup.sh +++ b/integration/eks-test/scripts/eks-setup.sh @@ -22,5 +22,5 @@ $KUBECTL_BIN apply -f "$CONFIGS/nginx-serviceexport.yaml" # Create client-hello pod $KUBECTL_BIN config use-context $IMPORT_CLS $KUBECTL_BIN apply -f "$CONFIGS/client-hello.yaml" -sleep 15s +sleep 15 $KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- apk add curl ## install curl diff --git a/integration/kind-test/scripts/common.sh b/integration/kind-test/scripts/common.sh index 37253a25..41b154d2 100755 --- a/integration/kind-test/scripts/common.sh +++ b/integration/kind-test/scripts/common.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash export KIND_BIN='./bin/kind' -export KUBECTL_BIN='./testbin/bin/kubectl' +export KUBECTL_BIN='kubectl' export LOGS='./integration/kind-test/testlog' export CONFIGS='./integration/kind-test/configs' export SCENARIOS='./integration/shared/scenarios' @@ -11,6 +11,6 @@ export ENDPT_PORT=80 export SERVICE_PORT=8080 export KIND_SHORT='cloud-map-e2e' export CLUSTER='kind-cloud-map-e2e' -export IMAGE='kindest/node:v1.19.16@sha256:dec41184d10deca01a08ea548197b77dc99eeacb56ff3e371af3193c86ca99f4' +export IMAGE='kindest/node:v1.20.15@sha256:a6ce604504db064c5e25921c6c0fffea64507109a1f2a512b1b562ac37d652f3' export EXPECTED_ENDPOINT_COUNT=5 export UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/kind-test/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh index c313b570..6c5a256c 100755 --- a/integration/kind-test/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -26,7 +26,7 @@ if [ "$exit_code" -eq 0 ] ; then fi echo "sleeping..." -sleep 2s +sleep 2 deployment=$($KUBECTL_BIN get deployment --namespace "$NAMESPACE" -o json | jq -r '.items[0].metadata.name') diff --git a/integration/shared/scripts/poll-endpoints.sh b/integration/shared/scripts/poll-endpoints.sh index 2e52aca2..eb6f4f40 100755 --- a/integration/shared/scripts/poll-endpoints.sh +++ b/integration/shared/scripts/poll-endpoints.sh @@ -13,7 +13,7 @@ do exit 1 fi - sleep 2s + sleep 2 if ! addresses=$($KUBECTL_BIN get endpointslices -o json --namespace "$NAMESPACE" | \ jq --arg SERVICE "$SERVICE" '.items[] | select(.metadata.ownerReferences[].name==$SERVICE) | .endpoints[].addresses[0]' 2> /dev/null) then From fce441213e39e19d211f09acfda1582e314886dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Jul 2022 21:07:24 -0700 Subject: [PATCH 089/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.15.13 to 1.15.14 (#163) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.15.13 to 1.15.14. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.15.13...config/v1.15.14) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index fdf98bd1..f2ba85e8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.7 - github.com/aws/aws-sdk-go-v2/config v1.15.13 + github.com/aws/aws-sdk-go-v2/config v1.15.14 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,13 +28,13 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.8 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.12 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.9 // indirect github.com/aws/smithy-go v1.12.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index dfbc3f14..b006e3eb 100644 --- a/go.sum +++ b/go.sum @@ -76,10 +76,10 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.16.7 h1:zfBwXus3u14OszRxGcqCDS4MfMCv10e8SMJ2r8Xm0Ns= github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= -github.com/aws/aws-sdk-go-v2/config v1.15.13 h1:CJH9zn/Enst7lDiGpoguVt0lZr5HcpNVlRJWbJ6qreo= -github.com/aws/aws-sdk-go-v2/config v1.15.13/go.mod h1:AcMu50uhV6wMBUlURnEXhr9b3fX6FLSTlEV89krTEGk= -github.com/aws/aws-sdk-go-v2/credentials v1.12.8 h1:niTa7zc7uyOP2ufri0jPESBt1h9yP3Zc0q+xzih3h8o= -github.com/aws/aws-sdk-go-v2/credentials v1.12.8/go.mod h1:P2Hd4Sy7mXRxPNcQMPBmqszSJoDXexX8XEDaT6lucO0= +github.com/aws/aws-sdk-go-v2/config v1.15.14 h1:+BqpqlydTq4c2et9Daury7gE+o67P4lbk7eybiCBNc4= +github.com/aws/aws-sdk-go-v2/config v1.15.14/go.mod h1:CQBv+VVv8rR5z2xE+Chdh5m+rFfsqeY4k0veEZeq6QM= +github.com/aws/aws-sdk-go-v2/credentials v1.12.9 h1:DloAJr0/jbvm0iVRFDFh8GlWxrOd9XKyX82U+dfVeZs= +github.com/aws/aws-sdk-go-v2/credentials v1.12.9/go.mod h1:2Vavxl1qqQXJ8MUcQZTsIEW8cwenFCWYXtLRPba3L/o= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 h1:VfBdn2AxwMbFyJN/lF/xuT3SakomJ86PZu3rCxb5K0s= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8/go.mod h1:oL1Q3KuCq1D4NykQnIvtRiBGLUXhcpY5pl6QZB2XEPU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 h1:2C0pYHcUBmdzPj+EKNC4qj97oK6yjrUhc1KoSodglvk= @@ -92,8 +92,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 h1:oKnAXxSF2F github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8/go.mod h1:rDVhIMAX9N2r8nWxDUlbubvvaFMnfsm+3jAV7q+rpM4= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 h1:jv1gOqDBXIFKrxay5gbZ0Ii0wZeXPQ8LHdwjzU6A5To= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8/go.mod h1:G5MzJj6NHYmeI8cN2PRd+QkGqbFT3jPM+Q0h0GL9lZQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.11 h1:XOJWXNFXJyapJqQuCIPfftsOf0XZZioM0kK6OPRt9MY= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.11/go.mod h1:MO4qguFjs3wPGcCSpQ7kOFTwRvb+eu+fn+1vKleGHUk= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.12 h1:760bUnTX/+d693FT6T6Oa7PZHfEQT9XMFZeM5IQIB0A= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.12/go.mod h1:MO4qguFjs3wPGcCSpQ7kOFTwRvb+eu+fn+1vKleGHUk= github.com/aws/aws-sdk-go-v2/service/sts v1.16.9 h1:yOfILxyjmtr2ubRkRJldlHDFBhf5vw4CzhbwWIBmimQ= github.com/aws/aws-sdk-go-v2/service/sts v1.16.9/go.mod h1:O1IvkYxr+39hRf960Us6j0x1P8pDqhTX+oXM5kQNl/Y= github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0= From fcc266d9a96b074eaf9be550565020845d564a63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Jul 2022 21:09:40 -0700 Subject: [PATCH 090/163] Bump k8s.io/apimachinery from 0.24.2 to 0.24.3 (#166) Bumps [k8s.io/apimachinery](https://github.com/kubernetes/apimachinery) from 0.24.2 to 0.24.3. - [Release notes](https://github.com/kubernetes/apimachinery/releases) - [Commits](https://github.com/kubernetes/apimachinery/compare/v0.24.2...v0.24.3) --- updated-dependencies: - dependency-name: k8s.io/apimachinery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f2ba85e8..2d7e7a43 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.0 k8s.io/api v0.24.2 - k8s.io/apimachinery v0.24.2 + k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.2 sigs.k8s.io/controller-runtime v0.12.3 ) diff --git a/go.sum b/go.sum index b006e3eb..97ca37ca 100644 --- a/go.sum +++ b/go.sum @@ -979,8 +979,9 @@ k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= -k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apimachinery v0.24.3 h1:hrFiNSA2cBZqllakVYyH/VyEh4B581bQRmqATJSeQTg= +k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= From ce5c2d2c72a282ed756095634ed65711fab4abbe Mon Sep 17 00:00:00 2001 From: Fred Wang <58385135+fredjywang@users.noreply.github.com> Date: Thu, 28 Jul 2022 17:03:57 -0700 Subject: [PATCH 091/163] Add ServiceType to Cloud Map and Endpoints (#170) --- .../shared/scenarios/export_service.go | 1 + pkg/cloudmap/client_test.go | 4 ++ .../multicluster/serviceexport_controller.go | 3 ++ pkg/controllers/multicluster/utils.go | 9 +++++ pkg/model/types.go | 37 ++++++++++++++++--- pkg/model/types_test.go | 26 +++++++++++++ samples/example-headless.yaml | 11 ++++++ test/test-constants.go | 7 +++- 8 files changed, 91 insertions(+), 7 deletions(-) create mode 100755 samples/example-headless.yaml diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index 45b0b864..3d7caa57 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -58,6 +58,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po Protocol: string(v1.ProtocolTCP), }, EndpointPort: endpointPort, + ServiceType: model.ClusterSetIPType, // in scenario, we assume ClusterSetIP type Attributes: make(map[string]string), }) } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index c22b580e..bc03a9c4 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -291,6 +291,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { model.ServicePortAttr: test.ServicePortStr1, model.ServiceProtocolAttr: test.Protocol1, model.ServiceTargetPortAttr: test.PortStr1, + model.ServiceTypeAttr: test.SvcType, } attrs2 := map[string]string{ model.EndpointIpv4Attr: test.EndptIp2, @@ -301,6 +302,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { model.ServicePortAttr: test.ServicePortStr2, model.ServiceProtocolAttr: test.Protocol2, model.ServiceTargetPortAttr: test.PortStr2, + model.ServiceTypeAttr: test.SvcType, } tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). @@ -370,6 +372,7 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { model.ServicePortAttr: test.ServicePortStr1, model.ServiceProtocolAttr: test.Protocol1, model.ServiceTargetPortAttr: test.PortStr1, + model.ServiceTypeAttr: test.SvcType, }, }, { @@ -383,6 +386,7 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { model.ServicePortAttr: test.ServicePortStr2, model.ServiceProtocolAttr: test.Protocol2, model.ServiceTargetPortAttr: test.PortStr2, + model.ServiceTypeAttr: test.SvcType, }, }, } diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index c6a31685..6bc456d8 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -225,6 +225,8 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. return nil, err } + serviceType := ExtractServiceType(svc) + servicePortMap := make(map[string]model.Port) for _, svcPort := range svc.Spec.Ports { servicePortMap[svcPort.Name] = ServicePortToPort(svcPort) @@ -249,6 +251,7 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. IP: IP, EndpointPort: port, ServicePort: servicePortMap[*endpointPort.Name], + ServiceType: serviceType, Attributes: attributes, }) } diff --git a/pkg/controllers/multicluster/utils.go b/pkg/controllers/multicluster/utils.go index 75acbb94..a9affa00 100644 --- a/pkg/controllers/multicluster/utils.go +++ b/pkg/controllers/multicluster/utils.go @@ -226,3 +226,12 @@ func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string) *discovery AddressType: discovery.AddressTypeIPv4, } } + +// ExtractServiceType finds the ServiceType of a given service as Headless/ClusterSetIP +func ExtractServiceType(svc *v1.Service) model.ServiceType { + if svc.Spec.ClusterIP == "None" { + return model.HeadlessType + } + + return model.ClusterSetIPType +} diff --git a/pkg/model/types.go b/pkg/model/types.go index edbca8ff..5c6193f0 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -33,12 +33,20 @@ type Service struct { Endpoints []*Endpoint } +const ( + HeadlessType ServiceType = "Headless" + ClusterSetIPType ServiceType = "ClusterSetIP" +) + +type ServiceType string + // Endpoint holds basic values and attributes for an endpoint. type Endpoint struct { Id string IP string EndpointPort Port ServicePort Port + ServiceType ServiceType Attributes map[string]string } @@ -60,10 +68,11 @@ const ( ServicePortAttr = "SERVICE_PORT" ServiceTargetPortAttr = "SERVICE_TARGET_PORT" ServiceProtocolAttr = "SERVICE_PROTOCOL" + ServiceTypeAttr = "SERVICE_TYPE" ) // NewEndpointFromInstance converts a Cloud Map HttpInstanceSummary to an endpoint. -func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (endpointPtr *Endpoint, err error) { +func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) { endpoint := Endpoint{ Id: *inst.InstanceId, Attributes: make(map[string]string), @@ -73,18 +82,30 @@ func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (endpointPtr *Endp attributes[key] = value } - // Remove and set the IP, Port, Port - if endpoint.IP, err = removeStringAttr(attributes, EndpointIpv4Attr); err != nil { + // Remove and set the IP, Port, Service Port, ServiceType + ip, err := removeStringAttr(attributes, EndpointIpv4Attr) + if err != nil { return nil, err } + endpoint.IP = ip - if endpoint.EndpointPort, err = endpointPortFromAttr(attributes); err != nil { + endpointPort, err := endpointPortFromAttr(attributes) + if err != nil { return nil, err } + endpoint.EndpointPort = endpointPort - if endpoint.ServicePort, err = servicePortFromAttr(attributes); err != nil { + servicePort, err := servicePortFromAttr(attributes) + if err != nil { return nil, err } + endpoint.ServicePort = servicePort + + serviceTypeStr, err := removeStringAttr(attributes, ServiceTypeAttr) + if err != nil { + return nil, err + } + endpoint.ServiceType = ServiceType(serviceTypeStr) // Add the remaining attributes endpoint.Attributes = attributes @@ -156,6 +177,7 @@ func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs[ServicePortAttr] = strconv.Itoa(int(e.ServicePort.Port)) attrs[ServiceTargetPortAttr] = e.ServicePort.TargetPort attrs[ServiceProtocolAttr] = e.ServicePort.Protocol + attrs[ServiceTypeAttr] = e.ServiceType.String() for key, val := range e.Attributes { attrs[key] = val @@ -186,6 +208,11 @@ func EndpointIdFromIPAddressAndPort(address string, port Port) string { return fmt.Sprintf("%s-%s-%d", strings.ToLower(port.Protocol), address, port.Port) } +// Gives string representation for ServiceType +func (serviceType ServiceType) String() string { + return string(serviceType) +} + func ConvertNamespaceType(nsType types.NamespaceType) (namespaceType NamespaceType) { switch nsType { case types.NamespaceTypeDnsPrivate: diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index 019308fa..b580acab 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -9,6 +9,7 @@ import ( var instId = "my-instance" var ip = "192.168.0.1" +var serviceType = ClusterSetIPType.String() func TestNewEndpointFromInstance(t *testing.T) { tests := []struct { @@ -30,6 +31,7 @@ func TestNewEndpointFromInstance(t *testing.T) { ServiceProtocolAttr: "TCP", ServicePortAttr: "65535", ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, "custom-attr": "custom-val", }, }, @@ -47,6 +49,7 @@ func TestNewEndpointFromInstance(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, + ServiceType: ServiceType(serviceType), Attributes: map[string]string{ "custom-attr": "custom-val", }, @@ -65,6 +68,7 @@ func TestNewEndpointFromInstance(t *testing.T) { ServiceProtocolAttr: "TCP", ServicePortAttr: "99999", ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, "custom-attr": "custom-val", }, }, @@ -92,6 +96,24 @@ func TestNewEndpointFromInstance(t *testing.T) { }, wantErr: true, }, + { + name: "missing ServiceType", + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, + Attributes: map[string]string{ + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + "custom-attr": "custom-val", + }, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -113,6 +135,7 @@ func TestEndpoint_GetAttributes(t *testing.T) { IP string EndpointPort Port ServicePort Port + ServiceType ServiceType Attributes map[string]string } tests := []struct { @@ -135,6 +158,7 @@ func TestEndpoint_GetAttributes(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, + ServiceType: ServiceType(serviceType), Attributes: map[string]string{ "custom-attr": "custom-val", }, @@ -148,6 +172,7 @@ func TestEndpoint_GetAttributes(t *testing.T) { ServiceProtocolAttr: "TCP", ServicePortAttr: "30", ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, "custom-attr": "custom-val", }, }, @@ -159,6 +184,7 @@ func TestEndpoint_GetAttributes(t *testing.T) { IP: tt.fields.IP, EndpointPort: tt.fields.EndpointPort, ServicePort: tt.fields.ServicePort, + ServiceType: tt.fields.ServiceType, Attributes: tt.fields.Attributes, } if got := e.GetCloudMapAttributes(); !reflect.DeepEqual(got, tt.want) { diff --git a/samples/example-headless.yaml b/samples/example-headless.yaml new file mode 100755 index 00000000..b5dce655 --- /dev/null +++ b/samples/example-headless.yaml @@ -0,0 +1,11 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: example + name: my-service +spec: + clusterIP: None + selector: + app: nginx + ports: + - port: 80 diff --git a/test/test-constants.go b/test/test-constants.go index efd10116..91f90c55 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -32,6 +32,7 @@ const ( OpId1 = "operation-id-1" OpId2 = "operation-id-2" OpStart = 1 + SvcType = "ClusterSetIP" ) func GetTestHttpNamespace() *model.Namespace { @@ -81,7 +82,8 @@ func GetTestEndpoint1() *model.Endpoint { TargetPort: PortStr1, Protocol: Protocol1, }, - Attributes: make(map[string]string), + ServiceType: model.ClusterSetIPType, + Attributes: make(map[string]string), } } @@ -100,7 +102,8 @@ func GetTestEndpoint2() *model.Endpoint { TargetPort: PortStr2, Protocol: Protocol2, }, - Attributes: make(map[string]string), + ServiceType: model.ClusterSetIPType, + Attributes: make(map[string]string), } } From 674fdc4fcccc437c86a1ef6decae15ca2c21a869 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Jul 2022 11:34:40 -0700 Subject: [PATCH 092/163] Bump k8s.io/api from 0.24.2 to 0.24.3 (#167) Bumps [k8s.io/api](https://github.com/kubernetes/api) from 0.24.2 to 0.24.3. - [Release notes](https://github.com/kubernetes/api/releases) - [Commits](https://github.com/kubernetes/api/compare/v0.24.2...v0.24.3) --- updated-dependencies: - dependency-name: k8s.io/api dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2d7e7a43..40a20737 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.0 - k8s.io/api v0.24.2 + k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.2 sigs.k8s.io/controller-runtime v0.12.3 diff --git a/go.sum b/go.sum index 97ca37ca..302cd738 100644 --- a/go.sum +++ b/go.sum @@ -975,8 +975,9 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= +k8s.io/api v0.24.3 h1:tt55QEmKd6L2k5DP6G/ZzdMQKvG5ro4H4teClqm0sTY= +k8s.io/api v0.24.3/go.mod h1:elGR/XSZrS7z7cSZPzVWaycpJuGIw57j9b95/1PdJNI= k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= From a95829af4d93b1154545c2524c815ba3a9c62250 Mon Sep 17 00:00:00 2001 From: Fred Wang <58385135+fredjywang@users.noreply.github.com> Date: Fri, 29 Jul 2022 11:44:30 -0700 Subject: [PATCH 093/163] Unit test for ExtractServiceType function (#171) --- pkg/controllers/multicluster/utils_test.go | 58 ++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/pkg/controllers/multicluster/utils_test.go b/pkg/controllers/multicluster/utils_test.go index 9edc386e..f97d8dff 100644 --- a/pkg/controllers/multicluster/utils_test.go +++ b/pkg/controllers/multicluster/utils_test.go @@ -489,3 +489,61 @@ func TestCreateServiceImportStruct(t *testing.T) { }) } } + +func TestExtractServiceType(t *testing.T) { + tests := []struct { + name string + svc *v1.Service + want model.ServiceType + }{ + { + name: "cluster ip type", + svc: &v1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: test.SvcName, + Namespace: test.HttpNsName, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Name: test.PortName1, + Protocol: test.Protocol1, + Port: test.ServicePort1, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: test.Port1}, + }}, + ClusterIP: "10.108.89.43", + }, + Status: v1.ServiceStatus{}, + }, + want: model.ClusterSetIPType, + }, + { + name: "headless type", + svc: &v1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: test.SvcName, + Namespace: test.HttpNsName, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Name: test.PortName1, + Protocol: test.Protocol1, + Port: test.ServicePort1, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: test.Port1}, + }}, + ClusterIP: "None", + }, + Status: v1.ServiceStatus{}, + }, + want: model.HeadlessType, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ExtractServiceType(tt.svc); got != tt.want { + t.Errorf("ExtractServiceType() = %v, want %v", got, tt.want) + } + }) + } +} From 0354a3d494219b297ae8d8624c2877b9b50237e4 Mon Sep 17 00:00:00 2001 From: Matthew Goodman Date: Fri, 29 Jul 2022 16:01:09 -0700 Subject: [PATCH 094/163] Propagate ClusterID to Cloud Map & Endpoints (#165) ClusterId, ClusterSetId getting exported to Cloud Map & getting propagated to endpoints as CLUSTER_ID and CLUSTERSET_ID attributes --- .../bases/about.k8s.io_clusterproperties.yaml | 59 +++++++++ config/crd/kustomization.yaml | 6 + .../annotation_for_clusterproperties.yaml | 7 ++ .../cainjection_in_clusterproperties.yaml | 7 ++ .../patches/webhook_in_clusterproperties.yaml | 16 +++ ...ut_v1alpha1_clusterproperty_clusterId.yaml | 8 ++ ...v1alpha1_clusterproperty_clustersetId.yaml | 8 ++ integration/janitor/api.go | 3 +- .../shared/scenarios/export_service.go | 3 +- main.go | 23 ++-- .../about/v1alpha1/clusterproperty_types.go | 52 ++++++++ pkg/apis/about/v1alpha1/groupversion_info.go | 20 +++ .../about/v1alpha1/zz_generated.deepcopy.go | 115 ++++++++++++++++++ pkg/cloudmap/api.go | 21 +++- pkg/cloudmap/api_test.go | 18 ++- pkg/cloudmap/client.go | 25 ++-- pkg/cloudmap/client_test.go | 10 +- pkg/common/cluster.go | 65 ++++++++++ .../multicluster/cloudmap_controller.go | 20 ++- .../multicluster/cloudmap_controller_test.go | 11 +- .../multicluster/serviceexport_controller.go | 63 +++++++++- .../serviceexport_controller_test.go | 82 +++++++++++-- pkg/controllers/multicluster/suite_test.go | 10 +- pkg/model/types.go | 16 ++- pkg/model/types_test.go | 60 ++++++++- test/test-constants.go | 42 ++++++- 26 files changed, 713 insertions(+), 57 deletions(-) create mode 100644 config/crd/bases/about.k8s.io_clusterproperties.yaml create mode 100644 config/crd/patches/annotation_for_clusterproperties.yaml create mode 100644 config/crd/patches/cainjection_in_clusterproperties.yaml create mode 100644 config/crd/patches/webhook_in_clusterproperties.yaml create mode 100644 config/samples/about_v1alpha1_clusterproperty_clusterId.yaml create mode 100644 config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml create mode 100644 pkg/apis/about/v1alpha1/clusterproperty_types.go create mode 100644 pkg/apis/about/v1alpha1/groupversion_info.go create mode 100644 pkg/apis/about/v1alpha1/zz_generated.deepcopy.go create mode 100644 pkg/common/cluster.go diff --git a/config/crd/bases/about.k8s.io_clusterproperties.yaml b/config/crd/bases/about.k8s.io_clusterproperties.yaml new file mode 100644 index 00000000..45de7c04 --- /dev/null +++ b/config/crd/bases/about.k8s.io_clusterproperties.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: clusterproperties.about.k8s.io +spec: + group: about.k8s.io + names: + kind: ClusterProperty + listKind: ClusterPropertyList + plural: clusterproperties + singular: clusterproperty + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.value + name: value + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterProperty is the Schema for the clusterproperties API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterPropertySpec defines the desired state of ClusterProperty + properties: + value: + description: ClusterProperty value + minLength: 1 + type: string + required: + - value + type: object + status: + description: ClusterPropertyStatus defines the observed state of ClusterProperty + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 8dac89bc..6dc8a039 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,6 +2,7 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: +- bases/about.k8s.io_clusterproperties.yaml - bases/multicluster.x-k8s.io_serviceexports.yaml - bases/multicluster.x-k8s.io_serviceimports.yaml #+kubebuilder:scaffold:crdkustomizeresource @@ -9,16 +10,21 @@ resources: patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_clusterproperties.yaml #- patches/webhook_in_serviceexports.yaml #- patches/webhook_in_serviceimports.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_clusterproperties.yaml #- patches/cainjection_in_serviceexports.yaml #- patches/cainjection_in_serviceimports.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch +# Patch adds an annotation to pass protected groups approval required to use domain "k8s.io" +- patches/annotation_for_clusterproperties.yaml + # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: - kustomizeconfig.yaml diff --git a/config/crd/patches/annotation_for_clusterproperties.yaml b/config/crd/patches/annotation_for_clusterproperties.yaml new file mode 100644 index 00000000..8dc306ce --- /dev/null +++ b/config/crd/patches/annotation_for_clusterproperties.yaml @@ -0,0 +1,7 @@ +# The following patch adds an annotation to pass protected groups approval required to use domain "k8s.io" +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: "https://github.com/kubernetes/enhancements/pull/3084" + name: clusterproperties.about.k8s.io diff --git a/config/crd/patches/cainjection_in_clusterproperties.yaml b/config/crd/patches/cainjection_in_clusterproperties.yaml new file mode 100644 index 00000000..31e5f39b --- /dev/null +++ b/config/crd/patches/cainjection_in_clusterproperties.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: clusterproperties.about.k8s.io diff --git a/config/crd/patches/webhook_in_clusterproperties.yaml b/config/crd/patches/webhook_in_clusterproperties.yaml new file mode 100644 index 00000000..c1880095 --- /dev/null +++ b/config/crd/patches/webhook_in_clusterproperties.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterproperties.about.k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/samples/about_v1alpha1_clusterproperty_clusterId.yaml b/config/samples/about_v1alpha1_clusterproperty_clusterId.yaml new file mode 100644 index 00000000..76bc3b02 --- /dev/null +++ b/config/samples/about_v1alpha1_clusterproperty_clusterId.yaml @@ -0,0 +1,8 @@ +# An example object of `id.k8s.io ClusterProperty` + +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: sample-mcs-clusterid diff --git a/config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml b/config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml new file mode 100644 index 00000000..10d3e82c --- /dev/null +++ b/config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml @@ -0,0 +1,8 @@ +# An example object of `clusterset.k8s.io ClusterProperty`: + +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: sample-mcs-clustersetid \ No newline at end of file diff --git a/integration/janitor/api.go b/integration/janitor/api.go index 6e00a400..c66f09cc 100644 --- a/integration/janitor/api.go +++ b/integration/janitor/api.go @@ -4,6 +4,7 @@ import ( "context" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" ) @@ -21,7 +22,7 @@ type serviceDiscoveryJanitorApi struct { func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config) ServiceDiscoveryJanitorApi { return &serviceDiscoveryJanitorApi{ - ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg), + ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg, common.ClusterUtils{}), janitorFacade: NewSdkJanitorFacadeFromConfig(cfg), } } diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index 3d7caa57..1b45588f 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -8,6 +8,7 @@ import ( "time" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" multiclustercontrollers "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers/multicluster" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" @@ -69,7 +70,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po NsTTL: time.Second, SvcTTL: time.Second, EndptTTL: time.Second, - }), + }, common.ClusterUtils{}), expectedSvc: model.Service{ Namespace: nsName, Name: svcName, diff --git a/main.go b/main.go index 23483879..7f30de81 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" multiclustercontrollers "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers/multicluster" // +kubebuilder:scaffold:imports @@ -36,6 +37,8 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(multiclusterv1alpha1.AddToScheme(scheme)) + + utilruntime.Must(aboutv1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -84,21 +87,25 @@ func main() { log.Info("Running with AWS region", "AWS_REGION", awsCfg.Region) - serviceDiscoveryClient := cloudmap.NewDefaultServiceDiscoveryClient(&awsCfg) + clusterUtils := common.NewClusterUtils(mgr.GetClient()) + serviceDiscoveryClient := cloudmap.NewDefaultServiceDiscoveryClient(&awsCfg, clusterUtils) + if err = (&multiclustercontrollers.ServiceExportReconciler{ - Client: mgr.GetClient(), - Log: common.NewLogger("controllers", "ServiceExport"), - Scheme: mgr.GetScheme(), - CloudMap: serviceDiscoveryClient, + Client: mgr.GetClient(), + Log: common.NewLogger("controllers", "ServiceExport"), + Scheme: mgr.GetScheme(), + CloudMap: serviceDiscoveryClient, + ClusterUtils: clusterUtils, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "ServiceExport") os.Exit(1) } cloudMapReconciler := &multiclustercontrollers.CloudMapReconciler{ - Client: mgr.GetClient(), - Cloudmap: serviceDiscoveryClient, - Log: common.NewLogger("controllers", "Cloudmap"), + Client: mgr.GetClient(), + Cloudmap: serviceDiscoveryClient, + Log: common.NewLogger("controllers", "Cloudmap"), + ClusterUtils: clusterUtils, } if err = mgr.Add(cloudMapReconciler); err != nil { diff --git a/pkg/apis/about/v1alpha1/clusterproperty_types.go b/pkg/apis/about/v1alpha1/clusterproperty_types.go new file mode 100644 index 00000000..f3fd599e --- /dev/null +++ b/pkg/apis/about/v1alpha1/clusterproperty_types.go @@ -0,0 +1,52 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ClusterPropertySpec defines the desired state of ClusterProperty +type ClusterPropertySpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // ClusterProperty value + // +kubebuilder:validation:Maxlength=128000 + // +kubebuilder:validation:MinLength=1 + Value string `json:"value"` +} + +// ClusterPropertyStatus defines the observed state of ClusterProperty +type ClusterPropertyStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster + +// ClusterProperty is the Schema for the clusterproperties API +// +kubebuilder:printcolumn:name="value",type=string,JSONPath=`.spec.value` +// +kubebuilder:printcolumn:name="age",type=date,JSONPath=`.metadata.creationTimestamp` +type ClusterProperty struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ClusterPropertySpec `json:"spec,omitempty"` + Status ClusterPropertyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// ClusterPropertyList contains a list of ClusterProperty +type ClusterPropertyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterProperty `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ClusterProperty{}, &ClusterPropertyList{}) +} diff --git a/pkg/apis/about/v1alpha1/groupversion_info.go b/pkg/apis/about/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..82f56067 --- /dev/null +++ b/pkg/apis/about/v1alpha1/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1alpha1 contains API Schema definitions for the about v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=about.k8s.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "about.k8s.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/pkg/apis/about/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/about/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..476fb37f --- /dev/null +++ b/pkg/apis/about/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,115 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterProperty) DeepCopyInto(out *ClusterProperty) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterProperty. +func (in *ClusterProperty) DeepCopy() *ClusterProperty { + if in == nil { + return nil + } + out := new(ClusterProperty) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterProperty) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterPropertyList) DeepCopyInto(out *ClusterPropertyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterProperty, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPropertyList. +func (in *ClusterPropertyList) DeepCopy() *ClusterPropertyList { + if in == nil { + return nil + } + out := new(ClusterPropertyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterPropertyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterPropertySpec) DeepCopyInto(out *ClusterPropertySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPropertySpec. +func (in *ClusterPropertySpec) DeepCopy() *ClusterPropertySpec { + if in == nil { + return nil + } + out := new(ClusterPropertySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterPropertyStatus) DeepCopyInto(out *ClusterPropertyStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPropertyStatus. +func (in *ClusterPropertyStatus) DeepCopy() *ClusterPropertyStatus { + if in == nil { + return nil + } + out := new(ClusterPropertyStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index 38bad30f..ad33af9e 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -52,15 +52,17 @@ type ServiceDiscoveryApi interface { } type serviceDiscoveryApi struct { - log common.Logger - awsFacade AwsFacade + log common.Logger + awsFacade AwsFacade + clusterUtils common.ClusterUtils } // NewServiceDiscoveryApiFromConfig creates a new AWS Cloud Map API connection manager from an AWS client config. -func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { +func NewServiceDiscoveryApiFromConfig(cfg *aws.Config, clusterUtils common.ClusterUtils) ServiceDiscoveryApi { return &serviceDiscoveryApi{ - log: common.NewLogger("cloudmap"), - awsFacade: NewAwsFacadeFromConfig(cfg), + log: common.NewLogger("cloudmap"), + awsFacade: NewAwsFacadeFromConfig(cfg), + clusterUtils: clusterUtils, } } @@ -114,11 +116,20 @@ func (sdApi *serviceDiscoveryApi) GetServiceIdMap(ctx context.Context, nsId stri } func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string) (insts []types.HttpInstanceSummary, err error) { + clusterSetId, err := sdApi.clusterUtils.GetClusterSetId(ctx) + if err != nil { + sdApi.log.Error(err, "failed to retrieve clusterSetId") + return nil, err + } + out, err := sdApi.awsFacade.DiscoverInstances(ctx, &sd.DiscoverInstancesInput{ NamespaceName: aws.String(nsName), ServiceName: aws.String(svcName), HealthStatus: types.HealthStatusFilterAll, MaxResults: aws.Int32(1000), + QueryParameters: map[string]string{ + model.ClusterSetIdAttr: clusterSetId, + }, }) if err != nil { diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 8d70d9c0..cb90b8e6 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -6,8 +6,11 @@ import ( "fmt" "testing" + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" @@ -15,10 +18,12 @@ import ( "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestNewServiceDiscoveryApi(t *testing.T) { - sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}) + sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}, common.ClusterUtils{}) assert.NotNil(t, sdc) } @@ -101,6 +106,9 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { ServiceName: aws.String(test.SvcName), HealthStatus: types.HealthStatusFilterAll, MaxResults: aws.Int32(1000), + QueryParameters: map[string]string{ + model.ClusterSetIdAttr: test.ClustersetId, + }, }). Return(&sd.DiscoverInstancesOutput{ Instances: []types.HttpInstanceSummary{ @@ -321,8 +329,12 @@ func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { } func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmapMock.MockAwsFacade) ServiceDiscoveryApi { + scheme := runtime.NewScheme() + scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) + fakeClient := fake.NewClientBuilder().WithObjects(test.ClusterIdForTest(), test.ClusterSetIdForTest()).WithScheme(scheme).Build() return &serviceDiscoveryApi{ - log: common.NewLoggerWithLogr(testr.New(t)), - awsFacade: awsFacade, + log: common.NewLoggerWithLogr(testr.New(t)), + awsFacade: awsFacade, + clusterUtils: common.NewClusterUtils(fakeClient), } } diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 66eae577..1fdfa1a8 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -30,26 +30,29 @@ type ServiceDiscoveryClient interface { } type serviceDiscoveryClient struct { - log common.Logger - sdApi ServiceDiscoveryApi - cache ServiceDiscoveryClientCache + log common.Logger + sdApi ServiceDiscoveryApi + cache ServiceDiscoveryClientCache + clusterUtils common.ClusterUtils } // NewDefaultServiceDiscoveryClient creates a new service discovery client for AWS Cloud Map with default resource cache // from a given AWS client config. -func NewDefaultServiceDiscoveryClient(cfg *aws.Config) ServiceDiscoveryClient { +func NewDefaultServiceDiscoveryClient(cfg *aws.Config, clusterUtils common.ClusterUtils) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: common.NewLogger("cloudmap"), - sdApi: NewServiceDiscoveryApiFromConfig(cfg), - cache: NewDefaultServiceDiscoveryClientCache(), + log: common.NewLogger("cloudmap"), + sdApi: NewServiceDiscoveryApiFromConfig(cfg, clusterUtils), + cache: NewDefaultServiceDiscoveryClientCache(), + clusterUtils: clusterUtils, } } -func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCacheConfig) ServiceDiscoveryClient { +func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCacheConfig, clusterUtils common.ClusterUtils) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: common.NewLogger("cloudmap"), - sdApi: NewServiceDiscoveryApiFromConfig(cfg), - cache: NewServiceDiscoveryClientCache(cacheConfig), + log: common.NewLogger("cloudmap"), + sdApi: NewServiceDiscoveryApiFromConfig(cfg, clusterUtils), + cache: NewServiceDiscoveryClientCache(cacheConfig), + clusterUtils: clusterUtils, } } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index bc03a9c4..5729943f 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -24,7 +24,7 @@ type testSdClient struct { } func TestNewServiceDiscoveryClient(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}) + sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}, common.ClusterUtils{}) assert.NotNil(t, sdc) } @@ -283,6 +283,8 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) attrs1 := map[string]string{ + model.ClusterIdAttr: test.ClusterId, + model.ClusterSetIdAttr: test.ClustersetId, model.EndpointIpv4Attr: test.EndptIp1, model.EndpointPortAttr: test.PortStr1, model.EndpointPortNameAttr: test.PortName1, @@ -294,6 +296,8 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { model.ServiceTypeAttr: test.SvcType, } attrs2 := map[string]string{ + model.ClusterIdAttr: test.ClusterId, + model.ClusterSetIdAttr: test.ClustersetId, model.EndpointIpv4Attr: test.EndptIp2, model.EndpointPortAttr: test.PortStr2, model.EndpointPortNameAttr: test.PortName2, @@ -364,6 +368,8 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { { InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ + model.ClusterIdAttr: test.ClusterId, + model.ClusterSetIdAttr: test.ClustersetId, model.EndpointIpv4Attr: test.EndptIp1, model.EndpointPortAttr: test.PortStr1, model.EndpointPortNameAttr: test.PortName1, @@ -378,6 +384,8 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { { InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ + model.ClusterIdAttr: test.ClusterId, + model.ClusterSetIdAttr: test.ClustersetId, model.EndpointIpv4Attr: test.EndptIp2, model.EndpointPortAttr: test.PortStr2, model.EndpointPortNameAttr: test.PortName2, diff --git a/pkg/common/cluster.go b/pkg/common/cluster.go new file mode 100644 index 00000000..1966a4df --- /dev/null +++ b/pkg/common/cluster.go @@ -0,0 +1,65 @@ +package common + +import ( + "context" + "fmt" + + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ClusterIdName = "id.k8s.io" + ClusterSetIdName = "clusterset.k8s.io" +) + +// ClusterUtils provides utility functions for working with clusters +type ClusterUtils struct { + client client.Client + clusterId string + clusterSetId string +} + +// constructor +func NewClusterUtils(client client.Client) ClusterUtils { + return ClusterUtils{ + client: client, + } +} + +// retrieve the clusterId from the local field. If not set, retrieve from client +func (r *ClusterUtils) GetClusterId(ctx context.Context) (string, error) { + if r.clusterId != "" { + return r.clusterId, nil + } + clusterPropertyForClusterId := &aboutv1alpha1.ClusterProperty{} + err := r.client.Get(ctx, client.ObjectKey{Name: ClusterIdName}, clusterPropertyForClusterId) + if err != nil { + return "", err + } + if clusterPropertyForClusterId.Spec.Value == "" { + err := fmt.Errorf("ClusterId not found") + return "", err + } + r.clusterId = clusterPropertyForClusterId.Spec.Value + return r.clusterId, nil +} + +// retrieve the clusterSetId from the local field. If not set, retrieve from client +func (r *ClusterUtils) GetClusterSetId(ctx context.Context) (string, error) { + if r.clusterSetId != "" { + return r.clusterSetId, nil + } + clusterPropertyForClusterSetId := &aboutv1alpha1.ClusterProperty{} + err := r.client.Get(ctx, client.ObjectKey{Name: ClusterSetIdName}, clusterPropertyForClusterSetId) + if err != nil { + return "", err + } + if clusterPropertyForClusterSetId.Spec.Value == "" { + err := fmt.Errorf("ClusterSetId not found") + return "", err + } + r.clusterSetId = clusterPropertyForClusterSetId.Spec.Value + return r.clusterSetId, nil +} diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index a9fe22fd..276cb651 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -23,9 +23,10 @@ const ( // CloudMapReconciler reconciles state of Cloud Map services with local ServiceImport objects type CloudMapReconciler struct { - Client client.Client - Cloudmap cloudmap.ServiceDiscoveryClient - Log common.Logger + Client client.Client + Cloudmap cloudmap.ServiceDiscoveryClient + Log common.Logger + ClusterUtils common.ClusterUtils } // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch @@ -53,6 +54,19 @@ func (r *CloudMapReconciler) Start(ctx context.Context) error { // Reconcile triggers a single reconciliation round func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { + var err error + clusterId, err := r.ClusterUtils.GetClusterId(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve clusterId") + return err + } + clusterSetId, err := r.ClusterUtils.GetClusterSetId(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve clusterSetId") + return err + } + r.Log.Debug("ClusterId and ClusterSetId found", "ClusterId", clusterId, "ClusterSetId", clusterSetId) + namespaces := v1.NamespaceList{} if err := r.Client.List(ctx, &namespaces); err != nil { r.Log.Error(err, "unable to list cluster namespaces") diff --git a/pkg/controllers/multicluster/cloudmap_controller_test.go b/pkg/controllers/multicluster/cloudmap_controller_test.go index d1c42bcd..281e4493 100644 --- a/pkg/controllers/multicluster/cloudmap_controller_test.go +++ b/pkg/controllers/multicluster/cloudmap_controller_test.go @@ -6,6 +6,7 @@ import ( "testing" cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -28,8 +29,9 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { s := scheme.Scheme s.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceImportList{}, &multiclusterv1alpha1.ServiceImport{}) + s.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) - fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).WithObjects(test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() // create a mock cloudmap service discovery client mockController := gomock.NewController(t) @@ -75,8 +77,9 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { func getReconciler(t *testing.T, mockSDClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *CloudMapReconciler { return &CloudMapReconciler{ - Client: client, - Cloudmap: mockSDClient, - Log: common.NewLoggerWithLogr(testr.New(t)), + Client: client, + Cloudmap: mockSDClient, + Log: common.NewLoggerWithLogr(testr.New(t)), + ClusterUtils: common.NewClusterUtils(client), } } diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index 6bc456d8..535e4e98 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -35,10 +36,11 @@ const ( // ServiceExportReconciler reconciles a ServiceExport object type ServiceExportReconciler struct { - Client client.Client - Log common.Logger - Scheme *runtime.Scheme - CloudMap cloudmap.ServiceDiscoveryClient + Client client.Client + Log common.Logger + Scheme *runtime.Scheme + CloudMap cloudmap.ServiceDiscoveryClient + ClusterUtils common.ClusterUtils } // +kubebuilder:rbac:groups="",resources=services,verbs=get @@ -54,6 +56,19 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques name := req.NamespacedName r.Log.Debug("reconciling ServiceExport", "Namespace", namespace, "Name", name) + var err error + clusterId, err := r.ClusterUtils.GetClusterId(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve clusterId") + return ctrl.Result{}, err + } + clusterSetId, err := r.ClusterUtils.GetClusterSetId(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve clusterSetId") + return ctrl.Result{}, err + } + r.Log.Debug("ClusterId and ClusterSetId found", "ClusterId", clusterId, "ClusterSetId", clusterSetId) + serviceExport := multiclusterv1alpha1.ServiceExport{} if err := r.Client.Get(ctx, name, &serviceExport); err != nil { if errors.IsNotFound(err) { @@ -232,6 +247,17 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. servicePortMap[svcPort.Name] = ServicePortToPort(svcPort) } + clusterId, err := r.ClusterUtils.GetClusterId(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve clusterId") + return nil, err + } + clusterSetId, err := r.ClusterUtils.GetClusterSetId(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve clusterSetId") + return nil, err + } + for _, slice := range endpointSlices.Items { if slice.AddressType != discovery.AddressTypeIPv4 { return nil, fmt.Errorf("unsupported address type %s for service %s", slice.AddressType, svc.Name) @@ -243,6 +269,7 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. if version.GetVersion() != "" { attributes[K8sVersionAttr] = version.PackageName + " " + version.GetVersion() } + // TODO extract attributes - pod, node and other useful details if possible port := EndpointPortToPort(endpointPort) @@ -251,6 +278,8 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. IP: IP, EndpointPort: port, ServicePort: servicePortMap[*endpointPort.Name], + ClusterId: clusterId, + ClusterSetId: clusterSetId, ServiceType: serviceType, Attributes: attributes, }) @@ -273,6 +302,12 @@ func (r *ServiceExportReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.endpointSliceEventHandler()), builder.WithPredicates(r.endpointSliceFilter()), ). + // Watch for changes to ClusterProperty objects. If a ClusterProperty object is + // created, updated or deleted, the controller will reconcile all service exports + Watches( + &source.Kind{Type: &aboutv1alpha1.ClusterProperty{}}, + handler.EnqueueRequestsFromMapFunc(r.clusterPropertyEventHandler()), + ). Complete(r) } @@ -289,6 +324,26 @@ func (r *ServiceExportReconciler) endpointSliceEventHandler() handler.MapFunc { } } +func (r *ServiceExportReconciler) clusterPropertyEventHandler() handler.MapFunc { + // Return reconcile requests for all service exports + return func(object client.Object) []reconcile.Request { + serviceExports := &multiclusterv1alpha1.ServiceExportList{} + if err := r.Client.List(context.TODO(), serviceExports); err != nil { + r.Log.Error(err, "error listing services") + return nil + } + + result := make([]reconcile.Request, 0) + for _, serviceExport := range serviceExports.Items { + result = append(result, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: serviceExport.Name, + Namespace: serviceExport.Namespace, + }}) + } + return result + } +} + func (r *ServiceExportReconciler) endpointSliceFilter() predicate.Funcs { return predicate.Funcs{ GenericFunc: func(e event.GenericEvent) bool { diff --git a/pkg/controllers/multicluster/serviceexport_controller_test.go b/pkg/controllers/multicluster/serviceexport_controller_test.go index 327c9693..45eaaa16 100644 --- a/pkg/controllers/multicluster/serviceexport_controller_test.go +++ b/pkg/controllers/multicluster/serviceexport_controller_test.go @@ -2,8 +2,10 @@ package controllers import ( "context" + "fmt" cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -27,7 +29,7 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { // create a fake controller client and add some objects fakeClient := fake.NewClientBuilder(). WithScheme(getServiceExportScheme()). - WithObjects(k8sServiceForTest(), serviceExportForTest()). + WithObjects(k8sServiceForTest(), serviceExportForTest(), test.ClusterIdForTest(), test.ClusterSetIdForTest()). WithLists(&discovery.EndpointSliceList{ Items: []discovery.EndpointSlice{*endpointSliceForTest()}, }). @@ -75,7 +77,7 @@ func TestServiceExportReconciler_Reconcile_ExistingServiceExport(t *testing.T) { // create a fake controller client and add some objects fakeClient := fake.NewClientBuilder(). WithScheme(getServiceExportScheme()). - WithObjects(k8sServiceForTest(), serviceExportForTest()). + WithObjects(k8sServiceForTest(), serviceExportForTest(), test.ClusterIdForTest(), test.ClusterSetIdForTest()). WithLists(&discovery.EndpointSliceList{ Items: []discovery.EndpointSlice{*endpointSliceForTest()}, }). @@ -122,7 +124,7 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { serviceExportObj.Finalizers = []string{ServiceExportFinalizer} fakeClient := fake.NewClientBuilder(). WithScheme(getServiceExportScheme()). - WithObjects(serviceExportObj). + WithObjects(serviceExportObj, test.ClusterIdForTest(), test.ClusterSetIdForTest()). WithLists(&discovery.EndpointSliceList{ Items: []discovery.EndpointSlice{*endpointSliceForTest()}, }). @@ -159,8 +161,73 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { assert.Empty(t, serviceExport.Finalizers, "Finalizer removed from the service export") } +func TestServiceExportReconciler_Reconcile_NoClusterId(t *testing.T) { + // create a fake controller client and add some objects + fakeClient := fake.NewClientBuilder(). + WithScheme(getServiceExportScheme()). + WithObjects(k8sServiceForTest(), serviceExportForTest(), test.ClusterSetIdForTest()). + WithLists(&discovery.EndpointSliceList{ + Items: []discovery.EndpointSlice{*endpointSliceForTest()}, + }). + Build() + + // create a mock cloudmap service discovery client + mockController := gomock.NewController(t) + defer mockController.Finish() + + mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) + + reconciler := getServiceExportReconciler(t, mock, fakeClient) + + request := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: test.HttpNsName, + Name: test.SvcName, + }, + } + + // Reconciling should throw an error + got, err := reconciler.Reconcile(context.Background(), request) + expectedError := fmt.Errorf("clusterproperties.about.k8s.io \"id.k8s.io\" not found") + assert.ErrorContains(t, err, expectedError.Error()) + assert.Equal(t, ctrl.Result{}, got, "Result should be empty") +} + +func TestServiceExportReconciler_Reconcile_NoClustersetId(t *testing.T) { + // create a fake controller client and add some objects + fakeClient := fake.NewClientBuilder(). + WithScheme(getServiceExportScheme()). + WithObjects(k8sServiceForTest(), serviceExportForTest(), test.ClusterIdForTest()). + WithLists(&discovery.EndpointSliceList{ + Items: []discovery.EndpointSlice{*endpointSliceForTest()}, + }). + Build() + + // create a mock cloudmap service discovery client + mockController := gomock.NewController(t) + defer mockController.Finish() + + mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) + + reconciler := getServiceExportReconciler(t, mock, fakeClient) + + request := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: test.HttpNsName, + Name: test.SvcName, + }, + } + + // Reconciling should throw an error + got, err := reconciler.Reconcile(context.Background(), request) + expectedError := fmt.Errorf("clusterproperties.about.k8s.io \"clusterset.k8s.io\" not found") + assert.ErrorContains(t, err, expectedError.Error()) + assert.Equal(t, ctrl.Result{}, got, "Result should be empty") +} + func getServiceExportScheme() *runtime.Scheme { scheme := runtime.NewScheme() + scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) scheme.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceExport{}) scheme.AddKnownTypes(v1.SchemeGroupVersion, &v1.Service{}) scheme.AddKnownTypes(discovery.SchemeGroupVersion, &discovery.EndpointSlice{}, &discovery.EndpointSliceList{}) @@ -169,9 +236,10 @@ func getServiceExportScheme() *runtime.Scheme { func getServiceExportReconciler(t *testing.T, mockClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *ServiceExportReconciler { return &ServiceExportReconciler{ - Client: client, - Log: common.NewLoggerWithLogr(testr.New(t)), - Scheme: client.Scheme(), - CloudMap: mockClient, + Client: client, + Log: common.NewLoggerWithLogr(testr.New(t)), + Scheme: client.Scheme(), + CloudMap: mockClient, + ClusterUtils: common.NewClusterUtils(client), } } diff --git a/pkg/controllers/multicluster/suite_test.go b/pkg/controllers/multicluster/suite_test.go index c37817c3..6ea09944 100644 --- a/pkg/controllers/multicluster/suite_test.go +++ b/pkg/controllers/multicluster/suite_test.go @@ -37,8 +37,16 @@ var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) By("bootstrapping test environment") + + // TODO: Add CRDs for about.k8s.io_clusterproperties & add patch + + crds := []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases", "multicluster.x-k8s.io_serviceexports.yaml"), + filepath.Join("..", "..", "..", "config", "crd", "bases", "multicluster.x-k8s.io_serviceimports.yaml"), + } + testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: crds, ErrorIfCRDPathMissing: true, } diff --git a/pkg/model/types.go b/pkg/model/types.go index 5c6193f0..e71d94d9 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -46,6 +46,8 @@ type Endpoint struct { IP string EndpointPort Port ServicePort Port + ClusterId string + ClusterSetId string ServiceType ServiceType Attributes map[string]string } @@ -64,6 +66,8 @@ const ( EndpointPortAttr = "AWS_INSTANCE_PORT" EndpointPortNameAttr = "ENDPOINT_PORT_NAME" EndpointProtocolAttr = "ENDPOINT_PROTOCOL" + ClusterIdAttr = "CLUSTER_ID" + ClusterSetIdAttr = "CLUSTERSET_ID" ServicePortNameAttr = "SERVICE_PORT_NAME" ServicePortAttr = "SERVICE_PORT" ServiceTargetPortAttr = "SERVICE_TARGET_PORT" @@ -82,7 +86,7 @@ func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) attributes[key] = value } - // Remove and set the IP, Port, Service Port, ServiceType + // Remove and set the IP, Port, Service Port, ServiceType, ClusterId, ClusterSetId ip, err := removeStringAttr(attributes, EndpointIpv4Attr) if err != nil { return nil, err @@ -107,6 +111,14 @@ func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) } endpoint.ServiceType = ServiceType(serviceTypeStr) + if endpoint.ClusterId, err = removeStringAttr(attributes, ClusterIdAttr); err != nil { + return nil, err + } + + if endpoint.ClusterSetId, err = removeStringAttr(attributes, ClusterSetIdAttr); err != nil { + return nil, err + } + // Add the remaining attributes endpoint.Attributes = attributes @@ -169,6 +181,8 @@ func removeIntAttr(attributes map[string]string, attr string) (int32, error) { func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs := make(map[string]string) + attrs[ClusterIdAttr] = e.ClusterId + attrs[ClusterSetIdAttr] = e.ClusterSetId attrs[EndpointIpv4Attr] = e.IP attrs[EndpointPortAttr] = strconv.Itoa(int(e.EndpointPort.Port)) attrs[EndpointProtocolAttr] = e.EndpointPort.Protocol diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index b580acab..2ef9a66c 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -9,6 +9,8 @@ import ( var instId = "my-instance" var ip = "192.168.0.1" +var clusterId = "test-mcs-clusterId" +var clusterSetId = "test-mcs-clusterSetId" var serviceType = ClusterSetIPType.String() func TestNewEndpointFromInstance(t *testing.T) { @@ -23,6 +25,8 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, EndpointIpv4Attr: ip, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", @@ -49,7 +53,9 @@ func TestNewEndpointFromInstance(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, - ServiceType: ServiceType(serviceType), + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), Attributes: map[string]string{ "custom-attr": "custom-val", }, @@ -97,10 +103,50 @@ func TestNewEndpointFromInstance(t *testing.T) { wantErr: true, }, { - name: "missing ServiceType", + name: "missing clusterid", inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ + ClusterSetIdAttr: clusterSetId, + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + "custom-attr": "custom-val", + }, + }, + wantErr: true, + }, + { + name: "missing clustersetid", + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, + Attributes: map[string]string{ + ClusterIdAttr: clusterId, + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + "custom-attr": "custom-val", + }, + }, + wantErr: true, + }, + { + name: "missing service type", + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, + Attributes: map[string]string{ + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, EndpointIpv4Attr: ip, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", @@ -135,6 +181,8 @@ func TestEndpoint_GetAttributes(t *testing.T) { IP string EndpointPort Port ServicePort Port + ClusterId string + ClusterSetId string ServiceType ServiceType Attributes map[string]string } @@ -158,12 +206,16 @@ func TestEndpoint_GetAttributes(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, - ServiceType: ServiceType(serviceType), + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), Attributes: map[string]string{ "custom-attr": "custom-val", }, }, want: map[string]string{ + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, EndpointIpv4Attr: ip, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", @@ -184,6 +236,8 @@ func TestEndpoint_GetAttributes(t *testing.T) { IP: tt.fields.IP, EndpointPort: tt.fields.EndpointPort, ServicePort: tt.fields.ServicePort, + ClusterId: tt.fields.ClusterId, + ClusterSetId: tt.fields.ClusterSetId, ServiceType: tt.fields.ServiceType, Attributes: tt.fields.Attributes, } diff --git a/test/test-constants.go b/test/test-constants.go index 91f90c55..22b0e108 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -3,6 +3,11 @@ package test import ( "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" ) @@ -13,6 +18,8 @@ const ( DnsNsId = "dns-ns-id" SvcName = "svc-name" SvcId = "svc-id" + ClusterId = "test-mcs-clusterId" + ClustersetId = "test-mcs-clusterSetId" EndptId1 = "tcp-192_168_0_1-1" EndptId2 = "tcp-192_168_0_2-2" EndptIp1 = "192.168.0.1" @@ -82,8 +89,10 @@ func GetTestEndpoint1() *model.Endpoint { TargetPort: PortStr1, Protocol: Protocol1, }, - ServiceType: model.ClusterSetIPType, - Attributes: make(map[string]string), + ClusterId: ClusterId, + ClusterSetId: ClustersetId, + ServiceType: model.ClusterSetIPType, + Attributes: make(map[string]string), } } @@ -102,8 +111,10 @@ func GetTestEndpoint2() *model.Endpoint { TargetPort: PortStr2, Protocol: Protocol2, }, - ServiceType: model.ClusterSetIPType, - Attributes: make(map[string]string), + ClusterId: ClusterId, + ClusterSetId: ClustersetId, + ServiceType: model.ClusterSetIPType, + Attributes: make(map[string]string), } } @@ -111,9 +122,32 @@ func GetTestEndpoints(count int) (endpts []*model.Endpoint) { // use +3 offset go avoid collision with test endpoint 1 and 2 for i := 3; i < count+3; i++ { e := GetTestEndpoint1() + e.ClusterId = ClusterId e.Id = fmt.Sprintf("tcp-192_168_0_%d-1", i) e.IP = fmt.Sprintf("192.168.0.%d", i) endpts = append(endpts, e) } return endpts } + +func ClusterIdForTest() *aboutv1alpha1.ClusterProperty { + return &aboutv1alpha1.ClusterProperty{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ClusterIdName, + }, + Spec: aboutv1alpha1.ClusterPropertySpec{ + Value: ClusterId, + }, + } +} + +func ClusterSetIdForTest() *aboutv1alpha1.ClusterProperty { + return &aboutv1alpha1.ClusterProperty{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ClusterSetIdName, + }, + Spec: aboutv1alpha1.ClusterPropertySpec{ + Value: ClustersetId, + }, + } +} From b8a3b1e77e98c449cea1f74c565dd18b4b76c316 Mon Sep 17 00:00:00 2001 From: Matthew Goodman Date: Tue, 2 Aug 2022 14:21:41 -0700 Subject: [PATCH 095/163] Fix integration tests + update docs (#178) * Added managed by label - issue 110 * fixed integration tests * update docs + add samples * updated readme + contributing * update make install expected output * update contributing --- CONTRIBUTING.md | 12 ++++- README.md | 45 +++++++++++++++++-- integration/janitor/api.go | 3 +- .../kind-test/configs/e2e-clusterId.yaml | 8 ++++ .../kind-test/configs/e2e-clusterSetId.yaml | 8 ++++ integration/kind-test/scripts/run-tests.sh | 3 ++ .../shared/scenarios/export_service.go | 5 ++- pkg/cloudmap/api_test.go | 4 +- pkg/cloudmap/client_test.go | 10 ++--- pkg/common/cluster.go | 8 ++++ samples/example-clusterproperty.yaml | 17 +++++++ test/test-constants.go | 8 ++-- 12 files changed, 114 insertions(+), 17 deletions(-) create mode 100644 integration/kind-test/configs/e2e-clusterId.yaml create mode 100644 integration/kind-test/configs/e2e-clusterSetId.yaml create mode 100644 samples/example-clusterproperty.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 68a94ae7..6b974cb7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,14 +74,24 @@ kubectl create namespace example #### Run the controller from outside the cluster -To register the custom CRDs (`ServiceImport`, `ServiceExport`) in the cluster and create installers: +To register the custom CRDs (`ClusterProperties`, `ServiceImport`, `ServiceExport`) in the cluster and create installers: ```sh make install # ... +# customresourcedefinition.apiextensions.k8s.io/clusterproperties.about.k8s.io created # customresourcedefinition.apiextensions.k8s.io/serviceexports.multicluster.x-k8s.io created # customresourcedefinition.apiextensions.k8s.io/serviceimports.multicluster.x-k8s.io created ``` +Register a unique `id.k8s.io` and `clusterset.k8s.io` in your cluster: +```bash +kubectl apply -f samples/example-clusterproperty.yaml +# clusterproperty.about.k8s.io/id.k8s.io created +# clusterproperty.about.k8s.io/clusterset.k8s.io created +``` +> ⚠ **Note:** If you are creating multiple clusters, ensure you create unique `id.k8s.io` identifiers for each cluster. + + To run the controller, run the following command. The controller runs in an infinite loop so open another terminal to create CRDs. ```sh make run diff --git a/README.md b/README.md index 411d4668..6d4f8066 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,9 @@ Install the The CoreDNS multicluster plugin into each participating cluster. The To install the plugin, run the following commands. ```bash -kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-clusterrole.yaml -kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-configmap.yaml -kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-deployment.yaml +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-clusterrole.yaml" +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-configmap.yaml" +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/samples/coredns-deployment.yaml" ``` ### Install Controller @@ -63,6 +63,45 @@ The controller must have sufficient IAM permissions to perform required Cloud Ma ## Usage +### Configure `id.k8s.io` and `clusterset.k8s.io` + +`id.k8s.io` is a unique identifier that uniquely identifies a cluster. + +`clusterset.k8s.io` is a unique identifier that uniquely identifies the set of clusters in your multicluster. This will be the same across all clusters in the set. + +```yaml +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: [Your cluster identifier] +--- +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: [Your cluster set identifier] +``` + +**Example:** +```yaml +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: my-first-cluster +--- +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: my-clusterset +``` + ### Export services Then assuming you already have a Service installed, apply a `ServiceExport` yaml to the cluster in which you want to export a service. This can be done for each service you want to export. diff --git a/integration/janitor/api.go b/integration/janitor/api.go index c66f09cc..2e99ac63 100644 --- a/integration/janitor/api.go +++ b/integration/janitor/api.go @@ -5,6 +5,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" ) @@ -22,7 +23,7 @@ type serviceDiscoveryJanitorApi struct { func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config) ServiceDiscoveryJanitorApi { return &serviceDiscoveryJanitorApi{ - ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg, common.ClusterUtils{}), + ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)), janitorFacade: NewSdkJanitorFacadeFromConfig(cfg), } } diff --git a/integration/kind-test/configs/e2e-clusterId.yaml b/integration/kind-test/configs/e2e-clusterId.yaml new file mode 100644 index 00000000..cd1ee175 --- /dev/null +++ b/integration/kind-test/configs/e2e-clusterId.yaml @@ -0,0 +1,8 @@ +# An example object of `id.k8s.io ClusterProperty` + +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: test-mcs-clusterId diff --git a/integration/kind-test/configs/e2e-clusterSetId.yaml b/integration/kind-test/configs/e2e-clusterSetId.yaml new file mode 100644 index 00000000..dc884ba8 --- /dev/null +++ b/integration/kind-test/configs/e2e-clusterSetId.yaml @@ -0,0 +1,8 @@ +# An example object of `clusterset.k8s.io ClusterProperty`: + +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: test-mcs-clusterSetId \ No newline at end of file diff --git a/integration/kind-test/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh index 6c5a256c..8a36067d 100755 --- a/integration/kind-test/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -4,6 +4,9 @@ source ./integration/kind-test/scripts/common.sh +$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterId.yaml" +$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterSetId.yaml" + $KUBECTL_BIN apply -f "$CONFIGS/e2e-deployment.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-service.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-export.yaml" diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index 1b45588f..8205ef0e 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" multiclustercontrollers "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers/multicluster" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -59,6 +60,8 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po Protocol: string(v1.ProtocolTCP), }, EndpointPort: endpointPort, + ClusterId: test.ClusterId, + ClusterSetId: test.ClusterSetId, ServiceType: model.ClusterSetIPType, // in scenario, we assume ClusterSetIP type Attributes: make(map[string]string), }) @@ -70,7 +73,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po NsTTL: time.Second, SvcTTL: time.Second, EndptTTL: time.Second, - }, common.ClusterUtils{}), + }, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)), expectedSvc: model.Service{ Namespace: nsName, Name: svcName, diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index cb90b8e6..86dfe296 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -23,7 +23,7 @@ import ( ) func TestNewServiceDiscoveryApi(t *testing.T) { - sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}, common.ClusterUtils{}) + sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)) assert.NotNil(t, sdc) } @@ -107,7 +107,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { HealthStatus: types.HealthStatusFilterAll, MaxResults: aws.Int32(1000), QueryParameters: map[string]string{ - model.ClusterSetIdAttr: test.ClustersetId, + model.ClusterSetIdAttr: test.ClusterSetId, }, }). Return(&sd.DiscoverInstancesOutput{ diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 5729943f..5b1eb750 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -24,7 +24,7 @@ type testSdClient struct { } func TestNewServiceDiscoveryClient(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}, common.ClusterUtils{}) + sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)) assert.NotNil(t, sdc) } @@ -284,7 +284,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { attrs1 := map[string]string{ model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClustersetId, + model.ClusterSetIdAttr: test.ClusterSetId, model.EndpointIpv4Attr: test.EndptIp1, model.EndpointPortAttr: test.PortStr1, model.EndpointPortNameAttr: test.PortName1, @@ -297,7 +297,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { } attrs2 := map[string]string{ model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClustersetId, + model.ClusterSetIdAttr: test.ClusterSetId, model.EndpointIpv4Attr: test.EndptIp2, model.EndpointPortAttr: test.PortStr2, model.EndpointPortNameAttr: test.PortName2, @@ -369,7 +369,7 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClustersetId, + model.ClusterSetIdAttr: test.ClusterSetId, model.EndpointIpv4Attr: test.EndptIp1, model.EndpointPortAttr: test.PortStr1, model.EndpointPortNameAttr: test.PortName1, @@ -385,7 +385,7 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClustersetId, + model.ClusterSetIdAttr: test.ClusterSetId, model.EndpointIpv4Attr: test.EndptIp2, model.EndpointPortAttr: test.PortStr2, model.EndpointPortNameAttr: test.PortName2, diff --git a/pkg/common/cluster.go b/pkg/common/cluster.go index 1966a4df..75a977e2 100644 --- a/pkg/common/cluster.go +++ b/pkg/common/cluster.go @@ -28,6 +28,14 @@ func NewClusterUtils(client client.Client) ClusterUtils { } } +// constructor for tests +func NewClusterUtilsForTest(clusterId string, clusterSetId string) ClusterUtils { + return ClusterUtils{ + clusterId: clusterId, + clusterSetId: clusterSetId, + } +} + // retrieve the clusterId from the local field. If not set, retrieve from client func (r *ClusterUtils) GetClusterId(ctx context.Context) (string, error) { if r.clusterId != "" { diff --git a/samples/example-clusterproperty.yaml b/samples/example-clusterproperty.yaml new file mode 100644 index 00000000..295b6756 --- /dev/null +++ b/samples/example-clusterproperty.yaml @@ -0,0 +1,17 @@ +# An example object of `id.k8s.io ClusterProperty` + +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: sample-mcs-clusterid +--- +# An example object of `clusterset.k8s.io ClusterProperty`: + +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: sample-mcs-clustersetid \ No newline at end of file diff --git a/test/test-constants.go b/test/test-constants.go index 22b0e108..f88d59bb 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -19,7 +19,7 @@ const ( SvcName = "svc-name" SvcId = "svc-id" ClusterId = "test-mcs-clusterId" - ClustersetId = "test-mcs-clusterSetId" + ClusterSetId = "test-mcs-clusterSetId" EndptId1 = "tcp-192_168_0_1-1" EndptId2 = "tcp-192_168_0_2-2" EndptIp1 = "192.168.0.1" @@ -90,7 +90,7 @@ func GetTestEndpoint1() *model.Endpoint { Protocol: Protocol1, }, ClusterId: ClusterId, - ClusterSetId: ClustersetId, + ClusterSetId: ClusterSetId, ServiceType: model.ClusterSetIPType, Attributes: make(map[string]string), } @@ -112,7 +112,7 @@ func GetTestEndpoint2() *model.Endpoint { Protocol: Protocol2, }, ClusterId: ClusterId, - ClusterSetId: ClustersetId, + ClusterSetId: ClusterSetId, ServiceType: model.ClusterSetIPType, Attributes: make(map[string]string), } @@ -147,7 +147,7 @@ func ClusterSetIdForTest() *aboutv1alpha1.ClusterProperty { Name: common.ClusterSetIdName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ - Value: ClustersetId, + Value: ClusterSetId, }, } } From aab4c29ae855ce29eeee962420d8e4b838092615 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Aug 2022 14:35:55 -0700 Subject: [PATCH 096/163] Bump github.com/aws/aws-sdk-go-v2 from 1.16.7 to 1.16.8 (#177) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.16.7 to 1.16.8. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.16.7...v1.16.8) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 40a20737..c38c3a07 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.7 + github.com/aws/aws-sdk-go-v2 v1.16.8 github.com/aws/aws-sdk-go-v2/config v1.15.14 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 github.com/go-logr/logr v1.2.3 diff --git a/go.sum b/go.sum index 302cd738..cc3ad3d0 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,9 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.7 h1:zfBwXus3u14OszRxGcqCDS4MfMCv10e8SMJ2r8Xm0Ns= github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= +github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ= +github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2/config v1.15.14 h1:+BqpqlydTq4c2et9Daury7gE+o67P4lbk7eybiCBNc4= github.com/aws/aws-sdk-go-v2/config v1.15.14/go.mod h1:CQBv+VVv8rR5z2xE+Chdh5m+rFfsqeY4k0veEZeq6QM= github.com/aws/aws-sdk-go-v2/credentials v1.12.9 h1:DloAJr0/jbvm0iVRFDFh8GlWxrOd9XKyX82U+dfVeZs= From 8460148319867d2784a97d0a2ada9e59eec56e61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Aug 2022 14:36:59 -0700 Subject: [PATCH 097/163] Bump github.com/onsi/gomega from 1.19.0 to 1.20.0 (#169) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.19.0 to 1.20.0. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.19.0...v1.20.0) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c38c3a07..8ba0ac5f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.19.0 + github.com/onsi/gomega v1.20.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.0 k8s.io/api v0.24.3 @@ -74,15 +74,15 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index cc3ad3d0..dd03ec1a 100644 --- a/go.sum +++ b/go.sum @@ -402,15 +402,17 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -646,8 +648,9 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -738,8 +741,10 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -819,6 +824,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -934,8 +940,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 84f43fcd8e8815ca12ac1e7416d24c7d420c8853 Mon Sep 17 00:00:00 2001 From: Fred Wang <58385135+fredjywang@users.noreply.github.com> Date: Wed, 10 Aug 2022 10:41:05 -0700 Subject: [PATCH 098/163] Importing Headless Services Support for A/AAAA records (#180) --- .../multicluster/cloudmap_controller.go | 10 +- pkg/controllers/multicluster/utils.go | 28 ++++- pkg/controllers/multicluster/utils_test.go | 113 +++++++++++++++++- 3 files changed, 137 insertions(+), 14 deletions(-) diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index 276cb651..8c62ec45 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -138,7 +138,7 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se } // create ServiceImport if it doesn't exist - if svcImport, err = r.createAndGetServiceImport(ctx, svc.Namespace, svc.Name, importedSvcPorts); err != nil { + if svcImport, err = r.createAndGetServiceImport(ctx, svc, importedSvcPorts); err != nil { return err } } @@ -174,14 +174,14 @@ func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace str return existingServiceImport, err } -func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, namespace string, name string, servicePorts []*model.Port) (*multiclusterv1alpha1.ServiceImport, error) { - toCreate := CreateServiceImportStruct(namespace, name, servicePorts) +func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, svc *model.Service, servicePorts []*model.Port) (*multiclusterv1alpha1.ServiceImport, error) { + toCreate := CreateServiceImportStruct(svc, servicePorts) if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err } - r.Log.Info("created ServiceImport", "namespace", namespace, "name", name) + r.Log.Info("created ServiceImport", "namespace", svc.Namespace, "name", svc.Name) - return r.getServiceImport(ctx, namespace, name) + return r.getServiceImport(ctx, svc.Namespace, svc.Name) } func (r *CloudMapReconciler) getDerivedService(ctx context.Context, namespace string, name string) (*v1.Service, error) { diff --git a/pkg/controllers/multicluster/utils.go b/pkg/controllers/multicluster/utils.go index a9affa00..ce10744d 100644 --- a/pkg/controllers/multicluster/utils.go +++ b/pkg/controllers/multicluster/utils.go @@ -142,7 +142,7 @@ func DerivedName(namespace string, name string) string { } // CreateServiceImportStruct creates struct representation of a ServiceImport -func CreateServiceImportStruct(namespace string, name string, servicePorts []*model.Port) *multiclusterv1alpha1.ServiceImport { +func CreateServiceImportStruct(svc *model.Service, servicePorts []*model.Port) *multiclusterv1alpha1.ServiceImport { serviceImportPorts := make([]multiclusterv1alpha1.ServicePort, 0) for _, port := range servicePorts { serviceImportPorts = append(serviceImportPorts, PortToServiceImportPort(*port)) @@ -150,13 +150,13 @@ func CreateServiceImportStruct(namespace string, name string, servicePorts []*mo return &multiclusterv1alpha1.ServiceImport{ ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(namespace, name)}, + Namespace: svc.Namespace, + Name: svc.Name, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(svc.Namespace, svc.Name)}, }, Spec: multiclusterv1alpha1.ServiceImportSpec{ IPs: []string{}, - Type: multiclusterv1alpha1.ClusterSetIP, + Type: ServiceTypetoServiceImportType(svc.Endpoints[0].ServiceType), // assume each endpoint has the same serviceType Ports: serviceImportPorts, }, } @@ -174,7 +174,7 @@ func CreateDerivedServiceStruct(svcImport *multiclusterv1alpha1.ServiceImport, i svcPorts = append(svcPorts, PortToServicePort(*svcPort)) } - return &v1.Service{ + svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: svcImport.Namespace, Name: svcImport.Annotations[DerivedServiceAnnotation], @@ -185,6 +185,13 @@ func CreateDerivedServiceStruct(svcImport *multiclusterv1alpha1.ServiceImport, i Ports: svcPorts, }, } + + // if svcImport is Headless type, specify ClusterIP field to "None" + if svcImport.Spec.Type == multiclusterv1alpha1.Headless { + svc.Spec.ClusterIP = "None" + } + + return svc } func CreateEndpointForSlice(svc *v1.Service, ip string) discovery.Endpoint { @@ -235,3 +242,12 @@ func ExtractServiceType(svc *v1.Service) model.ServiceType { return model.ClusterSetIPType } + +// ServiceTypetoServiceImportType converts model service type to multicluster ServiceImport type +func ServiceTypetoServiceImportType(serviceType model.ServiceType) multiclusterv1alpha1.ServiceImportType { + if serviceType == model.HeadlessType { + return multiclusterv1alpha1.Headless + } + + return multiclusterv1alpha1.ClusterSetIP +} diff --git a/pkg/controllers/multicluster/utils_test.go b/pkg/controllers/multicluster/utils_test.go index f97d8dff..0feaede9 100644 --- a/pkg/controllers/multicluster/utils_test.go +++ b/pkg/controllers/multicluster/utils_test.go @@ -450,6 +450,7 @@ func TestPortsEqualIgnoreOrder(t *testing.T) { func TestCreateServiceImportStruct(t *testing.T) { type args struct { servicePorts []*model.Port + endpoints []*model.Endpoint } tests := []struct { name string @@ -461,7 +462,12 @@ func TestCreateServiceImportStruct(t *testing.T) { args: args{ servicePorts: []*model.Port{ {Name: test.PortName1, Protocol: test.Protocol1, Port: test.Port1}, - {Name: test.PortName2, Protocol: test.Protocol1, Port: test.Port2}, + {Name: test.PortName2, Protocol: test.Protocol2, Port: test.Port2}, + }, + endpoints: []*model.Endpoint{ + { + ServiceType: model.ClusterSetIPType, + }, }, }, want: multiclusterv1alpha1.ServiceImport{ @@ -475,7 +481,7 @@ func TestCreateServiceImportStruct(t *testing.T) { Type: multiclusterv1alpha1.ClusterSetIP, Ports: []multiclusterv1alpha1.ServicePort{ {Name: test.PortName1, Protocol: v1.ProtocolTCP, Port: test.Port1}, - {Name: test.PortName2, Protocol: v1.ProtocolTCP, Port: test.Port2}, + {Name: test.PortName2, Protocol: v1.ProtocolUDP, Port: test.Port2}, }, }, }, @@ -483,7 +489,7 @@ func TestCreateServiceImportStruct(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := CreateServiceImportStruct(test.HttpNsName, test.SvcName, tt.args.servicePorts); !reflect.DeepEqual(*got, tt.want) { + if got := CreateServiceImportStruct(test.GetTestServiceWithEndpoint(tt.args.endpoints), tt.args.servicePorts); !reflect.DeepEqual(*got, tt.want) { t.Errorf("CreateServiceImportStruct() = %v, want %v", got, tt.want) } }) @@ -547,3 +553,104 @@ func TestExtractServiceType(t *testing.T) { }) } } + +func TestServiceTypetoServiceImportType(t *testing.T) { + tests := []struct { + name string + svcType model.ServiceType + want multiclusterv1alpha1.ServiceImportType + }{ + { + name: "cluster ip type", + svcType: model.ClusterSetIPType, + want: multiclusterv1alpha1.ClusterSetIP, + }, + { + name: "headless type", + svcType: model.HeadlessType, + want: multiclusterv1alpha1.Headless, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ServiceTypetoServiceImportType(tt.svcType); got != tt.want { + t.Errorf("ServiceTypetoServiceImportType() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCreateDerivedServiceStruct(t *testing.T) { + type args struct { + servicePorts []*model.Port + svcImport *multiclusterv1alpha1.ServiceImport + } + tests := []struct { + name string + args args + want *v1.ServiceSpec + }{ + { + name: "cluster ip case", + args: args{ + servicePorts: []*model.Port{ + {Name: test.PortName1, Protocol: test.Protocol1, Port: test.Port1, TargetPort: "8080"}, + {Name: test.PortName2, Protocol: test.Protocol2, Port: test.Port2, TargetPort: "8080"}, + }, + svcImport: &multiclusterv1alpha1.ServiceImport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: test.HttpNsName, + Name: test.SvcName, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName)}, + }, + Spec: multiclusterv1alpha1.ServiceImportSpec{ + IPs: []string{}, + Type: multiclusterv1alpha1.ClusterSetIP, + }, + }, + }, + want: &v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + {Name: test.PortName1, Protocol: test.Protocol1, Port: test.Port1, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}}, + {Name: test.PortName2, Protocol: test.Protocol2, Port: test.Port2, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}}, + }, + }, + }, + { + name: "headless case", + args: args{ + servicePorts: []*model.Port{ + {Name: test.PortName1, Protocol: test.Protocol1, Port: test.Port1, TargetPort: "8080"}, + {Name: test.PortName2, Protocol: test.Protocol2, Port: test.Port2, TargetPort: "8080"}, + }, + svcImport: &multiclusterv1alpha1.ServiceImport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: test.HttpNsName, + Name: test.SvcName, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName)}, + }, + Spec: multiclusterv1alpha1.ServiceImportSpec{ + IPs: []string{}, + Type: multiclusterv1alpha1.Headless, + }, + }, + }, + want: &v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + {Name: test.PortName1, Protocol: test.Protocol1, Port: test.Port1, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}}, + {Name: test.PortName2, Protocol: test.Protocol2, Port: test.Port2, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}}, + }, + ClusterIP: "None", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := &CreateDerivedServiceStruct(tt.args.svcImport, tt.args.servicePorts).Spec; !reflect.DeepEqual(got, tt.want) { + t.Errorf("CreateDerivedServiceStruct() = %v, want %v", got, tt.want) + } + }) + } +} From bdfd9d7d8a55e6a8e154906608cfa6e36221ade9 Mon Sep 17 00:00:00 2001 From: Matthew Goodman Date: Thu, 11 Aug 2022 09:37:58 -0700 Subject: [PATCH 099/163] New DerivedServices Architecture + Source-Cluster label (#179) * Added managed by label - issue 110 * derived services + unit tests + label * revisions and delete svc bug * annotation readded. kind & janitor test changes. * fix expected test result. cluster-id -> cluster * rm derived services --- .../multicluster.x-k8s.io_serviceimports.yaml | 1 - integration/janitor/api.go | 5 +- integration/janitor/api_test.go | 3 +- integration/janitor/janitor.go | 5 +- integration/janitor/janitor_test.go | 2 +- integration/janitor/runner/main.go | 9 +- .../kind-test/configs/e2e-clusterId.yaml | 8 - .../kind-test/configs/e2e-clusterSetId.yaml | 8 - .../configs/e2e-clusterproperty.yaml | 13 ++ integration/kind-test/scripts/common.sh | 2 + integration/kind-test/scripts/run-tests.sh | 8 +- .../shared/scenarios/export_service.go | 9 +- integration/shared/scenarios/runner/main.go | 18 +- .../shared/scripts/cleanup-cloudmap.sh | 2 +- .../v1alpha1/serviceimport_types.go | 1 - pkg/cloudmap/api_test.go | 4 +- pkg/cloudmap/client.go | 17 ++ pkg/cloudmap/client_test.go | 31 +-- .../multicluster/cloudmap_controller.go | 102 +++++++--- .../multicluster/cloudmap_controller_test.go | 107 ++++++++-- .../multicluster/controllers_common_test.go | 2 +- .../multicluster/endpointslice_plan.go | 5 +- .../multicluster/endpointslice_plan_test.go | 3 + pkg/controllers/multicluster/utils.go | 101 +++++++--- pkg/controllers/multicluster/utils_test.go | 182 +++++++++++++++++- test/test-constants.go | 39 +++- 26 files changed, 539 insertions(+), 148 deletions(-) delete mode 100644 integration/kind-test/configs/e2e-clusterId.yaml delete mode 100644 integration/kind-test/configs/e2e-clusterSetId.yaml create mode 100644 integration/kind-test/configs/e2e-clusterproperty.yaml diff --git a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml index 075f5e44..3086e8d5 100644 --- a/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_serviceimports.yaml @@ -41,7 +41,6 @@ spec: is ClusterSetIP. items: type: string - maxItems: 1 type: array ports: items: diff --git a/integration/janitor/api.go b/integration/janitor/api.go index 2e99ac63..eebe6a06 100644 --- a/integration/janitor/api.go +++ b/integration/janitor/api.go @@ -5,7 +5,6 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" ) @@ -21,9 +20,9 @@ type serviceDiscoveryJanitorApi struct { janitorFacade SdkJanitorFacade } -func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config) ServiceDiscoveryJanitorApi { +func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config, clusterUtils common.ClusterUtils) ServiceDiscoveryJanitorApi { return &serviceDiscoveryJanitorApi{ - ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)), + ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg, clusterUtils), janitorFacade: NewSdkJanitorFacadeFromConfig(cfg), } } diff --git a/integration/janitor/api_test.go b/integration/janitor/api_test.go index 55a3b5b8..3d7d1514 100644 --- a/integration/janitor/api_test.go +++ b/integration/janitor/api_test.go @@ -5,6 +5,7 @@ import ( "testing" janitorMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" @@ -13,7 +14,7 @@ import ( ) func TestNewServiceDiscoveryJanitorApiFromConfig(t *testing.T) { - assert.NotNil(t, NewServiceDiscoveryJanitorApiFromConfig(&aws.Config{})) + assert.NotNil(t, NewServiceDiscoveryJanitorApiFromConfig(&aws.Config{}, common.ClusterUtils{})) } func TestServiceDiscoveryJanitorApi_DeleteNamespace_HappyCase(t *testing.T) { diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index 24127bdf..b2379256 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -6,6 +6,7 @@ import ( "os" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" ) @@ -22,7 +23,7 @@ type cloudMapJanitor struct { } // NewDefaultJanitor returns a new janitor object. -func NewDefaultJanitor() CloudMapJanitor { +func NewDefaultJanitor(clusterId string, clusterSetId string) CloudMapJanitor { awsCfg, err := config.LoadDefaultConfig(context.TODO()) if err != nil { @@ -31,7 +32,7 @@ func NewDefaultJanitor() CloudMapJanitor { } return &cloudMapJanitor{ - sdApi: NewServiceDiscoveryJanitorApiFromConfig(&awsCfg), + sdApi: NewServiceDiscoveryJanitorApiFromConfig(&awsCfg, common.NewClusterUtilsForTest(clusterId, clusterSetId)), fail: func() { os.Exit(1) }, } } diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index 196125e4..bdd36dfe 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -21,7 +21,7 @@ type testJanitor struct { } func TestNewDefaultJanitor(t *testing.T) { - assert.NotNil(t, NewDefaultJanitor()) + assert.NotNil(t, NewDefaultJanitor(test.ClusterId1, test.ClusterSetId1)) } func TestCleanupHappyCase(t *testing.T) { diff --git a/integration/janitor/runner/main.go b/integration/janitor/runner/main.go index 9711bfed..691c0d03 100644 --- a/integration/janitor/runner/main.go +++ b/integration/janitor/runner/main.go @@ -9,12 +9,15 @@ import ( ) func main() { - if len(os.Args) != 2 { - fmt.Println("Expected single namespace name argument") + if len(os.Args) != 4 { + fmt.Println("Expected namespace name, clusterId, clusterSetId arguments") os.Exit(1) } - j := janitor.NewDefaultJanitor() nsName := os.Args[1] + clusterId := os.Args[2] + clusterSetId := os.Args[3] + + j := janitor.NewDefaultJanitor(clusterId, clusterSetId) j.Cleanup(context.TODO(), nsName) } diff --git a/integration/kind-test/configs/e2e-clusterId.yaml b/integration/kind-test/configs/e2e-clusterId.yaml deleted file mode 100644 index cd1ee175..00000000 --- a/integration/kind-test/configs/e2e-clusterId.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# An example object of `id.k8s.io ClusterProperty` - -apiVersion: about.k8s.io/v1alpha1 -kind: ClusterProperty -metadata: - name: id.k8s.io -spec: - value: test-mcs-clusterId diff --git a/integration/kind-test/configs/e2e-clusterSetId.yaml b/integration/kind-test/configs/e2e-clusterSetId.yaml deleted file mode 100644 index dc884ba8..00000000 --- a/integration/kind-test/configs/e2e-clusterSetId.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# An example object of `clusterset.k8s.io ClusterProperty`: - -apiVersion: about.k8s.io/v1alpha1 -kind: ClusterProperty -metadata: - name: clusterset.k8s.io -spec: - value: test-mcs-clusterSetId \ No newline at end of file diff --git a/integration/kind-test/configs/e2e-clusterproperty.yaml b/integration/kind-test/configs/e2e-clusterproperty.yaml new file mode 100644 index 00000000..bfc24e13 --- /dev/null +++ b/integration/kind-test/configs/e2e-clusterproperty.yaml @@ -0,0 +1,13 @@ +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: kind-e2e-clusterid-1 +--- +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: kind-e2e-clustersetid-1 \ No newline at end of file diff --git a/integration/kind-test/scripts/common.sh b/integration/kind-test/scripts/common.sh index 41b154d2..768dbbec 100755 --- a/integration/kind-test/scripts/common.sh +++ b/integration/kind-test/scripts/common.sh @@ -11,6 +11,8 @@ export ENDPT_PORT=80 export SERVICE_PORT=8080 export KIND_SHORT='cloud-map-e2e' export CLUSTER='kind-cloud-map-e2e' +export CLUSTERID1='kind-e2e-clusterid-1' +export CLUSTERSETID1='kind-e2e-clustersetid-1' export IMAGE='kindest/node:v1.20.15@sha256:a6ce604504db064c5e25921c6c0fffea64507109a1f2a512b1b562ac37d652f3' export EXPECTED_ENDPOINT_COUNT=5 export UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/kind-test/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh index 8a36067d..343984dd 100755 --- a/integration/kind-test/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -4,9 +4,7 @@ source ./integration/kind-test/scripts/common.sh -$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterId.yaml" -$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterSetId.yaml" - +$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-deployment.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-service.yaml" $KUBECTL_BIN apply -f "$CONFIGS/e2e-export.yaml" @@ -20,7 +18,7 @@ mkdir -p "$LOGS" CTL_PID=$! echo "controller PID:$CTL_PID" -go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$endpts" +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$endpts" exit_code=$? if [ "$exit_code" -eq 0 ] ; then @@ -42,7 +40,7 @@ if [ "$exit_code" -eq 0 ] ; then exit $? fi - go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" + go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" exit_code=$? if [ "$exit_code" -eq 0 ] ; then diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index 8205ef0e..8698ba41 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -11,7 +11,6 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" multiclustercontrollers "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers/multicluster" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -34,7 +33,7 @@ type exportServiceScenario struct { expectedSvc model.Service } -func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, portStr string, servicePortStr string, ips string) (ExportServiceScenario, error) { +func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, clusterId string, clusterSetId string, portStr string, servicePortStr string, ips string) (ExportServiceScenario, error) { endpts := make([]*model.Endpoint, 0) port, parseError := strconv.ParseUint(portStr, 10, 16) @@ -60,8 +59,8 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po Protocol: string(v1.ProtocolTCP), }, EndpointPort: endpointPort, - ClusterId: test.ClusterId, - ClusterSetId: test.ClusterSetId, + ClusterId: clusterId, + ClusterSetId: clusterSetId, ServiceType: model.ClusterSetIPType, // in scenario, we assume ClusterSetIP type Attributes: make(map[string]string), }) @@ -73,7 +72,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, po NsTTL: time.Second, SvcTTL: time.Second, EndptTTL: time.Second, - }, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)), + }, common.NewClusterUtilsForTest(clusterId, clusterSetId)), expectedSvc: model.Service{ Namespace: nsName, Name: svcName, diff --git a/integration/shared/scenarios/runner/main.go b/integration/shared/scenarios/runner/main.go index 15995c0f..fd4f23b9 100644 --- a/integration/shared/scenarios/runner/main.go +++ b/integration/shared/scenarios/runner/main.go @@ -11,24 +11,26 @@ import ( ) func main() { - if len(os.Args) != 6 { - fmt.Println("Expected namespace, service, endpoint port, service port and endpoint IP list arguments") + if len(os.Args) != 8 { + fmt.Println("Expected namespace, service, clusterId, clusterSetId, endpoint port, service port and endpoint IP list arguments") os.Exit(1) } nsName := os.Args[1] svcName := os.Args[2] - port := os.Args[3] - servicePort := os.Args[4] - ips := os.Args[5] + clusterId := os.Args[3] + clusterSetId := os.Args[4] + port := os.Args[5] + servicePort := os.Args[6] + ips := os.Args[7] - testServiceExport(nsName, svcName, port, servicePort, ips) + testServiceExport(nsName, svcName, clusterId, clusterSetId, port, servicePort, ips) } -func testServiceExport(nsName string, svcName string, port string, servicePort string, ips string) { +func testServiceExport(nsName string, svcName string, clusterId string, clusterSetId string, port string, servicePort string, ips string) { fmt.Printf("Testing service export integration for namespace %s and service %s\n", nsName, svcName) - export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, port, servicePort, ips) + export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, clusterId, clusterSetId, port, servicePort, ips) if err != nil { fmt.Printf("Failed to setup service export integration test scenario: %s", err.Error()) os.Exit(1) diff --git a/integration/shared/scripts/cleanup-cloudmap.sh b/integration/shared/scripts/cleanup-cloudmap.sh index 0fbd7384..5ecf0d34 100755 --- a/integration/shared/scripts/cleanup-cloudmap.sh +++ b/integration/shared/scripts/cleanup-cloudmap.sh @@ -4,4 +4,4 @@ set -eo pipefail -go run ./integration/janitor/runner/main.go "$NAMESPACE" +go run ./integration/janitor/runner/main.go "$NAMESPACE" "$CLUSTERID1" "$CLUSTERSETID1" diff --git a/pkg/apis/multicluster/v1alpha1/serviceimport_types.go b/pkg/apis/multicluster/v1alpha1/serviceimport_types.go index 40989fdf..91c87478 100644 --- a/pkg/apis/multicluster/v1alpha1/serviceimport_types.go +++ b/pkg/apis/multicluster/v1alpha1/serviceimport_types.go @@ -37,7 +37,6 @@ type ServiceImportSpec struct { // +listType=atomic Ports []ServicePort `json:"ports"` // ip will be used as the VIP for this service when type is ClusterSetIP. - // +kubebuilder:validation:MaxItems:=1 // +optional IPs []string `json:"ips,omitempty"` // type defines the type of this service. diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 86dfe296..3de3e140 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -23,7 +23,7 @@ import ( ) func TestNewServiceDiscoveryApi(t *testing.T) { - sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)) + sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId1, test.ClusterSetId1)) assert.NotNil(t, sdc) } @@ -107,7 +107,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { HealthStatus: types.HealthStatusFilterAll, MaxResults: aws.Int32(1000), QueryParameters: map[string]string{ - model.ClusterSetIdAttr: test.ClusterSetId, + model.ClusterSetIdAttr: test.ClusterSetId1, }, }). Return(&sd.DiscoverInstancesOutput{ diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 1fdfa1a8..d770e251 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -204,6 +204,23 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s for _, endpt := range endpts { endptId := endpt.Id + + // only delete endpoint if its clusterid & clustersetid is the same as this cluster + clusterId, clusterIdErr := sdc.clusterUtils.GetClusterId(ctx) + if clusterIdErr != nil { + return clusterIdErr + } + clusterSetId, clusterSetIdErr := sdc.clusterUtils.GetClusterSetId(ctx) + if clusterSetIdErr != nil { + return clusterSetIdErr + } + + if endpt.ClusterId != clusterId || endpt.ClusterSetId != clusterSetId { + sdc.log.Debug("skipping endpoint deletion as different clusterid", "serviceName", svcName, "endpointId", endptId, "clusterId", endpt.ClusterId) + continue + } + + // add operation to delete endpoint opCollector.Add(func() (opId string, err error) { return sdc.sdApi.DeregisterInstance(ctx, svcId, endptId) }) diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 5b1eb750..e666fd0d 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -24,7 +24,7 @@ type testSdClient struct { } func TestNewServiceDiscoveryClient(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId, test.ClusterSetId)) + sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId1, test.ClusterSetId1)) assert.NotNil(t, sdc) } @@ -283,8 +283,8 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) attrs1 := map[string]string{ - model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClusterSetId, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, model.EndpointIpv4Attr: test.EndptIp1, model.EndpointPortAttr: test.PortStr1, model.EndpointPortNameAttr: test.PortName1, @@ -296,8 +296,8 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { model.ServiceTypeAttr: test.SvcType, } attrs2 := map[string]string{ - model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClusterSetId, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, model.EndpointIpv4Attr: test.EndptIp2, model.EndpointPortAttr: test.PortStr2, model.EndpointPortNameAttr: test.PortName2, @@ -342,8 +342,10 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { tc.mockCache.EXPECT().EvictEndpoints(test.HttpNsName, test.SvcName) err := tc.client.DeleteEndpoints(context.TODO(), test.HttpNsName, test.SvcName, - []*model.Endpoint{{Id: test.EndptId1}, {Id: test.EndptId2}}) - + []*model.Endpoint{ + {Id: test.EndptId1, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSetId1}, + {Id: test.EndptId2, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSetId1}, + }) assert.Nil(t, err) } @@ -353,9 +355,10 @@ func getTestSdClient(t *testing.T) *testSdClient { mockApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) return &testSdClient{ client: &serviceDiscoveryClient{ - log: common.NewLoggerWithLogr(testr.New(t)), - sdApi: mockApi, - cache: mockCache, + log: common.NewLoggerWithLogr(testr.New(t)), + sdApi: mockApi, + cache: mockCache, + clusterUtils: common.NewClusterUtilsForTest(test.ClusterId1, test.ClusterSetId1), }, mockApi: *mockApi, mockCache: *mockCache, @@ -368,8 +371,8 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { { InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ - model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClusterSetId, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, model.EndpointIpv4Attr: test.EndptIp1, model.EndpointPortAttr: test.PortStr1, model.EndpointPortNameAttr: test.PortName1, @@ -384,8 +387,8 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { { InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ - model.ClusterIdAttr: test.ClusterId, - model.ClusterSetIdAttr: test.ClusterSetId, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, model.EndpointIpv4Attr: test.EndptIp2, model.EndpointPortAttr: test.PortStr2, model.EndpointPortNameAttr: test.PortName2, diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index 8c62ec45..6d8dd8dc 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -131,6 +131,16 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se importedSvcPorts := ExtractServicePorts(svc.Endpoints) + clusterIdToEndpointsMap := make(map[string][]*model.Endpoint) + for _, ep := range svc.Endpoints { + clusterIdToEndpointsMap[ep.ClusterId] = append(clusterIdToEndpointsMap[ep.ClusterId], ep) + } + + clusterIds := make([]string, 0, len(clusterIdToEndpointsMap)) + for clusterId := range clusterIdToEndpointsMap { + clusterIds = append(clusterIds, clusterId) + } + svcImport, err := r.getServiceImport(ctx, svc.Namespace, svc.Name) if err != nil { if !errors.IsNotFound(err) { @@ -138,34 +148,60 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se } // create ServiceImport if it doesn't exist - if svcImport, err = r.createAndGetServiceImport(ctx, svc, importedSvcPorts); err != nil { + if svcImport, err = r.createAndGetServiceImport(ctx, svc, importedSvcPorts, clusterIds); err != nil { return err } } - derivedService, err := r.getDerivedService(ctx, svc.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) - if err != nil { - if !errors.IsNotFound(err) { + // get or create derived Service for each cluster the service is a member of + derivedServices := make([]*v1.Service, 0, len(clusterIds)) + for _, clusterId := range clusterIds { + endpoints := clusterIdToEndpointsMap[clusterId] + clusterImportedSvcPorts := ExtractServicePorts(endpoints) + + derivedService, err := r.getDerivedService(ctx, svc.Namespace, svc.Name, clusterId) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + + // create derived Service if it doesn't exist + if derivedService, err = r.createAndGetDerivedService(ctx, svcImport, clusterId, clusterImportedSvcPorts); err != nil { + return err + } + } + + // update derived Service ports to match imported ports if necessary + if err = r.updateDerivedService(ctx, derivedService, clusterImportedSvcPorts); err != nil { return err } - // create derived Service if it doesn't exist - if derivedService, err = r.createAndGetDerivedService(ctx, svcImport, importedSvcPorts); err != nil { + // update EndpointSlices of this derived Service + if err = r.updateEndpointSlices(ctx, svcImport, endpoints, derivedService, clusterId); err != nil { return err } - } - // update service import to match derived service cluster IP and imported ports if necessary - if err = r.updateServiceImport(ctx, svcImport, derivedService, importedSvcPorts); err != nil { - return err + derivedServices = append(derivedServices, derivedService) } - // update derived service ports to match imported ports if necessary - if err = r.updateDerivedService(ctx, derivedService, importedSvcPorts); err != nil { - return err + // remove any existing derived services that do not have any endpoints in cloud map + existingDerivedServices := &v1.ServiceList{} + existingDerivedSvcErr := r.Client.List(ctx, existingDerivedServices, client.InNamespace(svcImport.Namespace), client.MatchingLabels{LabelDerivedServiceOriginatingName: svcImport.Name}) + if existingDerivedSvcErr != nil { + r.Log.Error(existingDerivedSvcErr, "failed to list derived services") + return existingDerivedSvcErr + } + for _, derivedService := range existingDerivedServices.Items { + clusterId := derivedService.Labels[LabelSourceCluster] + if _, ok := clusterIdToEndpointsMap[clusterId]; !ok { + if err := r.DeleteDerivedServiceAndEndpointSlices(ctx, &derivedService); err != nil { + return err + } + } } - return r.updateEndpointSlices(ctx, svcImport, svc.Endpoints, derivedService) + // update service import to match derived service clusterIPs and imported ports if necessary + return r.updateServiceImport(ctx, svcImport, derivedServices, importedSvcPorts) } func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace string, name string) (*multiclusterv1alpha1.ServiceImport, error) { @@ -174,8 +210,8 @@ func (r *CloudMapReconciler) getServiceImport(ctx context.Context, namespace str return existingServiceImport, err } -func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, svc *model.Service, servicePorts []*model.Port) (*multiclusterv1alpha1.ServiceImport, error) { - toCreate := CreateServiceImportStruct(svc, servicePorts) +func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, svc *model.Service, servicePorts []*model.Port, clusterIds []string) (*multiclusterv1alpha1.ServiceImport, error) { + toCreate := CreateServiceImportStruct(svc, clusterIds, servicePorts) if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err } @@ -184,23 +220,24 @@ func (r *CloudMapReconciler) createAndGetServiceImport(ctx context.Context, svc return r.getServiceImport(ctx, svc.Namespace, svc.Name) } -func (r *CloudMapReconciler) getDerivedService(ctx context.Context, namespace string, name string) (*v1.Service, error) { +func (r *CloudMapReconciler) getDerivedService(ctx context.Context, namespace string, name string, clusterId string) (*v1.Service, error) { + derivedName := DerivedName(namespace, name, clusterId) existingService := &v1.Service{} - err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existingService) + err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: derivedName}, existingService) return existingService, err } -func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, svcPorts []*model.Port) (*v1.Service, error) { - toCreate := CreateDerivedServiceStruct(svcImport, svcPorts) +func (r *CloudMapReconciler) createAndGetDerivedService(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, clusterId string, svcPorts []*model.Port) (*v1.Service, error) { + toCreate := CreateDerivedServiceStruct(svcImport, svcPorts, clusterId) if err := r.Client.Create(ctx, toCreate); err != nil { return nil, err } r.Log.Info("created derived Service", "namespace", toCreate.Namespace, "name", toCreate.Name) - return r.getDerivedService(ctx, toCreate.Namespace, svcImport.Annotations[DerivedServiceAnnotation]) + return r.getDerivedService(ctx, svcImport.Namespace, svcImport.Name, clusterId) } -func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, desiredEndpoints []*model.Endpoint, svc *v1.Service) error { +func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, desiredEndpoints []*model.Endpoint, svc *v1.Service, clusterId string) error { existingSlicesList := discovery.EndpointSliceList{} if err := r.Client.List(ctx, &existingSlicesList, client.InNamespace(svc.Namespace), client.MatchingLabels{discovery.LabelServiceName: svc.Name}); err != nil { @@ -217,6 +254,7 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport Desired: desiredEndpoints, Service: svc, ServiceImportName: svcImport.Name, + ClusterId: clusterId, } changes := plan.CalculateChanges() @@ -245,11 +283,13 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport return nil } -func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, svc *v1.Service, importedSvcPorts []*model.Port) error { +func (r *CloudMapReconciler) updateServiceImport(ctx context.Context, svcImport *multiclusterv1alpha1.ServiceImport, derivedServices []*v1.Service, importedSvcPorts []*model.Port) error { updateRequired := false - if len(svcImport.Spec.IPs) != 1 || svcImport.Spec.IPs[0] != svc.Spec.ClusterIP { - r.Log.Debug("ServiceImport IP need update", "ServiceImport IPs", svcImport.Spec.IPs, "cluster IP", svc.Spec.ClusterIP) - svcImport.Spec.IPs = []string{svc.Spec.ClusterIP} + + clusterIPs := GetClusterIpsFromServices(derivedServices) + if !IPsEqualIgnoreOrder(svcImport.Spec.IPs, clusterIPs) { + r.Log.Info("ServiceImport IPs need update", "ServiceImport IPs", svcImport.Spec.IPs, "cluster IPs", clusterIPs) + svcImport.Spec.IPs = clusterIPs updateRequired = true } @@ -315,3 +355,13 @@ func (r *CloudMapReconciler) updateDerivedService(ctx context.Context, svc *v1.S return nil } + +func (r *CloudMapReconciler) DeleteDerivedServiceAndEndpointSlices(ctx context.Context, derivedService *v1.Service) error { + // delete EndpointSlices + if err := r.Client.DeleteAllOf(ctx, &discovery.EndpointSlice{}, client.InNamespace(derivedService.Namespace), client.MatchingLabels{discovery.LabelServiceName: derivedService.Name}); err != nil { + return err + } + // delete Service + r.Log.Info("deleting derived Service", "namespace", derivedService.Namespace, "name", derivedService.Name) + return r.Client.Delete(ctx, derivedService) +} diff --git a/pkg/controllers/multicluster/cloudmap_controller_test.go b/pkg/controllers/multicluster/cloudmap_controller_test.go index 281e4493..16b06ac4 100644 --- a/pkg/controllers/multicluster/cloudmap_controller_test.go +++ b/pkg/controllers/multicluster/cloudmap_controller_test.go @@ -25,13 +25,7 @@ import ( func TestCloudMapReconciler_Reconcile(t *testing.T) { // create a fake controller client and add some objects - objs := []runtime.Object{k8sNamespaceForTest()} - - s := scheme.Scheme - s.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceImportList{}, &multiclusterv1alpha1.ServiceImport{}) - s.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) - - fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).WithObjects(test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() + fakeClient := fake.NewClientBuilder().WithScheme(getCloudMapReconcilerScheme()).WithObjects(k8sNamespaceForTest(), test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() // create a mock cloudmap service discovery client mockController := gomock.NewController(t) @@ -60,19 +54,87 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { err = fakeClient.List(context.TODO(), derivedServiceList, client.InNamespace(test.HttpNsName)) assert.NoError(t, err) derivedService := derivedServiceList.Items[0] - assert.True(t, strings.Contains(derivedService.Name, "imported-"), "Derived service created", "service", derivedService.Name) - assert.Equal(t, int32(test.ServicePort1), derivedService.Spec.Ports[0].Port) - assert.Equal(t, int32(test.Port1), derivedService.Spec.Ports[0].TargetPort.IntVal) + assertDerivedService(t, &derivedService, test.ServicePort1, test.Port1) // assert endpoint slices are created endpointSliceList := &v1beta1.EndpointSliceList{} err = fakeClient.List(context.TODO(), endpointSliceList, client.InNamespace(test.HttpNsName)) assert.NoError(t, err) endpointSlice := endpointSliceList.Items[0] - assert.Equal(t, test.SvcName, endpointSlice.Labels["multicluster.kubernetes.io/service-name"], "Endpoint slice is created") - assert.Contains(t, endpointSlice.Labels, LabelEndpointSliceManagedBy, "Managed by label is added") - assert.Equal(t, int32(test.Port1), *endpointSlice.Ports[0].Port) - assert.Equal(t, test.EndptIp1, endpointSlice.Endpoints[0].Addresses[0]) + assertEndpointSlice(t, &endpointSlice, test.Port1, test.EndptIp1, test.ClusterId1) +} + +func TestCloudMapReconciler_Reconcile_MulticlusterService(t *testing.T) { + // create a fake controller client and add some objects + fakeClient := fake.NewClientBuilder().WithScheme(getCloudMapReconcilerScheme()).WithObjects(k8sNamespaceForTest(), test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() + + // create a mock cloudmap service discovery client + mockController := gomock.NewController(t) + defer mockController.Finish() + + mockSDClient := cloudmapMock.NewMockServiceDiscoveryClient(mockController) + // The service model in the Cloudmap. + mockSDClient.EXPECT().ListServices(context.TODO(), test.HttpNsName). + // The multicluster service has endpoints in different clusters (different ClusterIds) + Return([]*model.Service{test.GetTestMulticlusterService()}, nil) + + reconciler := getReconciler(t, mockSDClient, fakeClient) + + err := reconciler.Reconcile(context.TODO()) + if err != nil { + t.Fatalf("reconcile failed: (%v)", err) + } + + // assert service import object + svcImport := &multiclusterv1alpha1.ServiceImport{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Namespace: test.HttpNsName, Name: test.SvcName}, svcImport) + assert.NoError(t, err) + assert.Equal(t, test.SvcName, svcImport.Name, "Service imported") + + assert.Contains(t, svcImport.Status.Clusters, multiclusterv1alpha1.ClusterStatus{Cluster: test.ClusterId1}) + assert.Contains(t, svcImport.Status.Clusters, multiclusterv1alpha1.ClusterStatus{Cluster: test.ClusterId2}) + assert.Equal(t, 2, len(svcImport.Status.Clusters)) + + // assert derived services are successfully created + derivedServiceList := &v1.ServiceList{} + err = fakeClient.List(context.TODO(), derivedServiceList, client.InNamespace(test.HttpNsName)) + assert.NoError(t, err) + assert.Equal(t, 2, len(derivedServiceList.Items)) + + derivedServiceMap := map[string]v1.Service{} + for _, derivedService := range derivedServiceList.Items { + derivedServiceMap[derivedService.ObjectMeta.Name] = derivedService + } + + derivedService1 := derivedServiceMap[DerivedName(svcImport.Namespace, svcImport.Name, test.ClusterId1)] + assertDerivedService(t, &derivedService1, test.ServicePort1, test.Port1) + derivedService2 := derivedServiceMap[DerivedName(svcImport.Namespace, svcImport.Name, test.ClusterId2)] + assertDerivedService(t, &derivedService2, test.ServicePort2, test.Port2) + + // assert endpoint slices are created for each derived service + endpointSliceList := &v1beta1.EndpointSliceList{} + err = fakeClient.List(context.TODO(), endpointSliceList, client.InNamespace(test.HttpNsName)) + assert.NoError(t, err) + assert.Equal(t, 2, len(endpointSliceList.Items)) + + endpointSliceMap := make(map[string]v1beta1.EndpointSlice) + for _, endpointSlice := range endpointSliceList.Items { + endpointSliceName := endpointSlice.ObjectMeta.Name + derivedServiceName := endpointSliceName[:strings.LastIndex(endpointSliceName, "-")] + endpointSliceMap[derivedServiceName] = endpointSlice + } + + endpointSlice1 := endpointSliceMap[derivedService1.Name] + assertEndpointSlice(t, &endpointSlice1, test.Port1, test.EndptIp1, test.ClusterId1) + endpointSlice2 := endpointSliceMap[derivedService2.Name] + assertEndpointSlice(t, &endpointSlice2, test.Port2, test.EndptIp2, test.ClusterId2) +} + +func getCloudMapReconcilerScheme() *runtime.Scheme { + scheme := scheme.Scheme + scheme.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceImportList{}, &multiclusterv1alpha1.ServiceImport{}) + scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) + return scheme } func getReconciler(t *testing.T, mockSDClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *CloudMapReconciler { @@ -83,3 +145,20 @@ func getReconciler(t *testing.T, mockSDClient *cloudmapMock.MockServiceDiscovery ClusterUtils: common.NewClusterUtils(client), } } + +func assertDerivedService(t *testing.T, derivedService *v1.Service, servicePort int, port int) { + assert.NotNil(t, derivedService) + assert.True(t, strings.Contains(derivedService.Name, "imported-"), "Derived service created", "service", derivedService.Name) + assert.Equal(t, int32(servicePort), derivedService.Spec.Ports[0].Port) + assert.Equal(t, int32(port), derivedService.Spec.Ports[0].TargetPort.IntVal) +} + +func assertEndpointSlice(t *testing.T, endpointSlice *v1beta1.EndpointSlice, port int, endptIp string, clusterId string) { + assert.NotNil(t, endpointSlice) + assert.Equal(t, test.SvcName, endpointSlice.Labels["multicluster.kubernetes.io/service-name"], "Endpoint slice is created") + assert.Equal(t, clusterId, endpointSlice.Labels["multicluster.kubernetes.io/source-cluster"], "Endpoint slice is created") + assert.Contains(t, endpointSlice.Labels, LabelEndpointSliceManagedBy, "Managed by label is added") + assert.Equal(t, int32(port), *endpointSlice.Ports[0].Port) + assert.Equal(t, 1, len(endpointSlice.Endpoints)) + assert.Equal(t, endptIp, endpointSlice.Endpoints[0].Addresses[0]) +} diff --git a/pkg/controllers/multicluster/controllers_common_test.go b/pkg/controllers/multicluster/controllers_common_test.go index 2917376a..e5a63246 100644 --- a/pkg/controllers/multicluster/controllers_common_test.go +++ b/pkg/controllers/multicluster/controllers_common_test.go @@ -72,7 +72,7 @@ func endpointSliceForTest() *discovery.EndpointSlice { func endpointSliceWithIpsAndPortsForTest(ips []string, ports []discovery.EndpointPort) *discovery.EndpointSlice { svc := k8sServiceForTest() - slice := CreateEndpointSliceStruct(svc, test.SvcName) + slice := CreateEndpointSliceStruct(svc, test.SvcName, test.ClusterId1) slice.Ports = ports testEndpoints := make([]discovery.Endpoint, 0) diff --git a/pkg/controllers/multicluster/endpointslice_plan.go b/pkg/controllers/multicluster/endpointslice_plan.go index ff47bdad..fc0f1af4 100644 --- a/pkg/controllers/multicluster/endpointslice_plan.go +++ b/pkg/controllers/multicluster/endpointslice_plan.go @@ -34,6 +34,9 @@ type EndpointSlicePlan struct { // Desired Endpoints Desired []*model.Endpoint + + // Cluster the EndpointSlice belongs to + ClusterId string } // CalculateChanges returns list of EndpointSlice Changes that need to applied @@ -144,7 +147,7 @@ func (p *EndpointSlicePlan) getOrCreateUnfilledEndpointSlice(changes *EndpointSl } // No existing slices can fill new endpoint requirements so create a new slice - sliceToCreate := CreateEndpointSliceStruct(p.Service, p.ServiceImportName) + sliceToCreate := CreateEndpointSliceStruct(p.Service, p.ServiceImportName, p.ClusterId) changes.Create = append(changes.Create, sliceToCreate) return sliceToCreate, true } diff --git a/pkg/controllers/multicluster/endpointslice_plan_test.go b/pkg/controllers/multicluster/endpointslice_plan_test.go index 354a1150..b4805bdf 100644 --- a/pkg/controllers/multicluster/endpointslice_plan_test.go +++ b/pkg/controllers/multicluster/endpointslice_plan_test.go @@ -177,6 +177,7 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { p := &EndpointSlicePlan{ Service: k8sServiceForTest(), ServiceImportName: test.SvcName, + ClusterId: test.ClusterId1, Current: tt.fields.Current, Desired: tt.fields.Desired, } @@ -194,6 +195,7 @@ func TestEndpointSlicePlan_MultipleSliceCreation(t *testing.T) { maxEndpointsPerSlice: 2, Service: k8sServiceForTest(), ServiceImportName: test.SvcName, + ClusterId: test.ClusterId1, Current: []*discovery.EndpointSlice{}, Desired: test.GetTestEndpoints(43), } @@ -206,6 +208,7 @@ func TestEndpointSlicePlan_PreferCreateOverMultipleSliceUpdate(t *testing.T) { maxEndpointsPerSlice: 2, Service: k8sServiceForTest(), ServiceImportName: test.SvcName, + ClusterId: test.ClusterId1, Current: []*discovery.EndpointSlice{endpointSliceForTest()}, Desired: []*model.Endpoint{test.GetTestEndpoint1()}, } diff --git a/pkg/controllers/multicluster/utils.go b/pkg/controllers/multicluster/utils.go index ce10744d..a434d2d2 100644 --- a/pkg/controllers/multicluster/utils.go +++ b/pkg/controllers/multicluster/utils.go @@ -3,10 +3,13 @@ package controllers import ( "crypto/sha256" "encoding/base32" + "encoding/json" "strings" multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,9 +24,15 @@ const ( // LabelServiceImportName indicates the name of the multi-cluster service that an EndpointSlice belongs to. LabelServiceImportName = "multicluster.kubernetes.io/service-name" + // LabelDerivedServiceOriginatingName indicates the name of the multi-cluster service that the derived service originated from. + LabelDerivedServiceOriginatingName = "multicluster.kubernetes.io/service-name" + // LabelEndpointSliceManagedBy indicates the name of the entity that manages the EndpointSlice. LabelEndpointSliceManagedBy = "endpointslice.kubernetes.io/managed-by" + // LabelSourceCluster indicates the id of the cluster the object was created for + LabelSourceCluster = "multicluster.kubernetes.io/source-cluster" + // ValueEndpointSliceManagedBy indicates the name of the entity that manages the EndpointSlice. ValueEndpointSliceManagedBy = "aws-cloud-map-mcs-controller-for-k8s" ) @@ -112,58 +121,76 @@ func ExtractEndpointPorts(endpoints []*model.Endpoint) (endpointPorts []*model.P } func PortsEqualIgnoreOrder(a, b []*model.Port) (equal bool) { - if len(a) != len(b) { - return false + idsA := make([]string, len(a)) + idsB := make([]string, len(b)) + for i, port := range a { + idsA[i] = port.GetID() } - - aMap := make(map[string]*model.Port) - for _, portA := range a { - aMap[portA.GetID()] = portA + for i, port := range b { + idsB[i] = port.GetID() } + less := func(x, y string) bool { return x < y } + equalIgnoreOrder := cmp.Diff(idsA, idsB, cmpopts.SortSlices(less)) == "" + return equalIgnoreOrder +} - for _, portB := range b { - portA, found := aMap[portB.GetID()] - if !found { - return false - } +func IPsEqualIgnoreOrder(a, b []string) (equal bool) { + less := func(x, y string) bool { return x < y } + equalIgnoreOrder := cmp.Diff(a, b, cmpopts.SortSlices(less)) == "" + return equalIgnoreOrder +} - if !portB.Equals(portA) { - return false - } +// GetClusterIpsFromServices returns list of ClusterIPs from services +func GetClusterIpsFromServices(services []*v1.Service) []string { + clusterIPs := make([]string, 0) + for _, svc := range services { + clusterIPs = append(clusterIPs, svc.Spec.ClusterIP) } - return true + return clusterIPs } // DerivedName computes the "placeholder" name for an imported service -func DerivedName(namespace string, name string) string { +func DerivedName(namespace string, name string, clusterId string) string { hash := sha256.New() - hash.Write([]byte(namespace + name)) + hash.Write([]byte(namespace + name + clusterId)) return "imported-" + strings.ToLower(base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(hash.Sum(nil)))[:10] } // CreateServiceImportStruct creates struct representation of a ServiceImport -func CreateServiceImportStruct(svc *model.Service, servicePorts []*model.Port) *multiclusterv1alpha1.ServiceImport { +func CreateServiceImportStruct(svc *model.Service, clusterIds []string, servicePorts []*model.Port) *multiclusterv1alpha1.ServiceImport { serviceImportPorts := make([]multiclusterv1alpha1.ServicePort, 0) for _, port := range servicePorts { serviceImportPorts = append(serviceImportPorts, PortToServiceImportPort(*port)) } + clusters := make([]multiclusterv1alpha1.ClusterStatus, 0) + for _, clusterId := range clusterIds { + clusters = append(clusters, multiclusterv1alpha1.ClusterStatus{ + Cluster: clusterId, + }) + } + return &multiclusterv1alpha1.ServiceImport{ ObjectMeta: metav1.ObjectMeta{ - Namespace: svc.Namespace, - Name: svc.Name, - Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(svc.Namespace, svc.Name)}, + Namespace: svc.Namespace, + Name: svc.Name, + Annotations: map[string]string{ + DerivedServiceAnnotation: CreateDerivedServiceAnnotation(svc.Namespace, svc.Name, clusterIds), + }, }, Spec: multiclusterv1alpha1.ServiceImportSpec{ IPs: []string{}, - Type: ServiceTypetoServiceImportType(svc.Endpoints[0].ServiceType), // assume each endpoint has the same serviceType + Type: ServiceTypetoServiceImportType(svc.Endpoints[0].ServiceType), Ports: serviceImportPorts, }, + Status: multiclusterv1alpha1.ServiceImportStatus{ + Clusters: clusters, + }, } } // CreateDerivedServiceStruct creates struct representation of a derived service -func CreateDerivedServiceStruct(svcImport *multiclusterv1alpha1.ServiceImport, importedSvcPorts []*model.Port) *v1.Service { +func CreateDerivedServiceStruct(svcImport *multiclusterv1alpha1.ServiceImport, importedSvcPorts []*model.Port, clusterId string) *v1.Service { ownerRef := metav1.NewControllerRef(svcImport, schema.GroupVersionKind{ Version: svcImport.TypeMeta.APIVersion, Kind: svcImport.TypeMeta.Kind, @@ -176,8 +203,12 @@ func CreateDerivedServiceStruct(svcImport *multiclusterv1alpha1.ServiceImport, i svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + LabelSourceCluster: clusterId, + LabelDerivedServiceOriginatingName: svcImport.Name, + }, Namespace: svcImport.Namespace, - Name: svcImport.Annotations[DerivedServiceAnnotation], + Name: DerivedName(svcImport.Namespace, svcImport.Name, clusterId), OwnerReferences: []metav1.OwnerReference{*ownerRef}, }, Spec: v1.ServiceSpec{ @@ -212,7 +243,7 @@ func CreateEndpointForSlice(svc *v1.Service, ip string) discovery.Endpoint { } } -func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string) *discovery.EndpointSlice { +func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string, clusterId string) *discovery.EndpointSlice { return &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -222,6 +253,8 @@ func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string) *discovery LabelServiceImportName: svcImportName, // 'managed-by' label set to controller LabelEndpointSliceManagedBy: ValueEndpointSliceManagedBy, + // 'source-cluster' label set to current cluster + LabelSourceCluster: clusterId, }, GenerateName: svc.Name + "-", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(svc, schema.GroupVersionKind{ @@ -239,10 +272,26 @@ func ExtractServiceType(svc *v1.Service) model.ServiceType { if svc.Spec.ClusterIP == "None" { return model.HeadlessType } - return model.ClusterSetIPType } +// CreateDerivedServiceAnnotation creates a JSON object containing a slice of maps of clusterIds and derived service names +func CreateDerivedServiceAnnotation(namespace string, name string, clusterIds []string) string { + clusters := make([]map[string]string, 0, len(clusterIds)) + for _, clusterId := range clusterIds { + clusters = append(clusters, map[string]string{ + "cluster": clusterId, + "derived-service": DerivedName(namespace, name, clusterId), + }) + } + // create JSON + jsonBytes, err := json.Marshal(clusters) + if err != nil { + return "" + } + return string(jsonBytes) +} + // ServiceTypetoServiceImportType converts model service type to multicluster ServiceImport type func ServiceTypetoServiceImportType(serviceType model.ServiceType) multiclusterv1alpha1.ServiceImportType { if serviceType == model.HeadlessType { diff --git a/pkg/controllers/multicluster/utils_test.go b/pkg/controllers/multicluster/utils_test.go index 0feaede9..7c86383f 100644 --- a/pkg/controllers/multicluster/utils_test.go +++ b/pkg/controllers/multicluster/utils_test.go @@ -2,11 +2,13 @@ package controllers import ( "reflect" + "strconv" "testing" multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" "k8s.io/api/discovery/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -447,10 +449,131 @@ func TestPortsEqualIgnoreOrder(t *testing.T) { } } +func TestIPsEqualIgnoreOrder(t *testing.T) { + type args struct { + ipsA []string + ipsB []string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "ips equal same order", + args: args{ + ipsA: []string{ + test.ClusterIp1, + test.ClusterIp2, + }, + ipsB: []string{ + test.ClusterIp1, + test.ClusterIp2, + }, + }, + want: true, + }, + { + name: "ips equal different order", + args: args{ + ipsA: []string{ + test.ClusterIp1, + test.ClusterIp2, + }, + ipsB: []string{ + test.ClusterIp2, + test.ClusterIp1, + }, + }, + want: true, + }, + { + name: "ips not equal", + args: args{ + ipsA: []string{ + test.ClusterIp1, + test.ClusterIp2, + }, + ipsB: []string{ + test.ClusterIp1, + "10.10.10.3", + }, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IPsEqualIgnoreOrder(tt.args.ipsA, tt.args.ipsB); !(got == tt.want) { + t.Errorf("IPsEqualIgnoreOrder() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetClusterIpsFromServices(t *testing.T) { + type args struct { + services []*v1.Service + } + + tests := []struct { + name string + args args + want []string + }{ + { + name: "happy case", + args: args{ + services: []*v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: test.ClusterIp1, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: test.ClusterIp2, + }, + }, + }}, + want: []string{ + test.ClusterIp1, test.ClusterIp2, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetClusterIpsFromServices(tt.args.services); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetClusterIpsFromServices() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDerivedService(t *testing.T) { + const numTests = 100 + derivedServiceMap := make(map[string]bool) + for i := 0; i < numTests; i++ { + namespace := test.HttpNsName + name := "test-svcname-" + strconv.Itoa(i) + clusterId := "test-clusterid-" + strconv.Itoa(i) + derivedService := DerivedName(namespace, name, clusterId) + assert.NotContains(t, derivedServiceMap, derivedService, "derived service already exists") + derivedServiceMap[derivedService] = true + } + assert.Equal(t, numTests, len(derivedServiceMap)) + assert.True(t, DerivedName(test.HttpNsName, test.SvcName, test.ClusterId1) != DerivedName(test.HttpNsName, test.SvcName, test.ClusterId2)) +} + func TestCreateServiceImportStruct(t *testing.T) { type args struct { servicePorts []*model.Port endpoints []*model.Endpoint + clusterIds []string } tests := []struct { name string @@ -460,6 +583,7 @@ func TestCreateServiceImportStruct(t *testing.T) { { name: "happy case", args: args{ + clusterIds: []string{test.ClusterId1, test.ClusterId2}, servicePorts: []*model.Port{ {Name: test.PortName1, Protocol: test.Protocol1, Port: test.Port1}, {Name: test.PortName2, Protocol: test.Protocol2, Port: test.Port2}, @@ -472,9 +596,11 @@ func TestCreateServiceImportStruct(t *testing.T) { }, want: multiclusterv1alpha1.ServiceImport{ ObjectMeta: metav1.ObjectMeta{ - Namespace: test.HttpNsName, - Name: test.SvcName, - Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName)}, + Namespace: test.HttpNsName, + Name: test.SvcName, + Annotations: map[string]string{ + DerivedServiceAnnotation: CreateDerivedServiceAnnotation(test.HttpNsName, test.SvcName, []string{test.ClusterId1, test.ClusterId2}), + }, }, Spec: multiclusterv1alpha1.ServiceImportSpec{ IPs: []string{}, @@ -484,13 +610,23 @@ func TestCreateServiceImportStruct(t *testing.T) { {Name: test.PortName2, Protocol: v1.ProtocolUDP, Port: test.Port2}, }, }, + Status: multiclusterv1alpha1.ServiceImportStatus{ + Clusters: []multiclusterv1alpha1.ClusterStatus{ + { + Cluster: test.ClusterId1, + }, + { + Cluster: test.ClusterId2, + }, + }, + }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := CreateServiceImportStruct(test.GetTestServiceWithEndpoint(tt.args.endpoints), tt.args.servicePorts); !reflect.DeepEqual(*got, tt.want) { - t.Errorf("CreateServiceImportStruct() = %v, want %v", got, tt.want) + if got := CreateServiceImportStruct(test.GetTestServiceWithEndpoint(tt.args.endpoints), tt.args.clusterIds, tt.args.servicePorts); !reflect.DeepEqual(*got, tt.want) { + t.Errorf("CreateServiceImportStruct() = %v, want %v", *got, tt.want) } }) } @@ -601,7 +737,7 @@ func TestCreateDerivedServiceStruct(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: test.HttpNsName, Name: test.SvcName, - Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName)}, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName, test.ClusterId1)}, }, Spec: multiclusterv1alpha1.ServiceImportSpec{ IPs: []string{}, @@ -628,7 +764,7 @@ func TestCreateDerivedServiceStruct(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: test.HttpNsName, Name: test.SvcName, - Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName)}, + Annotations: map[string]string{DerivedServiceAnnotation: DerivedName(test.HttpNsName, test.SvcName, test.ClusterId1)}, }, Spec: multiclusterv1alpha1.ServiceImportSpec{ IPs: []string{}, @@ -648,9 +784,39 @@ func TestCreateDerivedServiceStruct(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := &CreateDerivedServiceStruct(tt.args.svcImport, tt.args.servicePorts).Spec; !reflect.DeepEqual(got, tt.want) { + if got := &CreateDerivedServiceStruct(tt.args.svcImport, tt.args.servicePorts, test.ClusterId1).Spec; !reflect.DeepEqual(got, tt.want) { t.Errorf("CreateDerivedServiceStruct() = %v, want %v", got, tt.want) } }) } } + +func TestCreateDerivedServiceAnnotation(t *testing.T) { + type args struct { + namespace string + name string + clusterIds []string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "create derived service annotation", + args: args{ + namespace: test.HttpNsName, + name: test.SvcName, + clusterIds: []string{test.ClusterId1, test.ClusterId2}, + }, + want: "[{\"cluster\":\"test-mcs-clusterid-1\",\"derived-service\":\"imported-vm6pdvp7di\"},{\"cluster\":\"test-mcs-clusterid-2\",\"derived-service\":\"imported-i8hm9c3um2\"}]", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CreateDerivedServiceAnnotation(tt.args.namespace, tt.args.name, tt.args.clusterIds); got != tt.want { + t.Errorf("CreateDerivedServiceAnnotation() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/test/test-constants.go b/test/test-constants.go index f88d59bb..89c98551 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -18,8 +18,10 @@ const ( DnsNsId = "dns-ns-id" SvcName = "svc-name" SvcId = "svc-id" - ClusterId = "test-mcs-clusterId" - ClusterSetId = "test-mcs-clusterSetId" + ClusterId1 = "test-mcs-clusterid-1" + ClusterSetId1 = "test-mcs-clustersetid-1" + ClusterId2 = "test-mcs-clusterid-2" + ClusterSetId2 = "test-mcs-clustersetid-2" EndptId1 = "tcp-192_168_0_1-1" EndptId2 = "tcp-192_168_0_2-2" EndptIp1 = "192.168.0.1" @@ -36,6 +38,8 @@ const ( Protocol2 = "UDP" ServicePort2 = 22 ServicePortStr2 = "22" + ClusterIp1 = "10.10.10.1" + ClusterIp2 = "10.10.10.2" OpId1 = "operation-id-1" OpId2 = "operation-id-2" OpStart = 1 @@ -74,6 +78,15 @@ func GetTestServiceWithEndpoint(endpoints []*model.Endpoint) *model.Service { } } +func GetTestMulticlusterService() *model.Service { + // Service has two endpoints belonging to two different clusters in the same clusterset + return &model.Service{ + Namespace: HttpNsName, + Name: SvcName, + Endpoints: GetMulticlusterTestEndpoints(), + } +} + func GetTestEndpoint1() *model.Endpoint { return &model.Endpoint{ Id: EndptId1, @@ -89,8 +102,8 @@ func GetTestEndpoint1() *model.Endpoint { TargetPort: PortStr1, Protocol: Protocol1, }, - ClusterId: ClusterId, - ClusterSetId: ClusterSetId, + ClusterId: ClusterId1, + ClusterSetId: ClusterSetId1, ServiceType: model.ClusterSetIPType, Attributes: make(map[string]string), } @@ -111,18 +124,26 @@ func GetTestEndpoint2() *model.Endpoint { TargetPort: PortStr2, Protocol: Protocol2, }, - ClusterId: ClusterId, - ClusterSetId: ClusterSetId, + ClusterId: ClusterId1, + ClusterSetId: ClusterSetId1, ServiceType: model.ClusterSetIPType, Attributes: make(map[string]string), } } +func GetMulticlusterTestEndpoints() []*model.Endpoint { + endpoint1 := GetTestEndpoint1() + endpoint2 := GetTestEndpoint2() + // Set Different ClusterIds + endpoint2.ClusterId = ClusterId2 + return []*model.Endpoint{endpoint1, endpoint2} +} + func GetTestEndpoints(count int) (endpts []*model.Endpoint) { // use +3 offset go avoid collision with test endpoint 1 and 2 for i := 3; i < count+3; i++ { e := GetTestEndpoint1() - e.ClusterId = ClusterId + e.ClusterId = ClusterId1 e.Id = fmt.Sprintf("tcp-192_168_0_%d-1", i) e.IP = fmt.Sprintf("192.168.0.%d", i) endpts = append(endpts, e) @@ -136,7 +157,7 @@ func ClusterIdForTest() *aboutv1alpha1.ClusterProperty { Name: common.ClusterIdName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ - Value: ClusterId, + Value: ClusterId1, }, } } @@ -147,7 +168,7 @@ func ClusterSetIdForTest() *aboutv1alpha1.ClusterProperty { Name: common.ClusterSetIdName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ - Value: ClusterSetId, + Value: ClusterSetId1, }, } } From b5cdcb13cd0215f0511c0f6947d4d178426a2ec5 Mon Sep 17 00:00:00 2001 From: Matthew Goodman Date: Thu, 11 Aug 2022 11:34:49 -0700 Subject: [PATCH 100/163] EKS Integration Test Fixes (#187) * Added managed by label - issue 110 * derived services + unit tests + label * revisions and delete svc bug * annotation readded. kind & janitor test changes. * fix expected test result. cluster-id -> cluster * working eks integration tests --- config/rbac/role.yaml | 12 +++++++++++ .../eks-test/configs/coredns-clusterrole.yaml | 11 ++++++++++ .../configs/e2e-clusterproperty-1.yaml | 13 ++++++++++++ .../configs/e2e-clusterproperty-2.yaml | 13 ++++++++++++ integration/eks-test/scripts/eks-DNS-test.sh | 20 +++++++++++++++---- integration/eks-test/scripts/eks-cleanup.sh | 5 +++-- integration/eks-test/scripts/eks-common.sh | 3 +++ integration/eks-test/scripts/eks-run-tests.sh | 4 ++-- integration/eks-test/scripts/eks-setup.sh | 10 +++++++++- .../multicluster/cloudmap_controller.go | 1 + .../multicluster/serviceexport_controller.go | 1 + 11 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 integration/eks-test/configs/e2e-clusterproperty-1.yaml create mode 100644 integration/eks-test/configs/e2e-clusterproperty-2.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 307f5ea8..8c546e77 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -23,6 +23,18 @@ rules: - list - update - watch +- apiGroups: + - about.k8s.io + resources: + - clusterproperties + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - discovery.k8s.io resources: diff --git a/integration/eks-test/configs/coredns-clusterrole.yaml b/integration/eks-test/configs/coredns-clusterrole.yaml index 242212fa..ac2e542a 100644 --- a/integration/eks-test/configs/coredns-clusterrole.yaml +++ b/integration/eks-test/configs/coredns-clusterrole.yaml @@ -23,6 +23,17 @@ rules: - nodes verbs: - get +- apiGroups: + - about.k8s.io + resources: + - clusterproperties + verbs: + - create + - get + - list + - patch + - update + - watch - apiGroups: - discovery.k8s.io resources: diff --git a/integration/eks-test/configs/e2e-clusterproperty-1.yaml b/integration/eks-test/configs/e2e-clusterproperty-1.yaml new file mode 100644 index 00000000..8371963f --- /dev/null +++ b/integration/eks-test/configs/e2e-clusterproperty-1.yaml @@ -0,0 +1,13 @@ +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: eks-e2e-clusterid-1 +--- +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: eks-e2e-clustersetid-1 \ No newline at end of file diff --git a/integration/eks-test/configs/e2e-clusterproperty-2.yaml b/integration/eks-test/configs/e2e-clusterproperty-2.yaml new file mode 100644 index 00000000..239aa8fa --- /dev/null +++ b/integration/eks-test/configs/e2e-clusterproperty-2.yaml @@ -0,0 +1,13 @@ +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: id.k8s.io +spec: + value: eks-e2e-clusterid-2 +--- +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: eks-e2e-clustersetid-1 \ No newline at end of file diff --git a/integration/eks-test/scripts/eks-DNS-test.sh b/integration/eks-test/scripts/eks-DNS-test.sh index 8bdf7578..3a838dc3 100755 --- a/integration/eks-test/scripts/eks-DNS-test.sh +++ b/integration/eks-test/scripts/eks-DNS-test.sh @@ -4,22 +4,34 @@ echo "verifying cross-cluster service consumption..." +# Install curl if not installed $KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- curl --version &>/dev/null exit_code=$? - -# Install curl if not installed if [ "$exit_code" -eq 126 ]; then + echo "curl not installed, installing..." $KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- apk add curl fi +# Perform an nslookup to cluster-local CoreDNS +echo "performing nslookup..." +$KUBECTL_BIN exec -it $CLIENT_POD -n $NAMESPACE /bin/sh -- nslookup $SERVICE.$NAMESPACE.svc.clusterset.local +exit_code=$? + +if [ "$exit_code" -ne 0 ]; then + echo "ERROR: Unable to nslookup service $SERVICE.$NAMESPACE.svc.clusterset.local" + exit $exit_code +fi +sleep 5 + # Call to DNS server, if unable to reach, importing cluster is not able to properly consume service -$KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- curl -s $SERVICE.$NAMESPACE.svc.clusterset.local +echo "performing curl..." +$KUBECTL_BIN exec -it $CLIENT_POD -n $NAMESPACE /bin/sh -- curl $SERVICE.$NAMESPACE.svc.clusterset.local exit_code=$? if [ "$exit_code" -ne 0 ]; then + echo "ERROR: Unable to reach service $SERVICE.$NAMESPACE.svc.clusterset.local" exit $exit_code fi echo "confirmed service consumption" exit 0 - diff --git a/integration/eks-test/scripts/eks-cleanup.sh b/integration/eks-test/scripts/eks-cleanup.sh index 203f8cb4..7dc99aa9 100755 --- a/integration/eks-test/scripts/eks-cleanup.sh +++ b/integration/eks-test/scripts/eks-cleanup.sh @@ -39,5 +39,6 @@ $KUBECTL_BIN delete -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/conf echo "EKS clusters cleaned!" -./integration/shared/scripts/cleanup-cloudmap.sh - +# ./integration/shared/scripts/cleanup-cloudmap.sh +go run ./integration/janitor/runner/main.go "$NAMESPACE" "$CLUSTERID1" "$CLUSTERSETID1" +go run ./integration/janitor/runner/main.go "$NAMESPACE" "$CLUSTERID2" "$CLUSTERSETID1" diff --git a/integration/eks-test/scripts/eks-common.sh b/integration/eks-test/scripts/eks-common.sh index c3ce0264..3a1848d7 100755 --- a/integration/eks-test/scripts/eks-common.sh +++ b/integration/eks-test/scripts/eks-common.sh @@ -13,5 +13,8 @@ export ENDPT_PORT=80 export SERVICE_PORT=80 # from nginx-service.yaml export EXPORT_CLS='cls1' export IMPORT_CLS='cls2' +export CLUSTERID1='eks-e2e-clusterid-1' +export CLUSTERID2='eks-e2e-clusterid-2' +export CLUSTERSETID1='eks-e2e-clustersetid-1' export EXPECTED_ENDPOINT_COUNT=3 export UPDATED_ENDPOINT_COUNT=4 \ No newline at end of file diff --git a/integration/eks-test/scripts/eks-run-tests.sh b/integration/eks-test/scripts/eks-run-tests.sh index e8ccb4ff..451eba78 100755 --- a/integration/eks-test/scripts/eks-run-tests.sh +++ b/integration/eks-test/scripts/eks-run-tests.sh @@ -11,7 +11,7 @@ if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT fi # Runner to verify expected endpoints are exported to Cloud Map -go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$endpts" +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$endpts" exit_code=$? # Check imported endpoints in importing cluster @@ -47,7 +47,7 @@ if [ "$exit_code" -eq 0 ] ; then fi if [ "$exit_code" -eq 0 ] ; then - go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" + go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" exit_code=$? fi diff --git a/integration/eks-test/scripts/eks-setup.sh b/integration/eks-test/scripts/eks-setup.sh index d605930b..190cdbfa 100755 --- a/integration/eks-test/scripts/eks-setup.sh +++ b/integration/eks-test/scripts/eks-setup.sh @@ -2,6 +2,15 @@ source ./integration/eks-test/scripts/eks-common.sh +# Apply ClusterProperties +$KUBECTL_BIN config use-context $EXPORT_CLS +make install +$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-1.yaml" + +$KUBECTL_BIN config use-context $IMPORT_CLS +make install +$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-2.yaml" + # Call helper for service account and controller installation ./integration/eks-test/scripts/eks-setup-helper.sh $EXPORT_CLS ./integration/eks-test/scripts/eks-setup-helper.sh $IMPORT_CLS @@ -23,4 +32,3 @@ $KUBECTL_BIN apply -f "$CONFIGS/nginx-serviceexport.yaml" $KUBECTL_BIN config use-context $IMPORT_CLS $KUBECTL_BIN apply -f "$CONFIGS/client-hello.yaml" sleep 15 -$KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- apk add curl ## install curl diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index 6d8dd8dc..7dfd61ee 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -31,6 +31,7 @@ type CloudMapReconciler struct { // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch // +kubebuilder:rbac:groups="",resources=services,verbs=create;get;list;watch;update;delete +// +kubebuilder:rbac:groups=about.k8s.io,resources=clusterproperties,verbs=create;get;list;watch;update;patch;delete // +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;create;watch;update;delete // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceimports,verbs=create;get;list;watch;update;patch;delete diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index 535e4e98..aecbb1b7 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -44,6 +44,7 @@ type ServiceExportReconciler struct { } // +kubebuilder:rbac:groups="",resources=services,verbs=get +// +kubebuilder:rbac:groups=about.k8s.io,resources=clusterproperties,verbs=create;get;list;watch;update;patch;delete // +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;watch;create // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceexports,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceexports/finalizers,verbs=get;update From 4d7cea7a15660057c51f0baa30491f7755f22be6 Mon Sep 17 00:00:00 2001 From: Fred Wang <58385135+fredjywang@users.noreply.github.com> Date: Thu, 18 Aug 2022 10:50:38 -0700 Subject: [PATCH 101/163] Headless services local integration test (#194) --- Makefile | 2 +- integration/eks-test/scripts/eks-common.sh | 1 + integration/eks-test/scripts/eks-run-tests.sh | 4 +- integration/eks-test/scripts/eks-setup.sh | 10 +- .../kind-test/configs/e2e-client-hello.yaml | 12 ++ .../configs/e2e-coredns-deployment.yaml | 137 ++++++++++++++++++ .../configs/e2e-headless-export.yaml | 5 + .../kind-test/configs/e2e-headless.yaml | 12 ++ integration/kind-test/scripts/DNS-test.sh | 67 +++++++++ integration/kind-test/scripts/common.sh | 9 +- integration/kind-test/scripts/run-helper.sh | 21 +++ integration/kind-test/scripts/run-tests.sh | 48 ++++-- integration/kind-test/scripts/setup-kind.sh | 14 +- .../configs/coredns-clusterrole.yaml | 0 .../configs/coredns-configmap.yaml | 0 .../shared/scenarios/export_service.go | 4 +- integration/shared/scenarios/runner/main.go | 13 +- integration/shared/scripts/test-import.sh | 1 + 18 files changed, 324 insertions(+), 36 deletions(-) create mode 100644 integration/kind-test/configs/e2e-client-hello.yaml create mode 100644 integration/kind-test/configs/e2e-coredns-deployment.yaml create mode 100644 integration/kind-test/configs/e2e-headless-export.yaml create mode 100644 integration/kind-test/configs/e2e-headless.yaml create mode 100755 integration/kind-test/scripts/DNS-test.sh create mode 100755 integration/kind-test/scripts/run-helper.sh rename integration/{eks-test => shared}/configs/coredns-clusterrole.yaml (100%) rename integration/{eks-test => shared}/configs/coredns-configmap.yaml (100%) diff --git a/Makefile b/Makefile index 3b00d575..49e41e3d 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ kind-integration-setup: build kind ## Setup the integration test using kind clus @./integration/kind-test/scripts/setup-kind.sh kind-integration-run: ## Run the integration test controller - @./integration/kind-test/scripts/run-tests.sh + @./integration/kind-test/scripts/run-helper.sh kind-integration-cleanup: kind ## Cleanup integration test resources in Cloud Map and local kind cluster @./integration/kind-test/scripts/cleanup-kind.sh diff --git a/integration/eks-test/scripts/eks-common.sh b/integration/eks-test/scripts/eks-common.sh index 3a1848d7..95242547 100755 --- a/integration/eks-test/scripts/eks-common.sh +++ b/integration/eks-test/scripts/eks-common.sh @@ -8,6 +8,7 @@ export SCENARIOS='./integration/shared/scenarios' export NAMESPACE='aws-cloud-map-mcs-eks-e2e' export MCS_NAMESPACE='cloud-map-mcs-system' export SERVICE='nginx-hello' +export SERVICE_TYPE='ClusterSetIP' export CLIENT_POD='client-hello' export ENDPT_PORT=80 export SERVICE_PORT=80 # from nginx-service.yaml diff --git a/integration/eks-test/scripts/eks-run-tests.sh b/integration/eks-test/scripts/eks-run-tests.sh index 451eba78..160daec4 100755 --- a/integration/eks-test/scripts/eks-run-tests.sh +++ b/integration/eks-test/scripts/eks-run-tests.sh @@ -11,7 +11,7 @@ if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT fi # Runner to verify expected endpoints are exported to Cloud Map -go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$endpts" +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$endpts" exit_code=$? # Check imported endpoints in importing cluster @@ -47,7 +47,7 @@ if [ "$exit_code" -eq 0 ] ; then fi if [ "$exit_code" -eq 0 ] ; then - go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" + go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$updated_endpoints" exit_code=$? fi diff --git a/integration/eks-test/scripts/eks-setup.sh b/integration/eks-test/scripts/eks-setup.sh index 190cdbfa..c97e8674 100755 --- a/integration/eks-test/scripts/eks-setup.sh +++ b/integration/eks-test/scripts/eks-setup.sh @@ -2,19 +2,17 @@ source ./integration/eks-test/scripts/eks-common.sh +# Call helper for service account and controller installation +./integration/eks-test/scripts/eks-setup-helper.sh $EXPORT_CLS +./integration/eks-test/scripts/eks-setup-helper.sh $IMPORT_CLS + # Apply ClusterProperties $KUBECTL_BIN config use-context $EXPORT_CLS -make install $KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-1.yaml" $KUBECTL_BIN config use-context $IMPORT_CLS -make install $KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-2.yaml" -# Call helper for service account and controller installation -./integration/eks-test/scripts/eks-setup-helper.sh $EXPORT_CLS -./integration/eks-test/scripts/eks-setup-helper.sh $IMPORT_CLS - # Installing service $KUBECTL_BIN config use-context $EXPORT_CLS $KUBECTL_BIN create namespace $NAMESPACE diff --git a/integration/kind-test/configs/e2e-client-hello.yaml b/integration/kind-test/configs/e2e-client-hello.yaml new file mode 100644 index 00000000..969ebd54 --- /dev/null +++ b/integration/kind-test/configs/e2e-client-hello.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: client-hello + namespace: aws-cloud-map-mcs-e2e +spec: + containers: + - command: + - sleep + - "1d" + image: alpine + name: client-hello \ No newline at end of file diff --git a/integration/kind-test/configs/e2e-coredns-deployment.yaml b/integration/kind-test/configs/e2e-coredns-deployment.yaml new file mode 100644 index 00000000..00606eba --- /dev/null +++ b/integration/kind-test/configs/e2e-coredns-deployment.yaml @@ -0,0 +1,137 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + # eks.amazonaws.com/component: coredns + k8s-app: kube-dns + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + progressDeadlineSeconds: 600 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + # eks.amazonaws.com/component: coredns + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + # annotations: + # eks.amazonaws.com/compute-type: ec2 + creationTimestamp: null + labels: + # eks.amazonaws.com/component: coredns + k8s-app: kube-dns + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.4 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + - mountPath: /tmp + name: tmp + dnsPolicy: Default + priorityClassName: system-cluster-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: coredns + serviceAccountName: coredns + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + volumes: + - emptyDir: {} + name: tmp + - configMap: + defaultMode: 420 + items: + - key: Corefile + path: Corefile + name: coredns + name: config-volume \ No newline at end of file diff --git a/integration/kind-test/configs/e2e-headless-export.yaml b/integration/kind-test/configs/e2e-headless-export.yaml new file mode 100644 index 00000000..1129d5a1 --- /dev/null +++ b/integration/kind-test/configs/e2e-headless-export.yaml @@ -0,0 +1,5 @@ +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-headless diff --git a/integration/kind-test/configs/e2e-headless.yaml b/integration/kind-test/configs/e2e-headless.yaml new file mode 100644 index 00000000..f40a693f --- /dev/null +++ b/integration/kind-test/configs/e2e-headless.yaml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-headless +spec: + clusterIP: None + selector: + app: coredns + ports: + - port: 8080 + targetPort: 80 diff --git a/integration/kind-test/scripts/DNS-test.sh b/integration/kind-test/scripts/DNS-test.sh new file mode 100755 index 00000000..8b70e408 --- /dev/null +++ b/integration/kind-test/scripts/DNS-test.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# Testing service consumption with dnsutils pod + +echo "verifying single-cluster service consumption..." + +# Helper function to verify DNS results +checkDNS() { + endpt_count=$(echo "$1" | wc -l | xargs) + + if [ "$2" = "Headless" ]; then + if [ "$endpt_count" -ne "$3" ]; then + echo "ERROR: Found $endpt_count endpoints, expected $3 endpoints" + exit 1 + fi + fi + + if [ "$2" = "ClusterSetIP" ]; then + if [ "$endpt_count" -ne 1 ]; then + echo "ERROR: Found $endpt_count endpoints, expected 1 endpoint" + exit 1 + fi + fi +} + +# Add pod +$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-client-hello.yaml" +$KUBECTL_BIN wait --for=condition=ready pod/$DNS_POD -n $NAMESPACE # wait until pod is deployed + +# Install dig if not installed +$KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig -v &>/dev/null +exit_code=$? +if [ "$exit_code" -ne 0 ]; then + echo "dig not installed, installing..." + $KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- apk add --update bind-tools +fi + +# Perform a dig to cluster-local CoreDNS +# TODO: parse dig outputs for more precise verification - check specifics IPs? +echo "performing dig for A/AAAA records..." +addresses=$($KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) +exit_code=$? +echo "$addresses" + +if [ "$exit_code" -ne 0 ]; then + echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local" + exit $exit_code +fi + +# verify DNS results +checkDNS "$addresses" "$SERVICE_TYPE" "$1" + +echo "performing dig for SRV records..." +addresses=$($KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local. SRV +short) +exit_code=$? +echo "$addresses" + +if [ "$exit_code" -ne 0 ]; then + echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local" + exit $exit_code +fi + +# verify DNS results +checkDNS "$addresses" "$SERVICE_TYPE" "$1" + +echo "confirmed service consumption" +exit 0 diff --git a/integration/kind-test/scripts/common.sh b/integration/kind-test/scripts/common.sh index 768dbbec..1c5ae5a9 100755 --- a/integration/kind-test/scripts/common.sh +++ b/integration/kind-test/scripts/common.sh @@ -3,16 +3,19 @@ export KIND_BIN='./bin/kind' export KUBECTL_BIN='kubectl' export LOGS='./integration/kind-test/testlog' -export CONFIGS='./integration/kind-test/configs' +export KIND_CONFIGS='./integration/kind-test/configs' +export SHARED_CONFIGS='./integration/shared/configs' export SCENARIOS='./integration/shared/scenarios' export NAMESPACE='aws-cloud-map-mcs-e2e' -export SERVICE='e2e-service' export ENDPT_PORT=80 export SERVICE_PORT=8080 +export CLUSTERIP_SERVICE='e2e-service' +export HEADLESS_SERVICE='e2e-headless' export KIND_SHORT='cloud-map-e2e' export CLUSTER='kind-cloud-map-e2e' export CLUSTERID1='kind-e2e-clusterid-1' export CLUSTERSETID1='kind-e2e-clustersetid-1' -export IMAGE='kindest/node:v1.20.15@sha256:a6ce604504db064c5e25921c6c0fffea64507109a1f2a512b1b562ac37d652f3' +export DNS_POD='client-hello' +export IMAGE='kindest/node:v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207' export EXPECTED_ENDPOINT_COUNT=5 export UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/kind-test/scripts/run-helper.sh b/integration/kind-test/scripts/run-helper.sh new file mode 100755 index 00000000..02411c1d --- /dev/null +++ b/integration/kind-test/scripts/run-helper.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Helper to run test and passing different Service names + +source ./integration/kind-test/scripts/common.sh + +# ClusterIP service test +$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-service.yaml" +$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-export.yaml" +./integration/kind-test/scripts/run-tests.sh "$CLUSTERIP_SERVICE" "ClusterSetIP" +exit_code=$? + +# Headless service test +if [ "$exit_code" -eq 0 ] ; then + $KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-headless.yaml" + $KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-headless-export.yaml" + ./integration/kind-test/scripts/run-tests.sh "$HEADLESS_SERVICE" "Headless" + exit_code=$? +fi + +exit $exit_code \ No newline at end of file diff --git a/integration/kind-test/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh index 343984dd..6ec5e482 100755 --- a/integration/kind-test/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -3,11 +3,10 @@ # Runs the AWS Cloud Map MCS Controller for K8s as a background process and tests services have been exported source ./integration/kind-test/scripts/common.sh +export SERVICE=$1 +export SERVICE_TYPE=$2 -$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty.yaml" -$KUBECTL_BIN apply -f "$CONFIGS/e2e-deployment.yaml" -$KUBECTL_BIN apply -f "$CONFIGS/e2e-service.yaml" -$KUBECTL_BIN apply -f "$CONFIGS/e2e-export.yaml" +echo "testing service: $SERVICE" if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") ; then exit $? @@ -18,7 +17,7 @@ mkdir -p "$LOGS" CTL_PID=$! echo "controller PID:$CTL_PID" -go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$endpts" +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$endpts" exit_code=$? if [ "$exit_code" -eq 0 ] ; then @@ -26,27 +25,48 @@ if [ "$exit_code" -eq 0 ] ; then exit_code=$? fi +if [ "$exit_code" -eq 0 ] ; then + ./integration/kind-test/scripts/DNS-test.sh "$EXPECTED_ENDPOINT_COUNT" + exit_code=$? +fi + echo "sleeping..." sleep 2 -deployment=$($KUBECTL_BIN get deployment --namespace "$NAMESPACE" -o json | jq -r '.items[0].metadata.name') +if [ "$exit_code" -eq 0 ] ; then + deployment=$($KUBECTL_BIN get deployment --namespace "$NAMESPACE" -o json | jq -r '.items[0].metadata.name') -echo "scaling the deployment $deployment to $UPDATED_ENDPOINT_COUNT" -$KUBECTL_BIN scale deployment/"$deployment" --replicas="$UPDATED_ENDPOINT_COUNT" --namespace "$NAMESPACE" -exit_code=$? + echo "scaling the deployment $deployment to $UPDATED_ENDPOINT_COUNT" + $KUBECTL_BIN scale deployment/"$deployment" --replicas="$UPDATED_ENDPOINT_COUNT" --namespace "$NAMESPACE" + exit_code=$? +fi if [ "$exit_code" -eq 0 ] ; then if ! updated_endpoints=$(./integration/shared/scripts/poll-endpoints.sh "$UPDATED_ENDPOINT_COUNT") ; then exit $? fi +fi - go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$updated_endpoints" +if [ "$exit_code" -eq 0 ] ; then + go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$updated_endpoints" exit_code=$? +fi - if [ "$exit_code" -eq 0 ] ; then - ./integration/shared/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" - exit_code=$? - fi +if [ "$exit_code" -eq 0 ] ; then + ./integration/shared/scripts/test-import.sh "$UPDATED_ENDPOINT_COUNT" "$updated_endpoints" + exit_code=$? +fi + +if [ "$exit_code" -eq 0 ] ; then + ./integration/kind-test/scripts/DNS-test.sh "$UPDATED_ENDPOINT_COUNT" + exit_code=$? +fi + +# Scale deployment back down for future test and delete service export +if [ "$exit_code" -eq 0 ] ; then + $KUBECTL_BIN scale deployment/"$deployment" --replicas="$EXPECTED_ENDPOINT_COUNT" --namespace "$NAMESPACE" + $KUBECTL_BIN delete ServiceExport $SERVICE -n $NAMESPACE + sleep 5 fi echo "killing controller PID:$CTL_PID" diff --git a/integration/kind-test/scripts/setup-kind.sh b/integration/kind-test/scripts/setup-kind.sh index e8c9b8ee..b04361f2 100755 --- a/integration/kind-test/scripts/setup-kind.sh +++ b/integration/kind-test/scripts/setup-kind.sh @@ -11,7 +11,17 @@ source ./integration/kind-test/scripts/common.sh $KIND_BIN create cluster --name "$KIND_SHORT" --image "$IMAGE" $KUBECTL_BIN config use-context "$CLUSTER" -$KUBECTL_BIN create namespace "$NAMESPACE" make install -exit 0 +# Install CoreDNS plugin +$KUBECTL_BIN apply -f "$SHARED_CONFIGS/coredns-clusterrole.yaml" +$KUBECTL_BIN apply -f "$SHARED_CONFIGS/coredns-configmap.yaml" +$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-coredns-deployment.yaml" + +$KUBECTL_BIN create namespace "$NAMESPACE" + +# Add ClusterId and ClusterSetId +$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-clusterproperty.yaml" + +# Deploy pods +$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-deployment.yaml" diff --git a/integration/eks-test/configs/coredns-clusterrole.yaml b/integration/shared/configs/coredns-clusterrole.yaml similarity index 100% rename from integration/eks-test/configs/coredns-clusterrole.yaml rename to integration/shared/configs/coredns-clusterrole.yaml diff --git a/integration/eks-test/configs/coredns-configmap.yaml b/integration/shared/configs/coredns-configmap.yaml similarity index 100% rename from integration/eks-test/configs/coredns-configmap.yaml rename to integration/shared/configs/coredns-configmap.yaml diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index 8698ba41..09c41e72 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -33,7 +33,7 @@ type exportServiceScenario struct { expectedSvc model.Service } -func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, clusterId string, clusterSetId string, portStr string, servicePortStr string, ips string) (ExportServiceScenario, error) { +func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, clusterId string, clusterSetId string, portStr string, servicePortStr string, serviceType string, ips string) (ExportServiceScenario, error) { endpts := make([]*model.Endpoint, 0) port, parseError := strconv.ParseUint(portStr, 10, 16) @@ -61,7 +61,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, cl EndpointPort: endpointPort, ClusterId: clusterId, ClusterSetId: clusterSetId, - ServiceType: model.ClusterSetIPType, // in scenario, we assume ClusterSetIP type + ServiceType: model.ServiceType(serviceType), Attributes: make(map[string]string), }) } diff --git a/integration/shared/scenarios/runner/main.go b/integration/shared/scenarios/runner/main.go index fd4f23b9..ce94c9b4 100644 --- a/integration/shared/scenarios/runner/main.go +++ b/integration/shared/scenarios/runner/main.go @@ -11,8 +11,8 @@ import ( ) func main() { - if len(os.Args) != 8 { - fmt.Println("Expected namespace, service, clusterId, clusterSetId, endpoint port, service port and endpoint IP list arguments") + if len(os.Args) != 9 { + fmt.Println("Expected namespace, service, clusterId, clusterSetId, endpoint port, service port, serviceType, and endpoint IP list as arguments") os.Exit(1) } @@ -22,15 +22,16 @@ func main() { clusterSetId := os.Args[4] port := os.Args[5] servicePort := os.Args[6] - ips := os.Args[7] + serviceType := os.Args[7] + ips := os.Args[8] - testServiceExport(nsName, svcName, clusterId, clusterSetId, port, servicePort, ips) + testServiceExport(nsName, svcName, clusterId, clusterSetId, port, servicePort, serviceType, ips) } -func testServiceExport(nsName string, svcName string, clusterId string, clusterSetId string, port string, servicePort string, ips string) { +func testServiceExport(nsName string, svcName string, clusterId string, clusterSetId string, port string, servicePort string, serviceType string, ips string) { fmt.Printf("Testing service export integration for namespace %s and service %s\n", nsName, svcName) - export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, clusterId, clusterSetId, port, servicePort, ips) + export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, clusterId, clusterSetId, port, servicePort, serviceType, ips) if err != nil { fmt.Printf("Failed to setup service export integration test scenario: %s", err.Error()) os.Exit(1) diff --git a/integration/shared/scripts/test-import.sh b/integration/shared/scripts/test-import.sh index ed986331..d496bcf8 100755 --- a/integration/shared/scripts/test-import.sh +++ b/integration/shared/scripts/test-import.sh @@ -12,6 +12,7 @@ import_count=0 poll_count=0 while ((import_count < expected_endpoint_count)) do + sleep 1 if ((poll_count++ > 30)) ; then echo "timed out polling for import endpoints" exit 1 From bed17fe53c471f63b4786692ec06c79b1bd200e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Aug 2022 11:06:22 -0700 Subject: [PATCH 102/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.15.14 to 1.17.1 (#195) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.15.14 to 1.17.1. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.15.14...config/v1.17.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 24 ++++++++++++------------ go.sum | 41 ++++++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 8ba0ac5f..e9283416 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.8 - github.com/aws/aws-sdk-go-v2/config v1.15.14 + github.com/aws/aws-sdk-go-v2 v1.16.11 + github.com/aws/aws-sdk-go-v2/config v1.17.1 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 + github.com/google/go-cmp v0.5.8 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.20.0 github.com/pkg/errors v0.9.1 @@ -28,15 +29,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.9 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.9 // indirect - github.com/aws/smithy-go v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 // indirect + github.com/aws/smithy-go v1.12.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -52,7 +53,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.8 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect diff --git a/go.sum b/go.sum index dd03ec1a..36863f18 100644 --- a/go.sum +++ b/go.sum @@ -75,30 +75,33 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= -github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ= -github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= -github.com/aws/aws-sdk-go-v2/config v1.15.14 h1:+BqpqlydTq4c2et9Daury7gE+o67P4lbk7eybiCBNc4= -github.com/aws/aws-sdk-go-v2/config v1.15.14/go.mod h1:CQBv+VVv8rR5z2xE+Chdh5m+rFfsqeY4k0veEZeq6QM= -github.com/aws/aws-sdk-go-v2/credentials v1.12.9 h1:DloAJr0/jbvm0iVRFDFh8GlWxrOd9XKyX82U+dfVeZs= -github.com/aws/aws-sdk-go-v2/credentials v1.12.9/go.mod h1:2Vavxl1qqQXJ8MUcQZTsIEW8cwenFCWYXtLRPba3L/o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 h1:VfBdn2AxwMbFyJN/lF/xuT3SakomJ86PZu3rCxb5K0s= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8/go.mod h1:oL1Q3KuCq1D4NykQnIvtRiBGLUXhcpY5pl6QZB2XEPU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 h1:2C0pYHcUBmdzPj+EKNC4qj97oK6yjrUhc1KoSodglvk= +github.com/aws/aws-sdk-go-v2 v1.16.11 h1:xM1ZPSvty3xVmdxiGr7ay/wlqv+MWhH0rMlyLdbC0YQ= +github.com/aws/aws-sdk-go-v2 v1.16.11/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo= +github.com/aws/aws-sdk-go-v2/config v1.17.1 h1:BWxTjokU/69BZ4DnLrZco6OvBDii6ToEdfBL/y5I1nA= +github.com/aws/aws-sdk-go-v2/config v1.17.1/go.mod h1:uOxDHjBemNTF2Zos+fgG0NNfE86wn1OAHDTGxjMEYi0= +github.com/aws/aws-sdk-go-v2/credentials v1.12.14 h1:AtVG/amkjbDBfnPr/tuW2IG18HGNznP6L12Dx0rLz+Q= +github.com/aws/aws-sdk-go-v2/credentials v1.12.14/go.mod h1:opAndTyq+YN7IpVG57z2CeNuXSQMqTYxGGlYH0m0RMY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 h1:wgJBHO58Pc1V1QAnzdVM3JK3WbE/6eUF0JxCZ+/izz0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12/go.mod h1:aZ4vZnyUuxedC7eD4JyEHpGnCz+O2sHQEx3VvAwklSE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14/go.mod h1:kdjrMwHwrC3+FsKhNcCMJ7tUVj/8uSD5CZXeQ4wV6fM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8 h1:2J+jdlBJWEmTyAwC82Ym68xCykIvnSnIN18b8xHGlcc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 h1:OmiwoVyLKEqqD5GvB683dbSqxiOfvx4U2lDZhG2Esc4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18/go.mod h1:348MLhzV1GSlZSMusdwQpXKbhD7X2gbI/TxwAPKkYZQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8/go.mod h1:ZIV8GYoC6WLBW5KGs+o4rsc65/ozd+eQ0L31XF5VDwk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 h1:QquxR7NH3ULBsKC+NoTpilzbKKS+5AELfNREInbhvas= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15/go.mod h1:Tkrthp/0sNBShQQsamR7j/zY4p19tVTAs+nnqhH6R3c= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 h1:oKnAXxSF2FUvfgw8uzU/v9OTYorJJZ8eBmWhr9TWVVQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8/go.mod h1:rDVhIMAX9N2r8nWxDUlbubvvaFMnfsm+3jAV7q+rpM4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 h1:5mvQDtNWtI6H56+E4LUnLWEmATMB7oEh+Z9RurtIuC0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12/go.mod h1:ckaCVTEdGAxO6KwTGzgskxR1xM+iJW4lxMyDFVda2Fc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 h1:g5qq9sgtEzt2szMaDqQO6fqKe026T6dHTFJp5NsPzkQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19/go.mod h1:cVHo8KTuHjShb9V8/VjH3S/8+xPu16qx8fdGwmotJhE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12 h1:7iPTTX4SAI2U2VOogD7/gmHlsgnYSgoNHt7MSQXtG2M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12/go.mod h1:1TODGhheLWjpQWSuhYuAUWYTCKwEjx2iblIFKDHjeTc= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 h1:jv1gOqDBXIFKrxay5gbZ0Ii0wZeXPQ8LHdwjzU6A5To= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8/go.mod h1:G5MzJj6NHYmeI8cN2PRd+QkGqbFT3jPM+Q0h0GL9lZQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.12 h1:760bUnTX/+d693FT6T6Oa7PZHfEQT9XMFZeM5IQIB0A= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.12/go.mod h1:MO4qguFjs3wPGcCSpQ7kOFTwRvb+eu+fn+1vKleGHUk= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.9 h1:yOfILxyjmtr2ubRkRJldlHDFBhf5vw4CzhbwWIBmimQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.9/go.mod h1:O1IvkYxr+39hRf960Us6j0x1P8pDqhTX+oXM5kQNl/Y= -github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 h1:pXxu9u2z1UqSbjO9YA8kmFJBhFc1EVTDaf7A+S+Ivq8= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.17/go.mod h1:mS5xqLZc/6kc06IpXn5vRxdLaED+jEuaSRv5BxtnsiY= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 h1:dl8T0PJlN92rvEGOEUiD0+YPYdPEaCZK0TqHukvSfII= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.13/go.mod h1:Ru3QVMLygVs/07UQ3YDur1AQZZp2tUNje8wfloFttC0= github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.12.1 h1:yQRC55aXN/y1W10HgwHle01DRuV9Dpf31iGkotjt3Ag= +github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 6983cf67beeb0157fa277e43499e68f6462d94e4 Mon Sep 17 00:00:00 2001 From: Matthew Goodman Date: Mon, 29 Aug 2022 14:41:42 -0700 Subject: [PATCH 103/163] Propagate SvcExportCreationTimestamp to Endpoints (#200) * Added managed by label - issue 110 * Propagate CreationTimestamp to Endpoints * intg - remove creationtimestamp in endpoint cmp * Change to UnixMillis & Rename to SvcExportCreationTimestamp * fix var being stored in CM * set svcExportCreationTimestamp to 0 when errors or not set --- .../shared/scenarios/export_service.go | 2 + pkg/cloudmap/client_test.go | 93 ++++++++-------- .../multicluster/controllers_common_test.go | 8 +- .../multicluster/serviceexport_controller.go | 26 +++-- pkg/model/types.go | 58 ++++++---- pkg/model/types_test.go | 100 ++++++++++-------- test/test-constants.go | 83 ++++++++------- 7 files changed, 209 insertions(+), 161 deletions(-) diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index 09c41e72..28a432e6 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -112,6 +112,8 @@ func (e *exportServiceScenario) compareEndpoints(cmEndpoints []*model.Endpoint) for _, actual := range cmEndpoints { // Ignore K8S instance attribute for the purpose of this test. delete(actual.Attributes, multiclustercontrollers.K8sVersionAttr) + // Ignore SvcExportCreationTimestamp attribute for the purpose of this test by setting value to 0. + actual.SvcExportCreationTimestamp = 0 if expected.Equals(actual) { match = true break diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index e666fd0d..94edc212 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -2,6 +2,7 @@ package cloudmap import ( "context" + "strconv" "testing" cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" @@ -283,30 +284,32 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) attrs1 := map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp1, - model.EndpointPortAttr: test.PortStr1, - model.EndpointPortNameAttr: test.PortName1, - model.EndpointProtocolAttr: test.Protocol1, - model.ServicePortNameAttr: test.PortName1, - model.ServicePortAttr: test.ServicePortStr1, - model.ServiceProtocolAttr: test.Protocol1, - model.ServiceTargetPortAttr: test.PortStr1, - model.ServiceTypeAttr: test.SvcType, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, + model.ServiceTypeAttr: test.SvcType, + model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), } attrs2 := map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp2, - model.EndpointPortAttr: test.PortStr2, - model.EndpointPortNameAttr: test.PortName2, - model.EndpointProtocolAttr: test.Protocol2, - model.ServicePortNameAttr: test.PortName2, - model.ServicePortAttr: test.ServicePortStr2, - model.ServiceProtocolAttr: test.Protocol2, - model.ServiceTargetPortAttr: test.PortStr2, - model.ServiceTypeAttr: test.SvcType, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, + model.ServiceTypeAttr: test.SvcType, + model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), } tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). @@ -371,33 +374,35 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { { InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp1, - model.EndpointPortAttr: test.PortStr1, - model.EndpointPortNameAttr: test.PortName1, - model.EndpointProtocolAttr: test.Protocol1, - model.ServicePortNameAttr: test.PortName1, - model.ServicePortAttr: test.ServicePortStr1, - model.ServiceProtocolAttr: test.Protocol1, - model.ServiceTargetPortAttr: test.PortStr1, - model.ServiceTypeAttr: test.SvcType, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, + model.ServiceTypeAttr: test.SvcType, + model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), }, }, { InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp2, - model.EndpointPortAttr: test.PortStr2, - model.EndpointPortNameAttr: test.PortName2, - model.EndpointProtocolAttr: test.Protocol2, - model.ServicePortNameAttr: test.PortName2, - model.ServicePortAttr: test.ServicePortStr2, - model.ServiceProtocolAttr: test.Protocol2, - model.ServiceTargetPortAttr: test.PortStr2, - model.ServiceTypeAttr: test.SvcType, + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSetId1, + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, + model.ServiceTypeAttr: test.SvcType, + model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), }, }, } diff --git a/pkg/controllers/multicluster/controllers_common_test.go b/pkg/controllers/multicluster/controllers_common_test.go index e5a63246..edb2d1cb 100644 --- a/pkg/controllers/multicluster/controllers_common_test.go +++ b/pkg/controllers/multicluster/controllers_common_test.go @@ -1,6 +1,8 @@ package controllers import ( + "time" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" @@ -41,10 +43,12 @@ func k8sServiceForTest() *v1.Service { } func serviceExportForTest() *multiclusterv1alpha1.ServiceExport { + creationTimestamp := metav1.NewTime(time.UnixMilli(test.SvcExportCreationTimestamp)) return &multiclusterv1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ - Name: test.SvcName, - Namespace: test.HttpNsName, + Name: test.SvcName, + Namespace: test.HttpNsName, + CreationTimestamp: creationTimestamp, }, } } diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index aecbb1b7..0ec1cc17 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -139,7 +139,7 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor return ctrl.Result{}, err } - endpoints, err := r.extractEndpoints(ctx, service) + endpoints, err := r.extractEndpoints(ctx, service, serviceExport) if err != nil { r.Log.Error(err, "error extracting Endpoints", "namespace", serviceExport.Namespace, "name", serviceExport.Name) @@ -230,7 +230,7 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor return ctrl.Result{}, nil } -func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1.Service) ([]*model.Endpoint, error) { +func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1.Service, svcExport *multiclusterv1alpha1.ServiceExport) ([]*model.Endpoint, error) { result := make([]*model.Endpoint, 0) endpointSlices := discovery.EndpointSliceList{} @@ -259,6 +259,11 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. return nil, err } + var svcExportCreationTimestamp int64 = 0 + if !svcExport.ObjectMeta.CreationTimestamp.IsZero() { + svcExportCreationTimestamp = svcExport.ObjectMeta.CreationTimestamp.Time.UnixMilli() + } + for _, slice := range endpointSlices.Items { if slice.AddressType != discovery.AddressTypeIPv4 { return nil, fmt.Errorf("unsupported address type %s for service %s", slice.AddressType, svc.Name) @@ -275,14 +280,15 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. port := EndpointPortToPort(endpointPort) result = append(result, &model.Endpoint{ - Id: model.EndpointIdFromIPAddressAndPort(IP, port), - IP: IP, - EndpointPort: port, - ServicePort: servicePortMap[*endpointPort.Name], - ClusterId: clusterId, - ClusterSetId: clusterSetId, - ServiceType: serviceType, - Attributes: attributes, + Id: model.EndpointIdFromIPAddressAndPort(IP, port), + IP: IP, + EndpointPort: port, + ServicePort: servicePortMap[*endpointPort.Name], + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: serviceType, + SvcExportCreationTimestamp: svcExportCreationTimestamp, + Attributes: attributes, }) } } diff --git a/pkg/model/types.go b/pkg/model/types.go index e71d94d9..92355775 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -42,14 +42,15 @@ type ServiceType string // Endpoint holds basic values and attributes for an endpoint. type Endpoint struct { - Id string - IP string - EndpointPort Port - ServicePort Port - ClusterId string - ClusterSetId string - ServiceType ServiceType - Attributes map[string]string + Id string + IP string + EndpointPort Port + ServicePort Port + ClusterId string + ClusterSetId string + ServiceType ServiceType + SvcExportCreationTimestamp int64 + Attributes map[string]string } type Port struct { @@ -62,17 +63,18 @@ type Port struct { // Cloudmap Instances IP and Port is supposed to be AWS_INSTANCE_IPV4 and AWS_INSTANCE_PORT // Rest are custom attributes const ( - EndpointIpv4Attr = "AWS_INSTANCE_IPV4" - EndpointPortAttr = "AWS_INSTANCE_PORT" - EndpointPortNameAttr = "ENDPOINT_PORT_NAME" - EndpointProtocolAttr = "ENDPOINT_PROTOCOL" - ClusterIdAttr = "CLUSTER_ID" - ClusterSetIdAttr = "CLUSTERSET_ID" - ServicePortNameAttr = "SERVICE_PORT_NAME" - ServicePortAttr = "SERVICE_PORT" - ServiceTargetPortAttr = "SERVICE_TARGET_PORT" - ServiceProtocolAttr = "SERVICE_PROTOCOL" - ServiceTypeAttr = "SERVICE_TYPE" + SvcExportCreationTimestampAttr = "SVC_EXPORT_CREATION_TIMESTAMP" + EndpointIpv4Attr = "AWS_INSTANCE_IPV4" + EndpointPortAttr = "AWS_INSTANCE_PORT" + EndpointPortNameAttr = "ENDPOINT_PORT_NAME" + EndpointProtocolAttr = "ENDPOINT_PROTOCOL" + ClusterIdAttr = "CLUSTER_ID" + ClusterSetIdAttr = "CLUSTERSET_ID" + ServicePortNameAttr = "SERVICE_PORT_NAME" + ServicePortAttr = "SERVICE_PORT" + ServiceTargetPortAttr = "SERVICE_TARGET_PORT" + ServiceProtocolAttr = "SERVICE_PROTOCOL" + ServiceTypeAttr = "SERVICE_TYPE" ) // NewEndpointFromInstance converts a Cloud Map HttpInstanceSummary to an endpoint. @@ -119,6 +121,10 @@ func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) return nil, err } + if endpoint.SvcExportCreationTimestamp, err = removeTimestampAttr(attributes, SvcExportCreationTimestampAttr); err != nil { + endpoint.SvcExportCreationTimestamp = 0 + } + // Add the remaining attributes endpoint.Attributes = attributes @@ -177,6 +183,19 @@ func removeIntAttr(attributes map[string]string, attr string) (int32, error) { return 0, fmt.Errorf("cannot find the attribute %s", attr) } +func removeTimestampAttr(attributes map[string]string, attr string) (int64, error) { + if value, hasValue := attributes[attr]; hasValue { + parsedValue, parseError := strconv.ParseInt(value, 10, 64) + if parseError != nil { + return 0, fmt.Errorf("failed to parse the %s as int with error %s", + attr, parseError.Error()) + } + delete(attributes, attr) + return parsedValue, nil + } + return 0, fmt.Errorf("cannot find the attribute %s", attr) +} + // GetCloudMapAttributes extracts endpoint attributes for Cloud Map service instance registration. func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs := make(map[string]string) @@ -192,6 +211,7 @@ func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs[ServiceTargetPortAttr] = e.ServicePort.TargetPort attrs[ServiceProtocolAttr] = e.ServicePort.Protocol attrs[ServiceTypeAttr] = e.ServiceType.String() + attrs[SvcExportCreationTimestampAttr] = strconv.FormatInt(e.SvcExportCreationTimestamp, 10) for key, val := range e.Attributes { attrs[key] = val diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index 2ef9a66c..30bd48fb 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -2,6 +2,7 @@ package model import ( "reflect" + "strconv" "testing" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" @@ -12,6 +13,7 @@ var ip = "192.168.0.1" var clusterId = "test-mcs-clusterId" var clusterSetId = "test-mcs-clusterSetId" var serviceType = ClusterSetIPType.String() +var svcExportCreationTimestamp int64 = 1640995200000 func TestNewEndpointFromInstance(t *testing.T) { tests := []struct { @@ -25,18 +27,19 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - ClusterIdAttr: clusterId, - ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, - EndpointPortAttr: "80", - EndpointProtocolAttr: "TCP", - EndpointPortNameAttr: "http", - ServicePortNameAttr: "http", - ServiceProtocolAttr: "TCP", - ServicePortAttr: "65535", - ServiceTargetPortAttr: "80", - ServiceTypeAttr: serviceType, - "custom-attr": "custom-val", + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, + SvcExportCreationTimestampAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), + "custom-attr": "custom-val", }, }, want: &Endpoint{ @@ -53,9 +56,10 @@ func TestNewEndpointFromInstance(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, - ClusterId: clusterId, - ClusterSetId: clusterSetId, - ServiceType: ServiceType(serviceType), + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), + SvcExportCreationTimestamp: svcExportCreationTimestamp, Attributes: map[string]string{ "custom-attr": "custom-val", }, @@ -177,14 +181,15 @@ func TestNewEndpointFromInstance(t *testing.T) { func TestEndpoint_GetAttributes(t *testing.T) { type fields struct { - Id string - IP string - EndpointPort Port - ServicePort Port - ClusterId string - ClusterSetId string - ServiceType ServiceType - Attributes map[string]string + Id string + IP string + EndpointPort Port + ServicePort Port + ClusterId string + ClusterSetId string + ServiceType ServiceType + SvcExportCreationTimestamp int64 + Attributes map[string]string } tests := []struct { name string @@ -206,40 +211,43 @@ func TestEndpoint_GetAttributes(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, - ClusterId: clusterId, - ClusterSetId: clusterSetId, - ServiceType: ServiceType(serviceType), + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), + SvcExportCreationTimestamp: svcExportCreationTimestamp, Attributes: map[string]string{ "custom-attr": "custom-val", }, }, want: map[string]string{ - ClusterIdAttr: clusterId, - ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, - EndpointPortAttr: "80", - EndpointProtocolAttr: "TCP", - EndpointPortNameAttr: "http", - ServicePortNameAttr: "http", - ServiceProtocolAttr: "TCP", - ServicePortAttr: "30", - ServiceTargetPortAttr: "80", - ServiceTypeAttr: serviceType, - "custom-attr": "custom-val", + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "30", + ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, + SvcExportCreationTimestampAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), + "custom-attr": "custom-val", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { e := &Endpoint{ - Id: tt.fields.Id, - IP: tt.fields.IP, - EndpointPort: tt.fields.EndpointPort, - ServicePort: tt.fields.ServicePort, - ClusterId: tt.fields.ClusterId, - ClusterSetId: tt.fields.ClusterSetId, - ServiceType: tt.fields.ServiceType, - Attributes: tt.fields.Attributes, + Id: tt.fields.Id, + IP: tt.fields.IP, + EndpointPort: tt.fields.EndpointPort, + ServicePort: tt.fields.ServicePort, + ClusterId: tt.fields.ClusterId, + ClusterSetId: tt.fields.ClusterSetId, + ServiceType: tt.fields.ServiceType, + SvcExportCreationTimestamp: tt.fields.SvcExportCreationTimestamp, + Attributes: tt.fields.Attributes, } if got := e.GetCloudMapAttributes(); !reflect.DeepEqual(got, tt.want) { t.Errorf("GetAttributes() = %v, want %v", got, tt.want) diff --git a/test/test-constants.go b/test/test-constants.go index 89c98551..a689be86 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -12,38 +12,39 @@ import ( ) const ( - HttpNsName = "http-ns-name" - DnsNsName = "dns-ns-name" - HttpNsId = "http-ns-id" - DnsNsId = "dns-ns-id" - SvcName = "svc-name" - SvcId = "svc-id" - ClusterId1 = "test-mcs-clusterid-1" - ClusterSetId1 = "test-mcs-clustersetid-1" - ClusterId2 = "test-mcs-clusterid-2" - ClusterSetId2 = "test-mcs-clustersetid-2" - EndptId1 = "tcp-192_168_0_1-1" - EndptId2 = "tcp-192_168_0_2-2" - EndptIp1 = "192.168.0.1" - EndptIp2 = "192.168.0.2" - Port1 = 1 - PortStr1 = "1" - PortName1 = "http" - Protocol1 = "TCP" - ServicePort1 = 11 - ServicePortStr1 = "11" - Port2 = 2 - PortStr2 = "2" - PortName2 = "https" - Protocol2 = "UDP" - ServicePort2 = 22 - ServicePortStr2 = "22" - ClusterIp1 = "10.10.10.1" - ClusterIp2 = "10.10.10.2" - OpId1 = "operation-id-1" - OpId2 = "operation-id-2" - OpStart = 1 - SvcType = "ClusterSetIP" + HttpNsName = "http-ns-name" + DnsNsName = "dns-ns-name" + HttpNsId = "http-ns-id" + DnsNsId = "dns-ns-id" + SvcName = "svc-name" + SvcId = "svc-id" + ClusterId1 = "test-mcs-clusterid-1" + ClusterSetId1 = "test-mcs-clustersetid-1" + ClusterId2 = "test-mcs-clusterid-2" + ClusterSetId2 = "test-mcs-clustersetid-2" + EndptId1 = "tcp-192_168_0_1-1" + EndptId2 = "tcp-192_168_0_2-2" + EndptIp1 = "192.168.0.1" + EndptIp2 = "192.168.0.2" + Port1 = 1 + PortStr1 = "1" + PortName1 = "http" + Protocol1 = "TCP" + ServicePort1 = 11 + ServicePortStr1 = "11" + Port2 = 2 + PortStr2 = "2" + PortName2 = "https" + Protocol2 = "UDP" + ServicePort2 = 22 + ServicePortStr2 = "22" + ClusterIp1 = "10.10.10.1" + ClusterIp2 = "10.10.10.2" + OpId1 = "operation-id-1" + OpId2 = "operation-id-2" + OpStart = 1 + SvcType = "ClusterSetIP" + SvcExportCreationTimestamp int64 = 1640995200000 ) func GetTestHttpNamespace() *model.Namespace { @@ -102,10 +103,11 @@ func GetTestEndpoint1() *model.Endpoint { TargetPort: PortStr1, Protocol: Protocol1, }, - ClusterId: ClusterId1, - ClusterSetId: ClusterSetId1, - ServiceType: model.ClusterSetIPType, - Attributes: make(map[string]string), + ClusterId: ClusterId1, + ClusterSetId: ClusterSetId1, + ServiceType: model.ClusterSetIPType, + SvcExportCreationTimestamp: SvcExportCreationTimestamp, + Attributes: make(map[string]string), } } @@ -124,10 +126,11 @@ func GetTestEndpoint2() *model.Endpoint { TargetPort: PortStr2, Protocol: Protocol2, }, - ClusterId: ClusterId1, - ClusterSetId: ClusterSetId1, - ServiceType: model.ClusterSetIPType, - Attributes: make(map[string]string), + ClusterId: ClusterId1, + ClusterSetId: ClusterSetId1, + ServiceType: model.ClusterSetIPType, + SvcExportCreationTimestamp: SvcExportCreationTimestamp, + Attributes: make(map[string]string), } } From 3e8fde3b04b575df65e07a46498a36a86406dbdb Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Mon, 29 Aug 2022 21:14:02 -0700 Subject: [PATCH 104/163] Update dependabot.yml (#206) Reduce dependabot frequency. --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b444581e..9beafcac 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,4 +8,4 @@ updates: - package-ecosystem: "gomod" # See documentation for possible values directory: "/" # Location of package manifests schedule: - interval: "daily" + interval: "monthly" From cec6b51f1903488341a58231e606021880c83e51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 21:14:33 -0700 Subject: [PATCH 105/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#191) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.8 to 1.17.13. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.8...service/sns/v1.17.13) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index e9283416..cc03e784 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.11 github.com/aws/aws-sdk-go-v2/config v1.17.1 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.13 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 diff --git a/go.sum b/go.sum index 36863f18..73732508 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2 v1.16.11 h1:xM1ZPSvty3xVmdxiGr7ay/wlqv+MWhH0rMlyLdbC0YQ= github.com/aws/aws-sdk-go-v2 v1.16.11/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo= github.com/aws/aws-sdk-go-v2/config v1.17.1 h1:BWxTjokU/69BZ4DnLrZco6OvBDii6ToEdfBL/y5I1nA= @@ -83,23 +82,20 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.14 h1:AtVG/amkjbDBfnPr/tuW2IG18HG github.com/aws/aws-sdk-go-v2/credentials v1.12.14/go.mod h1:opAndTyq+YN7IpVG57z2CeNuXSQMqTYxGGlYH0m0RMY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 h1:wgJBHO58Pc1V1QAnzdVM3JK3WbE/6eUF0JxCZ+/izz0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12/go.mod h1:aZ4vZnyUuxedC7eD4JyEHpGnCz+O2sHQEx3VvAwklSE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14/go.mod h1:kdjrMwHwrC3+FsKhNcCMJ7tUVj/8uSD5CZXeQ4wV6fM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 h1:OmiwoVyLKEqqD5GvB683dbSqxiOfvx4U2lDZhG2Esc4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18/go.mod h1:348MLhzV1GSlZSMusdwQpXKbhD7X2gbI/TxwAPKkYZQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8/go.mod h1:ZIV8GYoC6WLBW5KGs+o4rsc65/ozd+eQ0L31XF5VDwk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 h1:5mvQDtNWtI6H56+E4LUnLWEmATMB7oEh+Z9RurtIuC0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12/go.mod h1:ckaCVTEdGAxO6KwTGzgskxR1xM+iJW4lxMyDFVda2Fc= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 h1:g5qq9sgtEzt2szMaDqQO6fqKe026T6dHTFJp5NsPzkQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19/go.mod h1:cVHo8KTuHjShb9V8/VjH3S/8+xPu16qx8fdGwmotJhE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12 h1:7iPTTX4SAI2U2VOogD7/gmHlsgnYSgoNHt7MSQXtG2M= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12/go.mod h1:1TODGhheLWjpQWSuhYuAUWYTCKwEjx2iblIFKDHjeTc= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8 h1:jv1gOqDBXIFKrxay5gbZ0Ii0wZeXPQ8LHdwjzU6A5To= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.8/go.mod h1:G5MzJj6NHYmeI8cN2PRd+QkGqbFT3jPM+Q0h0GL9lZQ= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.13 h1:IGZqdrpJN0NkjgjQAdxgKEm5lRgPUI3IC4XDHTL6fGQ= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.13/go.mod h1:pARAWVRnxP2jp+WP5FiUV26br9hEODC6NBryEJaHK80= github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 h1:pXxu9u2z1UqSbjO9YA8kmFJBhFc1EVTDaf7A+S+Ivq8= github.com/aws/aws-sdk-go-v2/service/sso v1.11.17/go.mod h1:mS5xqLZc/6kc06IpXn5vRxdLaED+jEuaSRv5BxtnsiY= github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 h1:dl8T0PJlN92rvEGOEUiD0+YPYdPEaCZK0TqHukvSfII= github.com/aws/aws-sdk-go-v2/service/sts v1.16.13/go.mod h1:Ru3QVMLygVs/07UQ3YDur1AQZZp2tUNje8wfloFttC0= -github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.12.1 h1:yQRC55aXN/y1W10HgwHle01DRuV9Dpf31iGkotjt3Ag= github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From 63fa7d20a417213f9008fddc8cdc471ea2bf1dbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 21:15:07 -0700 Subject: [PATCH 106/163] Bump github.com/onsi/gomega from 1.20.0 to 1.20.1 (#204) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.20.0 to 1.20.1. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.20.0...v1.20.1) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cc03e784..954e23e8 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.20.0 + github.com/onsi/gomega v1.20.1 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.0 k8s.io/api v0.24.3 diff --git a/go.sum b/go.sum index 73732508..885858fa 100644 --- a/go.sum +++ b/go.sum @@ -410,8 +410,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= -github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= From 5b0a75f459c4140b774c47dbcce905c61dd65c71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Aug 2022 00:16:06 -0700 Subject: [PATCH 107/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#209) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.13 to 1.17.14. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.13...service/sns/v1.17.14) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 16 ++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 954e23e8..a03feef9 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.11 + github.com/aws/aws-sdk-go-v2 v1.16.12 github.com/aws/aws-sdk-go-v2/config v1.17.1 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.13 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 @@ -31,13 +31,13 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 // indirect - github.com/aws/smithy-go v1.12.1 // indirect + github.com/aws/smithy-go v1.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 885858fa..131a63d1 100644 --- a/go.sum +++ b/go.sum @@ -74,30 +74,34 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.11 h1:xM1ZPSvty3xVmdxiGr7ay/wlqv+MWhH0rMlyLdbC0YQ= github.com/aws/aws-sdk-go-v2 v1.16.11/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo= +github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA= +github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= github.com/aws/aws-sdk-go-v2/config v1.17.1 h1:BWxTjokU/69BZ4DnLrZco6OvBDii6ToEdfBL/y5I1nA= github.com/aws/aws-sdk-go-v2/config v1.17.1/go.mod h1:uOxDHjBemNTF2Zos+fgG0NNfE86wn1OAHDTGxjMEYi0= github.com/aws/aws-sdk-go-v2/credentials v1.12.14 h1:AtVG/amkjbDBfnPr/tuW2IG18HGNznP6L12Dx0rLz+Q= github.com/aws/aws-sdk-go-v2/credentials v1.12.14/go.mod h1:opAndTyq+YN7IpVG57z2CeNuXSQMqTYxGGlYH0m0RMY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 h1:wgJBHO58Pc1V1QAnzdVM3JK3WbE/6eUF0JxCZ+/izz0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12/go.mod h1:aZ4vZnyUuxedC7eD4JyEHpGnCz+O2sHQEx3VvAwklSE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 h1:OmiwoVyLKEqqD5GvB683dbSqxiOfvx4U2lDZhG2Esc4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18/go.mod h1:348MLhzV1GSlZSMusdwQpXKbhD7X2gbI/TxwAPKkYZQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 h1:5mvQDtNWtI6H56+E4LUnLWEmATMB7oEh+Z9RurtIuC0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 h1:gC5mudiFrWGhzcdoWj1iCGUfrzCpQG0MQIQf0CXFFQQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19/go.mod h1:llxE6bwUZhuCas0K7qGiu5OgMis3N7kdWtFSxoHmJ7E= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12/go.mod h1:ckaCVTEdGAxO6KwTGzgskxR1xM+iJW4lxMyDFVda2Fc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 h1:qezY57na06d6kSE7uuB0N7XEflu914AXx/hg2L8Ykcw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13/go.mod h1:lB12mkZqCSo5PsdBFLNqc2M/OOYgNAy8UtaktyuWvE8= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 h1:g5qq9sgtEzt2szMaDqQO6fqKe026T6dHTFJp5NsPzkQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19/go.mod h1:cVHo8KTuHjShb9V8/VjH3S/8+xPu16qx8fdGwmotJhE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12 h1:7iPTTX4SAI2U2VOogD7/gmHlsgnYSgoNHt7MSQXtG2M= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12/go.mod h1:1TODGhheLWjpQWSuhYuAUWYTCKwEjx2iblIFKDHjeTc= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.13 h1:IGZqdrpJN0NkjgjQAdxgKEm5lRgPUI3IC4XDHTL6fGQ= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.13/go.mod h1:pARAWVRnxP2jp+WP5FiUV26br9hEODC6NBryEJaHK80= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 h1:pGdf7v3gzWs0zaocsz1Pshya2gqMIx8FUsnsN5YlWhQ= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14/go.mod h1:kJHF1cJv3Ua31h5UpHKO+9KJsrg9TTMiHCattw4KJpc= github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 h1:pXxu9u2z1UqSbjO9YA8kmFJBhFc1EVTDaf7A+S+Ivq8= github.com/aws/aws-sdk-go-v2/service/sso v1.11.17/go.mod h1:mS5xqLZc/6kc06IpXn5vRxdLaED+jEuaSRv5BxtnsiY= github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 h1:dl8T0PJlN92rvEGOEUiD0+YPYdPEaCZK0TqHukvSfII= github.com/aws/aws-sdk-go-v2/service/sts v1.16.13/go.mod h1:Ru3QVMLygVs/07UQ3YDur1AQZZp2tUNje8wfloFttC0= -github.com/aws/smithy-go v1.12.1 h1:yQRC55aXN/y1W10HgwHle01DRuV9Dpf31iGkotjt3Ag= github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0= +github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From d6769d891a7dbab47374f1a8489ad4df7ab1c6d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Aug 2022 11:24:37 -0700 Subject: [PATCH 108/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.17.1 to 1.17.2 (#208) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.17.1 to 1.17.2. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.17.1...config/v1.17.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 15 ++++++++------- go.sum | 34 ++++++++++++++++------------------ 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index a03feef9..8b594ab2 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.16.12 - github.com/aws/aws-sdk-go-v2/config v1.17.1 + github.com/aws/aws-sdk-go-v2/config v1.17.2 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -29,14 +29,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.14 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.18 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.14 // indirect github.com/aws/smithy-go v1.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 131a63d1..39f5fd5a 100644 --- a/go.sum +++ b/go.sum @@ -74,32 +74,30 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.11/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo= github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA= github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= -github.com/aws/aws-sdk-go-v2/config v1.17.1 h1:BWxTjokU/69BZ4DnLrZco6OvBDii6ToEdfBL/y5I1nA= -github.com/aws/aws-sdk-go-v2/config v1.17.1/go.mod h1:uOxDHjBemNTF2Zos+fgG0NNfE86wn1OAHDTGxjMEYi0= -github.com/aws/aws-sdk-go-v2/credentials v1.12.14 h1:AtVG/amkjbDBfnPr/tuW2IG18HGNznP6L12Dx0rLz+Q= -github.com/aws/aws-sdk-go-v2/credentials v1.12.14/go.mod h1:opAndTyq+YN7IpVG57z2CeNuXSQMqTYxGGlYH0m0RMY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 h1:wgJBHO58Pc1V1QAnzdVM3JK3WbE/6eUF0JxCZ+/izz0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12/go.mod h1:aZ4vZnyUuxedC7eD4JyEHpGnCz+O2sHQEx3VvAwklSE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18/go.mod h1:348MLhzV1GSlZSMusdwQpXKbhD7X2gbI/TxwAPKkYZQ= +github.com/aws/aws-sdk-go-v2/config v1.17.2 h1:V96WPd2a1H/MXGZjk4zto+KpYnwZI2kdIdy/cI8kYnQ= +github.com/aws/aws-sdk-go-v2/config v1.17.2/go.mod h1:jumS/AMwul4WaG8vyXsF6kUndG9zndR+yfYBwl4i9ds= +github.com/aws/aws-sdk-go-v2/credentials v1.12.15 h1:6DONxG9cR3pAuISj1Irh5u2SRqCfIJwyHNyDDes7SZw= +github.com/aws/aws-sdk-go-v2/credentials v1.12.15/go.mod h1:41zTC6U/78fUD7ZCa5NymTJANDjfqySg5YEAYVFl2Ic= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 h1:+uferi8SUDZtMloCDt24Zenyy/i71C/ua5mjUCpbpN0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13/go.mod h1:y0eXmsNBFIVjUE8ZBjES8myOHlMsXDz7qGT93+MVdjk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 h1:gC5mudiFrWGhzcdoWj1iCGUfrzCpQG0MQIQf0CXFFQQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19/go.mod h1:llxE6bwUZhuCas0K7qGiu5OgMis3N7kdWtFSxoHmJ7E= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12/go.mod h1:ckaCVTEdGAxO6KwTGzgskxR1xM+iJW4lxMyDFVda2Fc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 h1:qezY57na06d6kSE7uuB0N7XEflu914AXx/hg2L8Ykcw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13/go.mod h1:lB12mkZqCSo5PsdBFLNqc2M/OOYgNAy8UtaktyuWvE8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19 h1:g5qq9sgtEzt2szMaDqQO6fqKe026T6dHTFJp5NsPzkQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.19/go.mod h1:cVHo8KTuHjShb9V8/VjH3S/8+xPu16qx8fdGwmotJhE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12 h1:7iPTTX4SAI2U2VOogD7/gmHlsgnYSgoNHt7MSQXtG2M= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.12/go.mod h1:1TODGhheLWjpQWSuhYuAUWYTCKwEjx2iblIFKDHjeTc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 h1:GvszACAU8GSV3+Tant5GutW6smY8WavrP8ZuRS9Ku4Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20/go.mod h1:bfTcsThj5a9P5pIGRy0QudJ8k4+issxXX+O6Djnd5Cs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 h1:ObfthqDyhe7rMAOa7pqft6974VHIk8BAJB7kYdoIfTA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13/go.mod h1:V390DK4MQxLpDdXxFqizyz8KUxuWImkW/xzgXMz0yyk= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 h1:pGdf7v3gzWs0zaocsz1Pshya2gqMIx8FUsnsN5YlWhQ= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14/go.mod h1:kJHF1cJv3Ua31h5UpHKO+9KJsrg9TTMiHCattw4KJpc= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.17 h1:pXxu9u2z1UqSbjO9YA8kmFJBhFc1EVTDaf7A+S+Ivq8= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.17/go.mod h1:mS5xqLZc/6kc06IpXn5vRxdLaED+jEuaSRv5BxtnsiY= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.13 h1:dl8T0PJlN92rvEGOEUiD0+YPYdPEaCZK0TqHukvSfII= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.13/go.mod h1:Ru3QVMLygVs/07UQ3YDur1AQZZp2tUNje8wfloFttC0= -github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.18 h1:gTn1a/FbcOXK5LQS88dD5k+PKwyjVvhAEEwyN4c6eW8= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.18/go.mod h1:ytmEi5+qwcSNcV2pVA8PIb1DnKT/0Bu/K4nfJHwoM6c= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 h1:p48IfndYbRk3iDsoQAmVXdCKEM5+7Y50JAPikjwk8gI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1/go.mod h1:NY+G+8PW0ISyJ7/6t5mgOe6qpJiwZa9Jix05WPscJjg= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.14 h1:7kxso8VZLQ86Jg27QRBw6fjrQhQ8CMNMZ7SB0w7RQiA= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.14/go.mod h1:Y+BUV19q3OmQVqNUlbZ40zVi3NM6Biuxwkx/qdSD/CY= github.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0= github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From 4d3822013c2a096564c97f69fae479b81f992a4a Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Fri, 2 Sep 2022 10:49:53 -0700 Subject: [PATCH 109/163] Add support for Ready, Hostname and Nodename Endpoint attributes. And some other refactoring changes. (#210) * Add support for Ready, Hostname and Nodename Endpoint attributes. And some other refactoring changes. Changes: * Add support for Ready, Hostname and Nodename Endpoint attributes. * Refactor cluster properties changes - move to model package, simplify code. * Add controller watches for Service event. Filter out events if cluster properties is not defined. * Improve kind integration test - add curl test, use dnsutils pod for dig, use ngnix app for deployment * Fix build command test * Update the Leader resource name --- CONTRIBUTING.md | 106 ++++---- Makefile | 3 + config/manager/controller_manager_config.yaml | 2 +- integration/janitor/api.go | 5 +- integration/janitor/api_test.go | 3 +- integration/janitor/janitor.go | 20 +- integration/janitor/janitor_test.go | 12 +- ...eployment.yaml => coredns-deployment.yaml} | 4 +- .../kind-test/configs/dnsutils-pod.yaml | 14 ++ .../kind-test/configs/e2e-client-hello.yaml | 12 - .../configs/e2e-clusterset-ip-service.yaml | 16 ++ .../kind-test/configs/e2e-deployment.yaml | 12 +- integration/kind-test/configs/e2e-export.yaml | 5 - .../configs/e2e-headless-export.yaml | 5 - .../configs/e2e-headless-service.yaml | 17 ++ .../kind-test/configs/e2e-headless.yaml | 12 - .../kind-test/configs/e2e-service.yaml | 11 - integration/kind-test/scripts/DNS-test.sh | 67 ----- integration/kind-test/scripts/common.sh | 8 +- integration/kind-test/scripts/curl-test.sh | 21 ++ integration/kind-test/scripts/dns-test.sh | 61 +++++ integration/kind-test/scripts/run-helper.sh | 23 +- integration/kind-test/scripts/run-tests.sh | 33 ++- integration/kind-test/scripts/setup-kind.sh | 7 +- .../shared/scenarios/export_service.go | 14 +- main.go | 8 +- pkg/apis/about/v1alpha1/groupversion_info.go | 4 +- pkg/cloudmap/api.go | 33 +-- pkg/cloudmap/api_test.go | 20 +- pkg/cloudmap/client.go | 40 ++- pkg/cloudmap/client_test.go | 140 ++++++----- pkg/common/cluster.go | 73 ------ .../multicluster/cloudmap_controller.go | 16 +- .../multicluster/cloudmap_controller_test.go | 25 +- .../multicluster/controllers_common_test.go | 19 +- .../multicluster/endpointslice_plan.go | 4 +- .../multicluster/endpointslice_plan_test.go | 70 ++---- .../multicluster/serviceexport_controller.go | 231 ++++++++++-------- .../serviceexport_controller_test.go | 50 +--- pkg/controllers/multicluster/utils.go | 26 +- pkg/controllers/multicluster/utils_test.go | 10 +- pkg/model/cluster.go | 86 +++++++ pkg/model/cluster_test.go | 108 ++++++++ pkg/model/types.go | 81 ++++-- pkg/model/types_test.go | 110 ++++----- samples/coredns-deployment.yaml | 4 +- test/test-constants.go | 41 ++-- 47 files changed, 943 insertions(+), 749 deletions(-) rename integration/kind-test/configs/{e2e-coredns-deployment.yaml => coredns-deployment.yaml} (98%) create mode 100644 integration/kind-test/configs/dnsutils-pod.yaml delete mode 100644 integration/kind-test/configs/e2e-client-hello.yaml create mode 100644 integration/kind-test/configs/e2e-clusterset-ip-service.yaml delete mode 100644 integration/kind-test/configs/e2e-export.yaml delete mode 100644 integration/kind-test/configs/e2e-headless-export.yaml create mode 100644 integration/kind-test/configs/e2e-headless-service.yaml delete mode 100644 integration/kind-test/configs/e2e-headless.yaml delete mode 100644 integration/kind-test/configs/e2e-service.yaml delete mode 100755 integration/kind-test/scripts/DNS-test.sh create mode 100755 integration/kind-test/scripts/curl-test.sh create mode 100755 integration/kind-test/scripts/dns-test.sh delete mode 100644 pkg/common/cluster.go create mode 100644 pkg/model/cluster.go create mode 100644 pkg/model/cluster_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b974cb7..beae2a57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,30 +1,31 @@ # Contributing Guidelines -* [Architecture Overview](#architecture-overview) -* [Getting Started](#getting-started) - + [Local Setup](#local-setup) - - [Prerequisites](#prerequisites) - - [Cluster Setup](#cluster-setup) - - [Run the controller from outside the cluster](#run-the-controller-from-outside-the-cluster) - - [Build and deploy to the cluster](#build-and-deploy-to-the-cluster) - - [Run unit tests](#run-unit-tests) - - [Cleanup](#cleanup) - + [Deploying to a cluster](#deploying-to-a-cluster) -* [Integration testing](#integration-testing) -* [Build and push docker image to ECR](#build-and-push-docker-image-to-ecr) -* [Reporting Bugs/Feature Requests](#reporting-bugs-feature-requests) -* [Contributing via Pull Requests](#contributing-via-pull-requests) -* [Finding contributions to work on](#finding-contributions-to-work-on) -* [Code of Conduct](#code-of-conduct) -* [Security issue notifications](#security-issue-notifications) -* [Licensing](#licensing) - Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community. Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. + +* [Contributing Guidelines](#contributing-guidelines) + * [Architecture Overview](#architecture-overview) + * [Getting Started](#getting-started) + * [Build and Unit Tests](#build-and-unit-tests) + * [Local Setup](#local-setup) + * [Prerequisites](#prerequisites) + * [Kind Cluster Setup](#kind-cluster-setup) + * [Run the controller from outside the cluster](#run-the-controller-from-outside-the-cluster) + * [Build and deploy controller into the cluster](#build-and-deploy-controller-into-the-cluster) + * [Local integration testing](#local-integration-testing) + * [Build and push docker image](#build-and-push-docker-image) + * [Reporting Bugs/Feature Requests](#reporting-bugsfeature-requests) + * [Contributing via Pull Requests](#contributing-via-pull-requests) + * [Finding contributions to work on](#finding-contributions-to-work-on) + * [Code of Conduct](#code-of-conduct) + * [Security issue notifications](#security-issue-notifications) + * [Licensing](#licensing) + + ## Architecture Overview ![Architecture diagram](docs/architecture-overview.png?raw=true) @@ -34,6 +35,23 @@ information to effectively respond to your bug report or contribution. ## Getting Started +### Build and Unit Tests + +Use command below to run the unit test: +```sh +make test +``` + +Use command below to build: +```sh +make build +``` + +Use the command below to perform cleanup: +```sh +make clean +``` + ### Local Setup #### Prerequisites @@ -50,7 +68,7 @@ Note that this walk-through assumes throughout to operate in the `us-west-2` reg export AWS_REGION=us-west-2 ``` -#### Cluster Setup +#### Kind Cluster Setup Spin up a local Kubernetes cluster using `kind`: @@ -92,7 +110,7 @@ kubectl apply -f samples/example-clusterproperty.yaml > ⚠ **Note:** If you are creating multiple clusters, ensure you create unique `id.k8s.io` identifiers for each cluster. -To run the controller, run the following command. The controller runs in an infinite loop so open another terminal to create CRDs. +To run the controller, run the following command. The controller runs in an infinite loop so open another terminal to create CRDs. (Ctrl+C to exit) ```sh make run ``` @@ -114,7 +132,12 @@ cloudmap fetching a service {"namespaceName": "example", "servic cloudmap creating a new service {"namespace": "example", "name": "my-service"} ``` -#### Build and deploy to the cluster +Use the command below to remove the CRDs from the cluster: +```sh +make uninstall +``` + +#### Build and deploy controller into the cluster Build local `controller` docker image: ```sh @@ -136,7 +159,8 @@ kind load docker-image controller:local --name my-cluster Finally, create the controller resources in the cluster: ```sh make deploy IMG=controller:local AWS_REGION=us-west-2 -# customresourcedefinition.apiextensions.k8s.io/serviceexports.multicluster.x-k8s.io configured +# customresourcedefinition.apiextensions.k8s.io/clusterproperties.about.k8s.io created +# customresourcedefinition.apiextensions.k8s.io/serviceexports.multicluster.x-k8s.io created # customresourcedefinition.apiextensions.k8s.io/serviceimports.multicluster.x-k8s.io created # ... # deployment.apps/cloud-map-mcs-controller-manager created @@ -152,38 +176,13 @@ To remove the controller from your cluster, run: make undeploy ``` -#### Run unit tests - -Use command below to run the unit test: -```sh -make test -``` - -#### Cleanup - -Use the command below to perform cleanup: -```sh -make clean -``` - -Use the command below to remove the CRDs from the cluster: -```sh -make uninstall -``` - Use the command below to delete the cluster `my-cluster`: ```sh kind delete cluster --name my-cluster ``` -### Deploying to a cluster - -You must first push a Docker image containing the changes to a Docker repository like ECR, Github packages, or DockerHub. The repo is configured to use Github Actions to automatically publish the docker image upon push to `main` branch. The image URI will be `ghcr.io/[Your forked repo name here]` You can enable this for forked repos by enabling Github actions on your forked repo in the "Actions" tab of forked repo. - -If you are deploying to cluster using kustomize templates from the `config` directory, you will need to override the image URI away from `ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s` in order to use your own docker images. - +### Local integration testing -## Local integration testing The end-to-end integration test suite can be run locally to validate controller core functionality. This will provision a local Kind cluster and build and run the AWS Cloud Map MCS Controller for K8s. The test will verify service endpoints sync with AWS Cloud Map. If successful, the suite will then de-provision the local test cluster and delete AWS Cloud Map namespace `aws-cloud-map-mcs-e2e` along with test service and service instance resources: ```sh make kind-integration-suite @@ -194,10 +193,15 @@ If integration test suite fails for some reason, you can perform a cleanup: make kind-integration-cleanup ``` -## Build and push docker image to ECR +## Build and push docker image + +You must first push a Docker image containing the changes to a Docker repository like ECR, Github packages, or DockerHub. The repo is configured to use Github Actions to automatically publish the docker image upon push to `main` branch. The image URI will be `ghcr.io/[Your forked repo name here]` You can enable this for forked repos by enabling Github actions on your forked repo in the "Actions" tab of forked repo. + +If you are deploying to cluster using kustomize templates from the `config` directory, you will need to override the image URI away from `ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s` in order to use your own docker images. +To push the docker image into personal repo: ```sh -make docker-build docker-push IMG=.dkr.ecr..amazonaws.com/ +make docker-build docker-push IMG=[Your personal repo] ``` ## Reporting Bugs/Feature Requests diff --git a/Makefile b/Makefile index 49e41e3d..c141f0bd 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +# Silence update recommendations for Ginkgo +export ACK_GINKGO_DEPRECATIONS:=1.16.5 + GIT_COMMIT:=$(shell git describe --dirty --always) GIT_TAG:=$(shell git describe --dirty --always --tags) PKG:=github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index 8153cb17..55c709b8 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -8,4 +8,4 @@ webhook: port: 9443 leaderElection: leaderElect: true - resourceName: db692913.x-k8s.io + resourceName: aws-cloud-map-mcs-controller-for-k8s-lock diff --git a/integration/janitor/api.go b/integration/janitor/api.go index eebe6a06..6e00a400 100644 --- a/integration/janitor/api.go +++ b/integration/janitor/api.go @@ -4,7 +4,6 @@ import ( "context" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" ) @@ -20,9 +19,9 @@ type serviceDiscoveryJanitorApi struct { janitorFacade SdkJanitorFacade } -func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config, clusterUtils common.ClusterUtils) ServiceDiscoveryJanitorApi { +func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config) ServiceDiscoveryJanitorApi { return &serviceDiscoveryJanitorApi{ - ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg, clusterUtils), + ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg), janitorFacade: NewSdkJanitorFacadeFromConfig(cfg), } } diff --git a/integration/janitor/api_test.go b/integration/janitor/api_test.go index 3d7d1514..55a3b5b8 100644 --- a/integration/janitor/api_test.go +++ b/integration/janitor/api_test.go @@ -5,7 +5,6 @@ import ( "testing" janitorMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" @@ -14,7 +13,7 @@ import ( ) func TestNewServiceDiscoveryJanitorApiFromConfig(t *testing.T) { - assert.NotNil(t, NewServiceDiscoveryJanitorApiFromConfig(&aws.Config{}, common.ClusterUtils{})) + assert.NotNil(t, NewServiceDiscoveryJanitorApiFromConfig(&aws.Config{})) } func TestServiceDiscoveryJanitorApi_DeleteNamespace_HappyCase(t *testing.T) { diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index b2379256..73c4b937 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -6,7 +6,7 @@ import ( "os" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" ) @@ -18,8 +18,10 @@ type CloudMapJanitor interface { } type cloudMapJanitor struct { - sdApi ServiceDiscoveryJanitorApi - fail func() + clusterId string + clusterSetId string + sdApi ServiceDiscoveryJanitorApi + fail func() } // NewDefaultJanitor returns a new janitor object. @@ -32,8 +34,10 @@ func NewDefaultJanitor(clusterId string, clusterSetId string) CloudMapJanitor { } return &cloudMapJanitor{ - sdApi: NewServiceDiscoveryJanitorApiFromConfig(&awsCfg, common.NewClusterUtilsForTest(clusterId, clusterSetId)), - fail: func() { os.Exit(1) }, + clusterId: clusterId, + clusterSetId: clusterSetId, + sdApi: NewServiceDiscoveryJanitorApiFromConfig(&awsCfg), + fail: func() { os.Exit(1) }, } } @@ -73,7 +77,11 @@ func (j *cloudMapJanitor) Cleanup(ctx context.Context, nsName string) { } func (j *cloudMapJanitor) deregisterInstances(ctx context.Context, nsName string, svcName string, svcId string) { - insts, err := j.sdApi.DiscoverInstances(ctx, nsName, svcName) + queryParameters := map[string]string{ + model.ClusterSetIdAttr: j.clusterSetId, + } + + insts, err := j.sdApi.DiscoverInstances(ctx, nsName, svcName, &queryParameters) j.checkOrFail(err, fmt.Sprintf("service has %d instances to clean", len(insts)), "could not list instances to cleanup") diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index bdd36dfe..cd43bd6e 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -21,7 +21,7 @@ type testJanitor struct { } func TestNewDefaultJanitor(t *testing.T) { - assert.NotNil(t, NewDefaultJanitor(test.ClusterId1, test.ClusterSetId1)) + assert.NotNil(t, NewDefaultJanitor(test.ClusterId1, test.ClusterSet)) } func TestCleanupHappyCase(t *testing.T) { @@ -32,7 +32,9 @@ func TestCleanupHappyCase(t *testing.T) { Return(map[string]*model.Namespace{test.HttpNsName: test.GetTestHttpNamespace()}, nil) tj.mockApi.EXPECT().GetServiceIdMap(context.TODO(), test.HttpNsId). Return(map[string]string{test.SvcName: test.SvcId}, nil) - tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). + tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + model.ClusterSetIdAttr: test.ClusterSet, + }). Return([]types.HttpInstanceSummary{{InstanceId: aws.String(test.EndptId1)}}, nil) tj.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1). @@ -67,8 +69,10 @@ func getTestJanitor(t *testing.T) *testJanitor { failed := false return &testJanitor{ janitor: &cloudMapJanitor{ - sdApi: api, - fail: func() { failed = true }, + clusterId: test.ClusterId1, + clusterSetId: test.ClusterSet, + sdApi: api, + fail: func() { failed = true }, }, mockApi: api, failed: &failed, diff --git a/integration/kind-test/configs/e2e-coredns-deployment.yaml b/integration/kind-test/configs/coredns-deployment.yaml similarity index 98% rename from integration/kind-test/configs/e2e-coredns-deployment.yaml rename to integration/kind-test/configs/coredns-deployment.yaml index 00606eba..86b6e0f3 100644 --- a/integration/kind-test/configs/e2e-coredns-deployment.yaml +++ b/integration/kind-test/configs/coredns-deployment.yaml @@ -58,7 +58,7 @@ spec: - args: - -conf - /etc/coredns/Corefile - image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.4 + image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.6 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -134,4 +134,4 @@ spec: - key: Corefile path: Corefile name: coredns - name: config-volume \ No newline at end of file + name: config-volume diff --git a/integration/kind-test/configs/dnsutils-pod.yaml b/integration/kind-test/configs/dnsutils-pod.yaml new file mode 100644 index 00000000..9d408d78 --- /dev/null +++ b/integration/kind-test/configs/dnsutils-pod.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dnsutils + namespace: default +spec: + containers: + - command: + - sleep + - "3600" + image: registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3 + name: dnsutils + imagePullPolicy: IfNotPresent + restartPolicy: Always diff --git a/integration/kind-test/configs/e2e-client-hello.yaml b/integration/kind-test/configs/e2e-client-hello.yaml deleted file mode 100644 index 969ebd54..00000000 --- a/integration/kind-test/configs/e2e-client-hello.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: client-hello - namespace: aws-cloud-map-mcs-e2e -spec: - containers: - - command: - - sleep - - "1d" - image: alpine - name: client-hello \ No newline at end of file diff --git a/integration/kind-test/configs/e2e-clusterset-ip-service.yaml b/integration/kind-test/configs/e2e-clusterset-ip-service.yaml new file mode 100644 index 00000000..23a1c804 --- /dev/null +++ b/integration/kind-test/configs/e2e-clusterset-ip-service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-clusterset-ip-service +spec: + selector: + app: nginx-hello + ports: + - port: 80 +--- +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-clusterset-ip-service diff --git a/integration/kind-test/configs/e2e-deployment.yaml b/integration/kind-test/configs/e2e-deployment.yaml index d9eab694..0a04d4a5 100644 --- a/integration/kind-test/configs/e2e-deployment.yaml +++ b/integration/kind-test/configs/e2e-deployment.yaml @@ -2,21 +2,21 @@ apiVersion: apps/v1 kind: Deployment metadata: namespace: aws-cloud-map-mcs-e2e - name: coredns-deployment + name: nginx-hello-deployment labels: - app: coredns + app: nginx-hello spec: replicas: 5 selector: matchLabels: - app: coredns + app: nginx-hello template: metadata: labels: - app: coredns + app: nginx-hello spec: containers: - - name: coredns - image: k8s.gcr.io/coredns:1.7.0 + - name: nginx-hello + image: nginxdemos/hello:0.3-plain-text ports: - containerPort: 80 diff --git a/integration/kind-test/configs/e2e-export.yaml b/integration/kind-test/configs/e2e-export.yaml deleted file mode 100644 index a23e9f4c..00000000 --- a/integration/kind-test/configs/e2e-export.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: ServiceExport -apiVersion: multicluster.x-k8s.io/v1alpha1 -metadata: - namespace: aws-cloud-map-mcs-e2e - name: e2e-service diff --git a/integration/kind-test/configs/e2e-headless-export.yaml b/integration/kind-test/configs/e2e-headless-export.yaml deleted file mode 100644 index 1129d5a1..00000000 --- a/integration/kind-test/configs/e2e-headless-export.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: ServiceExport -apiVersion: multicluster.x-k8s.io/v1alpha1 -metadata: - namespace: aws-cloud-map-mcs-e2e - name: e2e-headless diff --git a/integration/kind-test/configs/e2e-headless-service.yaml b/integration/kind-test/configs/e2e-headless-service.yaml new file mode 100644 index 00000000..78f5b84f --- /dev/null +++ b/integration/kind-test/configs/e2e-headless-service.yaml @@ -0,0 +1,17 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-headless-service +spec: + clusterIP: None + selector: + app: nginx-hello + ports: + - port: 80 +--- +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: aws-cloud-map-mcs-e2e + name: e2e-headless-service diff --git a/integration/kind-test/configs/e2e-headless.yaml b/integration/kind-test/configs/e2e-headless.yaml deleted file mode 100644 index f40a693f..00000000 --- a/integration/kind-test/configs/e2e-headless.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - namespace: aws-cloud-map-mcs-e2e - name: e2e-headless -spec: - clusterIP: None - selector: - app: coredns - ports: - - port: 8080 - targetPort: 80 diff --git a/integration/kind-test/configs/e2e-service.yaml b/integration/kind-test/configs/e2e-service.yaml deleted file mode 100644 index 83669588..00000000 --- a/integration/kind-test/configs/e2e-service.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - namespace: aws-cloud-map-mcs-e2e - name: e2e-service -spec: - selector: - app: coredns - ports: - - port: 8080 - targetPort: 80 diff --git a/integration/kind-test/scripts/DNS-test.sh b/integration/kind-test/scripts/DNS-test.sh deleted file mode 100755 index 8b70e408..00000000 --- a/integration/kind-test/scripts/DNS-test.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -# Testing service consumption with dnsutils pod - -echo "verifying single-cluster service consumption..." - -# Helper function to verify DNS results -checkDNS() { - endpt_count=$(echo "$1" | wc -l | xargs) - - if [ "$2" = "Headless" ]; then - if [ "$endpt_count" -ne "$3" ]; then - echo "ERROR: Found $endpt_count endpoints, expected $3 endpoints" - exit 1 - fi - fi - - if [ "$2" = "ClusterSetIP" ]; then - if [ "$endpt_count" -ne 1 ]; then - echo "ERROR: Found $endpt_count endpoints, expected 1 endpoint" - exit 1 - fi - fi -} - -# Add pod -$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-client-hello.yaml" -$KUBECTL_BIN wait --for=condition=ready pod/$DNS_POD -n $NAMESPACE # wait until pod is deployed - -# Install dig if not installed -$KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig -v &>/dev/null -exit_code=$? -if [ "$exit_code" -ne 0 ]; then - echo "dig not installed, installing..." - $KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- apk add --update bind-tools -fi - -# Perform a dig to cluster-local CoreDNS -# TODO: parse dig outputs for more precise verification - check specifics IPs? -echo "performing dig for A/AAAA records..." -addresses=$($KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) -exit_code=$? -echo "$addresses" - -if [ "$exit_code" -ne 0 ]; then - echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local" - exit $exit_code -fi - -# verify DNS results -checkDNS "$addresses" "$SERVICE_TYPE" "$1" - -echo "performing dig for SRV records..." -addresses=$($KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local. SRV +short) -exit_code=$? -echo "$addresses" - -if [ "$exit_code" -ne 0 ]; then - echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local" - exit $exit_code -fi - -# verify DNS results -checkDNS "$addresses" "$SERVICE_TYPE" "$1" - -echo "confirmed service consumption" -exit 0 diff --git a/integration/kind-test/scripts/common.sh b/integration/kind-test/scripts/common.sh index 1c5ae5a9..fd079b5e 100755 --- a/integration/kind-test/scripts/common.sh +++ b/integration/kind-test/scripts/common.sh @@ -8,14 +8,14 @@ export SHARED_CONFIGS='./integration/shared/configs' export SCENARIOS='./integration/shared/scenarios' export NAMESPACE='aws-cloud-map-mcs-e2e' export ENDPT_PORT=80 -export SERVICE_PORT=8080 -export CLUSTERIP_SERVICE='e2e-service' -export HEADLESS_SERVICE='e2e-headless' +export SERVICE_PORT=80 +export CLUSTERIP_SERVICE='e2e-clusterset-ip-service' +export HEADLESS_SERVICE='e2e-headless-service' export KIND_SHORT='cloud-map-e2e' export CLUSTER='kind-cloud-map-e2e' export CLUSTERID1='kind-e2e-clusterid-1' export CLUSTERSETID1='kind-e2e-clustersetid-1' -export DNS_POD='client-hello' +export DNS_POD='dnsutils' export IMAGE='kindest/node:v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207' export EXPECTED_ENDPOINT_COUNT=5 export UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/kind-test/scripts/curl-test.sh b/integration/kind-test/scripts/curl-test.sh new file mode 100755 index 00000000..a96ba56e --- /dev/null +++ b/integration/kind-test/scripts/curl-test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Testing service consumption with dnsutils pod + +deployment=$1 + +echo "performing curl to $SERVICE.$NAMESPACE.svc.clusterset.local" +http_code=$($KUBECTL_BIN exec deployment/$deployment --namespace "$NAMESPACE" -- curl -s -o /dev/null -w "%{http_code}" $SERVICE.$NAMESPACE.svc.clusterset.local) +exit_code=$? + +if [ "$exit_code" -ne 0 ]; then + echo "ERROR: Unable to curl $SERVICE.$NAMESPACE.svc.clusterset.local" + exit $exit_code +fi + +if [ "$http_code" -ne "200" ]; then + echo "ERROR: curl $SERVICE.$NAMESPACE.svc.clusterset.local failed with $http_code" + exit 1 +fi + +exit 0 diff --git a/integration/kind-test/scripts/dns-test.sh b/integration/kind-test/scripts/dns-test.sh new file mode 100755 index 00000000..eeed7946 --- /dev/null +++ b/integration/kind-test/scripts/dns-test.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Helper function to verify DNS results +checkDNS() { + dns_addresses_count=$(echo "$1" | wc -l | xargs) + + if [ "$SERVICE_TYPE" = "Headless" ]; then + if [ "$dns_addresses_count" -ne "$expected_endpoint_count" ]; then + echo "ERROR: Found $dns_addresses_count endpoints, expected $expected_endpoint_count endpoints" + exit 1 + fi + fi + + if [ "$SERVICE_TYPE" = "ClusterSetIP" ]; then + if [ "$dns_addresses_count" -ne 1 ]; then + echo "ERROR: Found $dns_addresses_count endpoints, expected 1 endpoint" + exit 1 + fi + fi +} + +# Testing service consumption with dnsutils pod + +echo "verifying dns resolution..." + +expected_endpoint_count=$1 + +# Install dnsutils pod +$KUBECTL_BIN apply -f "$KIND_CONFIGS/dnsutils-pod.yaml" +$KUBECTL_BIN wait --for=condition=ready pod/$DNS_POD # wait until pod is deployed + +# Perform a dig to cluster-local CoreDNS +# TODO: parse dig outputs for more precise verification - check specifics IPs? +echo "performing dig for A/AAAA records..." +addresses=$($KUBECTL_BIN exec $DNS_POD -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) +exit_code=$? +echo "$addresses" + +if [ "$exit_code" -ne 0 ]; then + echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local" + exit $exit_code +fi + +# verify DNS results +checkDNS "$addresses" + +echo "performing dig for SRV records..." +addresses=$($KUBECTL_BIN exec $DNS_POD -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local. SRV +short) +exit_code=$? +echo "$addresses" + +if [ "$exit_code" -ne 0 ]; then + echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local" + exit $exit_code +fi + +# verify DNS results +checkDNS "$addresses" + +echo "confirmed dns resolution" +exit 0 diff --git a/integration/kind-test/scripts/run-helper.sh b/integration/kind-test/scripts/run-helper.sh index 02411c1d..9267654b 100755 --- a/integration/kind-test/scripts/run-helper.sh +++ b/integration/kind-test/scripts/run-helper.sh @@ -4,18 +4,23 @@ source ./integration/kind-test/scripts/common.sh +# create test namespace +$KUBECTL_BIN create namespace "$NAMESPACE" + # ClusterIP service test -$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-service.yaml" -$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-export.yaml" ./integration/kind-test/scripts/run-tests.sh "$CLUSTERIP_SERVICE" "ClusterSetIP" exit_code=$? +if [ "$exit_code" -ne 0 ] ; then + echo "ERROR: Testing $CLUSTERIP_SERVICE failed" + exit $exit_code +fi + +sleep 5 # Headless service test -if [ "$exit_code" -eq 0 ] ; then - $KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-headless.yaml" - $KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-headless-export.yaml" - ./integration/kind-test/scripts/run-tests.sh "$HEADLESS_SERVICE" "Headless" - exit_code=$? +./integration/kind-test/scripts/run-tests.sh "$HEADLESS_SERVICE" "Headless" +exit_code=$? +if [ "$exit_code" -ne 0 ] ; then + echo "ERROR: Testing $HEADLESS_SERVICE failed" + exit $exit_code fi - -exit $exit_code \ No newline at end of file diff --git a/integration/kind-test/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh index 6ec5e482..99eccd59 100755 --- a/integration/kind-test/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -6,7 +6,14 @@ source ./integration/kind-test/scripts/common.sh export SERVICE=$1 export SERVICE_TYPE=$2 -echo "testing service: $SERVICE" +# Deploy pods +$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-deployment.yaml" +# Get deployment +deployment=$($KUBECTL_BIN get deployment --namespace "$NAMESPACE" -o json | jq -r '.items[0].metadata.name') + +printf "\n***Testing Service: $SERVICE***\n" + +$KUBECTL_BIN apply -f "$KIND_CONFIGS/$SERVICE.yaml" if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT_COUNT") ; then exit $? @@ -17,7 +24,7 @@ mkdir -p "$LOGS" CTL_PID=$! echo "controller PID:$CTL_PID" -go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$endpts" +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$endpts" exit_code=$? if [ "$exit_code" -eq 0 ] ; then @@ -26,7 +33,12 @@ if [ "$exit_code" -eq 0 ] ; then fi if [ "$exit_code" -eq 0 ] ; then - ./integration/kind-test/scripts/DNS-test.sh "$EXPECTED_ENDPOINT_COUNT" + ./integration/kind-test/scripts/dns-test.sh "$EXPECTED_ENDPOINT_COUNT" + exit_code=$? +fi + +if [ "$exit_code" -eq 0 ] ; then + ./integration/kind-test/scripts/curl-test.sh "$deployment" exit_code=$? fi @@ -34,8 +46,6 @@ echo "sleeping..." sleep 2 if [ "$exit_code" -eq 0 ] ; then - deployment=$($KUBECTL_BIN get deployment --namespace "$NAMESPACE" -o json | jq -r '.items[0].metadata.name') - echo "scaling the deployment $deployment to $UPDATED_ENDPOINT_COUNT" $KUBECTL_BIN scale deployment/"$deployment" --replicas="$UPDATED_ENDPOINT_COUNT" --namespace "$NAMESPACE" exit_code=$? @@ -58,15 +68,18 @@ if [ "$exit_code" -eq 0 ] ; then fi if [ "$exit_code" -eq 0 ] ; then - ./integration/kind-test/scripts/DNS-test.sh "$UPDATED_ENDPOINT_COUNT" + ./integration/kind-test/scripts/dns-test.sh "$UPDATED_ENDPOINT_COUNT" exit_code=$? fi -# Scale deployment back down for future test and delete service export +echo "Test Successful. Cleaning up..." + +# Remove the deployment and delete service (should also delete ServiceExport) if [ "$exit_code" -eq 0 ] ; then - $KUBECTL_BIN scale deployment/"$deployment" --replicas="$EXPECTED_ENDPOINT_COUNT" --namespace "$NAMESPACE" - $KUBECTL_BIN delete ServiceExport $SERVICE -n $NAMESPACE - sleep 5 + $KUBECTL_BIN delete -f "$KIND_CONFIGS/e2e-deployment.yaml" + $KUBECTL_BIN delete Service $SERVICE -n $NAMESPACE + # TODO: verify service export is not found + # TODO: verify cloudmap resources are cleaned up fi echo "killing controller PID:$CTL_PID" diff --git a/integration/kind-test/scripts/setup-kind.sh b/integration/kind-test/scripts/setup-kind.sh index b04361f2..88ba6976 100755 --- a/integration/kind-test/scripts/setup-kind.sh +++ b/integration/kind-test/scripts/setup-kind.sh @@ -16,12 +16,7 @@ make install # Install CoreDNS plugin $KUBECTL_BIN apply -f "$SHARED_CONFIGS/coredns-clusterrole.yaml" $KUBECTL_BIN apply -f "$SHARED_CONFIGS/coredns-configmap.yaml" -$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-coredns-deployment.yaml" - -$KUBECTL_BIN create namespace "$NAMESPACE" +$KUBECTL_BIN apply -f "$KIND_CONFIGS/coredns-deployment.yaml" # Add ClusterId and ClusterSetId $KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-clusterproperty.yaml" - -# Deploy pods -$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-deployment.yaml" diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index 28a432e6..c1c60871 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -8,8 +8,6 @@ import ( "time" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" - multiclustercontrollers "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers/multicluster" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" @@ -58,6 +56,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, cl TargetPort: portStr, Protocol: string(v1.ProtocolTCP), }, + Ready: true, EndpointPort: endpointPort, ClusterId: clusterId, ClusterSetId: clusterSetId, @@ -72,7 +71,7 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, cl NsTTL: time.Second, SvcTTL: time.Second, EndptTTL: time.Second, - }, common.NewClusterUtilsForTest(clusterId, clusterSetId)), + }, model.NewClusterUtilsWithValues(clusterId, clusterSetId)), expectedSvc: model.Service{ Namespace: nsName, Name: svcName, @@ -111,9 +110,12 @@ func (e *exportServiceScenario) compareEndpoints(cmEndpoints []*model.Endpoint) match := false for _, actual := range cmEndpoints { // Ignore K8S instance attribute for the purpose of this test. - delete(actual.Attributes, multiclustercontrollers.K8sVersionAttr) - // Ignore SvcExportCreationTimestamp attribute for the purpose of this test by setting value to 0. - actual.SvcExportCreationTimestamp = 0 + delete(actual.Attributes, model.K8sVersionAttr) + // Ignore ServiceExportCreationTimestamp attribute for the purpose of this test by setting value to 0. + actual.ServiceExportCreationTimestamp = 0 + // Ignore Nodename and Hostname, as they can be platform dependent + actual.Nodename = "" + actual.Hostname = "" if expected.Equals(actual) { match = true break diff --git a/main.go b/main.go index 7f30de81..2eebe4d4 100644 --- a/main.go +++ b/main.go @@ -5,9 +5,9 @@ import ( "flag" "os" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" "github.com/aws/aws-sdk-go-v2/config" @@ -70,7 +70,7 @@ func main() { Port: 9443, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, - LeaderElectionID: "db692913.x-k8s.io", + LeaderElectionID: "aws-cloud-map-mcs-controller-for-k8s-lock", }) if err != nil { log.Error(err, "unable to start manager") @@ -87,7 +87,7 @@ func main() { log.Info("Running with AWS region", "AWS_REGION", awsCfg.Region) - clusterUtils := common.NewClusterUtils(mgr.GetClient()) + clusterUtils := model.NewClusterUtils(mgr.GetClient()) serviceDiscoveryClient := cloudmap.NewDefaultServiceDiscoveryClient(&awsCfg, clusterUtils) if err = (&multiclustercontrollers.ServiceExportReconciler{ diff --git a/pkg/apis/about/v1alpha1/groupversion_info.go b/pkg/apis/about/v1alpha1/groupversion_info.go index 82f56067..0b2f6df2 100644 --- a/pkg/apis/about/v1alpha1/groupversion_info.go +++ b/pkg/apis/about/v1alpha1/groupversion_info.go @@ -1,6 +1,6 @@ // Package v1alpha1 contains API Schema definitions for the about v1alpha1 API group -//+kubebuilder:object:generate=true -//+groupName=about.k8s.io +// +kubebuilder:object:generate=true +// +groupName=about.k8s.io package v1alpha1 import ( diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index ad33af9e..977aa2bb 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -27,7 +27,7 @@ type ServiceDiscoveryApi interface { GetServiceIdMap(ctx context.Context, namespaceId string) (serviceIdMap map[string]string, err error) // DiscoverInstances returns a list of service instances registered to a given service. - DiscoverInstances(ctx context.Context, nsName string, svcName string) (insts []types.HttpInstanceSummary, err error) + DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters *map[string]string) (insts []types.HttpInstanceSummary, err error) // ListOperations returns a map of operations to their status matching a list of filters. ListOperations(ctx context.Context, opFilters []types.OperationFilter) (operationStatusMap map[string]types.OperationStatus, err error) @@ -52,17 +52,15 @@ type ServiceDiscoveryApi interface { } type serviceDiscoveryApi struct { - log common.Logger - awsFacade AwsFacade - clusterUtils common.ClusterUtils + log common.Logger + awsFacade AwsFacade } // NewServiceDiscoveryApiFromConfig creates a new AWS Cloud Map API connection manager from an AWS client config. -func NewServiceDiscoveryApiFromConfig(cfg *aws.Config, clusterUtils common.ClusterUtils) ServiceDiscoveryApi { +func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { return &serviceDiscoveryApi{ - log: common.NewLogger("cloudmap"), - awsFacade: NewAwsFacadeFromConfig(cfg), - clusterUtils: clusterUtils, + log: common.NewLogger("cloudmap"), + awsFacade: NewAwsFacadeFromConfig(cfg), } } @@ -115,22 +113,17 @@ func (sdApi *serviceDiscoveryApi) GetServiceIdMap(ctx context.Context, nsId stri return serviceIdMap, nil } -func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string) (insts []types.HttpInstanceSummary, err error) { - clusterSetId, err := sdApi.clusterUtils.GetClusterSetId(ctx) - if err != nil { - sdApi.log.Error(err, "failed to retrieve clusterSetId") - return nil, err - } - - out, err := sdApi.awsFacade.DiscoverInstances(ctx, &sd.DiscoverInstancesInput{ +func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters *map[string]string) (insts []types.HttpInstanceSummary, err error) { + input := &sd.DiscoverInstancesInput{ NamespaceName: aws.String(nsName), ServiceName: aws.String(svcName), HealthStatus: types.HealthStatusFilterAll, MaxResults: aws.Int32(1000), - QueryParameters: map[string]string{ - model.ClusterSetIdAttr: clusterSetId, - }, - }) + } + if queryParameters != nil { + input.QueryParameters = *queryParameters + } + out, err := sdApi.awsFacade.DiscoverInstances(ctx, input) if err != nil { return insts, err diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 3de3e140..a898374d 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -19,12 +19,16 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestNewServiceDiscoveryApi(t *testing.T) { - sdc := NewServiceDiscoveryApiFromConfig(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId1, test.ClusterSetId1)) - assert.NotNil(t, sdc) + mockController := gomock.NewController(t) + defer mockController.Finish() + + awsFacade := cloudmapMock.NewMockAwsFacade(mockController) + + sdApi := getServiceDiscoveryApi(t, awsFacade) + assert.NotNil(t, sdApi) } func TestServiceDiscoveryApi_GetNamespaceMap_HappyCase(t *testing.T) { @@ -107,7 +111,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { HealthStatus: types.HealthStatusFilterAll, MaxResults: aws.Int32(1000), QueryParameters: map[string]string{ - model.ClusterSetIdAttr: test.ClusterSetId1, + model.ClusterSetIdAttr: test.ClusterSet, }, }). Return(&sd.DiscoverInstancesOutput{ @@ -117,7 +121,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { }, }, nil) - insts, err := sdApi.DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName) + insts, err := sdApi.DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{model.ClusterSetIdAttr: test.ClusterSet}) assert.Nil(t, err, "No error for happy case") assert.True(t, len(insts) == 2) assert.Equal(t, test.EndptId1, *insts[0].InstanceId) @@ -331,10 +335,8 @@ func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmapMock.MockAwsFacade) ServiceDiscoveryApi { scheme := runtime.NewScheme() scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) - fakeClient := fake.NewClientBuilder().WithObjects(test.ClusterIdForTest(), test.ClusterSetIdForTest()).WithScheme(scheme).Build() return &serviceDiscoveryApi{ - log: common.NewLoggerWithLogr(testr.New(t)), - awsFacade: awsFacade, - clusterUtils: common.NewClusterUtils(fakeClient), + log: common.NewLoggerWithLogr(testr.New(t)), + awsFacade: awsFacade, } } diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index d770e251..19ed6ba2 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -33,24 +33,24 @@ type serviceDiscoveryClient struct { log common.Logger sdApi ServiceDiscoveryApi cache ServiceDiscoveryClientCache - clusterUtils common.ClusterUtils + clusterUtils model.ClusterUtils } // NewDefaultServiceDiscoveryClient creates a new service discovery client for AWS Cloud Map with default resource cache // from a given AWS client config. -func NewDefaultServiceDiscoveryClient(cfg *aws.Config, clusterUtils common.ClusterUtils) ServiceDiscoveryClient { +func NewDefaultServiceDiscoveryClient(cfg *aws.Config, clusterUtils model.ClusterUtils) ServiceDiscoveryClient { return &serviceDiscoveryClient{ log: common.NewLogger("cloudmap"), - sdApi: NewServiceDiscoveryApiFromConfig(cfg, clusterUtils), + sdApi: NewServiceDiscoveryApiFromConfig(cfg), cache: NewDefaultServiceDiscoveryClientCache(), clusterUtils: clusterUtils, } } -func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCacheConfig, clusterUtils common.ClusterUtils) ServiceDiscoveryClient { +func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCacheConfig, clusterUtils model.ClusterUtils) ServiceDiscoveryClient { return &serviceDiscoveryClient{ log: common.NewLogger("cloudmap"), - sdApi: NewServiceDiscoveryApiFromConfig(cfg, clusterUtils), + sdApi: NewServiceDiscoveryApiFromConfig(cfg), cache: NewServiceDiscoveryClientCache(cacheConfig), clusterUtils: clusterUtils, } @@ -188,8 +188,7 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s return nil } - sdc.log.Info("deleting endpoints", "namespaceName", nsName, - "serviceName", svcName, "endpoints", endpts) + sdc.log.Info("deleting endpoints", "namespaceName", nsName, "serviceName", svcName, "endpoints", endpts) svcIdMap, err := sdc.getServiceIds(ctx, nsName) if err != nil { @@ -204,22 +203,6 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s for _, endpt := range endpts { endptId := endpt.Id - - // only delete endpoint if its clusterid & clustersetid is the same as this cluster - clusterId, clusterIdErr := sdc.clusterUtils.GetClusterId(ctx) - if clusterIdErr != nil { - return clusterIdErr - } - clusterSetId, clusterSetIdErr := sdc.clusterUtils.GetClusterSetId(ctx) - if clusterSetIdErr != nil { - return clusterSetIdErr - } - - if endpt.ClusterId != clusterId || endpt.ClusterSetId != clusterSetId { - sdc.log.Debug("skipping endpoint deletion as different clusterid", "serviceName", svcName, "endpointId", endptId, "clusterId", endpt.ClusterId) - continue - } - // add operation to delete endpoint opCollector.Add(func() (opId string, err error) { return sdc.sdApi.DeregisterInstance(ctx, svcId, endptId) @@ -247,7 +230,16 @@ func (sdc *serviceDiscoveryClient) getEndpoints(ctx context.Context, nsName stri return endpts, nil } - insts, err := sdc.sdApi.DiscoverInstances(ctx, nsName, svcName) + clusterProperties, err := sdc.clusterUtils.GetClusterProperties(ctx) + if err != nil { + sdc.log.Error(err, "failed to retrieve clusterSetId") + return nil, err + } + + queryParameters := map[string]string{ + model.ClusterSetIdAttr: clusterProperties.ClusterSetId(), + } + insts, err := sdc.sdApi.DiscoverInstances(ctx, nsName, svcName, &queryParameters) if err != nil { return nil, err } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 94edc212..7c2f8fc8 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -5,7 +5,11 @@ import ( "strconv" "testing" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" @@ -25,8 +29,9 @@ type testSdClient struct { } func TestNewServiceDiscoveryClient(t *testing.T) { - sdc := NewDefaultServiceDiscoveryClient(&aws.Config{}, common.NewClusterUtilsForTest(test.ClusterId1, test.ClusterSetId1)) - assert.NotNil(t, sdc) + tc := getTestSdClient(t) + defer tc.close() + assert.NotNil(t, tc) } func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { @@ -43,8 +48,10 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { tc.mockCache.EXPECT().CacheServiceIdMap(test.HttpNsName, getServiceIdMapForTest()) tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). - Return(getHttpInstanceSummaryForTest(), nil) + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + model.ClusterSetIdAttr: test.ClusterSet, + }).Return(getHttpInstanceSummaryForTest(), nil) + tc.mockCache.EXPECT().CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) @@ -110,7 +117,9 @@ func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { endptErr := errors.New("error listing endpoints") tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + model.ClusterSetIdAttr: test.ClusterSet, + }). Return([]types.HttpInstanceSummary{}, endptErr) svcs, err := tc.client.ListServices(context.TODO(), test.HttpNsName) @@ -255,7 +264,9 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { tc.mockCache.EXPECT().CacheServiceIdMap(test.HttpNsName, getServiceIdMapForTest()) tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return([]*model.Endpoint{}, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName). + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + model.ClusterSetIdAttr: test.ClusterSet, + }). Return(getHttpInstanceSummaryForTest(), nil) tc.mockCache.EXPECT().CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) @@ -284,32 +295,38 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) attrs1 := map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp1, - model.EndpointPortAttr: test.PortStr1, - model.EndpointPortNameAttr: test.PortName1, - model.EndpointProtocolAttr: test.Protocol1, - model.ServicePortNameAttr: test.PortName1, - model.ServicePortAttr: test.ServicePortStr1, - model.ServiceProtocolAttr: test.Protocol1, - model.ServiceTargetPortAttr: test.PortStr1, - model.ServiceTypeAttr: test.SvcType, - model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSet, + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.EndpointReadyAttr: test.EndptReadyTrue, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, + model.ServiceTypeAttr: test.SvcType, + model.EndpointHostnameAttr: test.Hostname, + model.EndpointNodeNameAttr: test.Nodename, + model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), } attrs2 := map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp2, - model.EndpointPortAttr: test.PortStr2, - model.EndpointPortNameAttr: test.PortName2, - model.EndpointProtocolAttr: test.Protocol2, - model.ServicePortNameAttr: test.PortName2, - model.ServicePortAttr: test.ServicePortStr2, - model.ServiceProtocolAttr: test.Protocol2, - model.ServiceTargetPortAttr: test.PortStr2, - model.ServiceTypeAttr: test.SvcType, - model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSet, + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.EndpointReadyAttr: test.EndptReadyTrue, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, + model.ServiceTypeAttr: test.SvcType, + model.EndpointHostnameAttr: test.Hostname, + model.EndpointNodeNameAttr: test.Nodename, + model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), } tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). @@ -346,8 +363,8 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { err := tc.client.DeleteEndpoints(context.TODO(), test.HttpNsName, test.SvcName, []*model.Endpoint{ - {Id: test.EndptId1, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSetId1}, - {Id: test.EndptId2, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSetId1}, + {Id: test.EndptId1, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSet}, + {Id: test.EndptId2, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSet}, }) assert.Nil(t, err) } @@ -356,12 +373,15 @@ func getTestSdClient(t *testing.T) *testSdClient { mockController := gomock.NewController(t) mockCache := cloudmapMock.NewMockServiceDiscoveryClientCache(mockController) mockApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) + scheme := runtime.NewScheme() + scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}, &aboutv1alpha1.ClusterPropertyList{}) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() return &testSdClient{ client: &serviceDiscoveryClient{ log: common.NewLoggerWithLogr(testr.New(t)), sdApi: mockApi, cache: mockCache, - clusterUtils: common.NewClusterUtilsForTest(test.ClusterId1, test.ClusterSetId1), + clusterUtils: model.NewClusterUtils(fakeClient), }, mockApi: *mockApi, mockCache: *mockCache, @@ -374,35 +394,41 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { { InstanceId: aws.String(test.EndptId1), Attributes: map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp1, - model.EndpointPortAttr: test.PortStr1, - model.EndpointPortNameAttr: test.PortName1, - model.EndpointProtocolAttr: test.Protocol1, - model.ServicePortNameAttr: test.PortName1, - model.ServicePortAttr: test.ServicePortStr1, - model.ServiceProtocolAttr: test.Protocol1, - model.ServiceTargetPortAttr: test.PortStr1, - model.ServiceTypeAttr: test.SvcType, - model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSet, + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.EndpointReadyAttr: test.EndptReadyTrue, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, + model.ServiceTypeAttr: test.SvcType, + model.EndpointHostnameAttr: test.Hostname, + model.EndpointNodeNameAttr: test.Nodename, + model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), }, }, { InstanceId: aws.String(test.EndptId2), Attributes: map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSetId1, - model.EndpointIpv4Attr: test.EndptIp2, - model.EndpointPortAttr: test.PortStr2, - model.EndpointPortNameAttr: test.PortName2, - model.EndpointProtocolAttr: test.Protocol2, - model.ServicePortNameAttr: test.PortName2, - model.ServicePortAttr: test.ServicePortStr2, - model.ServiceProtocolAttr: test.Protocol2, - model.ServiceTargetPortAttr: test.PortStr2, - model.ServiceTypeAttr: test.SvcType, - model.SvcExportCreationTimestampAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSet, + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.EndpointReadyAttr: test.EndptReadyTrue, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, + model.ServiceTypeAttr: test.SvcType, + model.EndpointHostnameAttr: test.Hostname, + model.EndpointNodeNameAttr: test.Nodename, + model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), }, }, } diff --git a/pkg/common/cluster.go b/pkg/common/cluster.go deleted file mode 100644 index 75a977e2..00000000 --- a/pkg/common/cluster.go +++ /dev/null @@ -1,73 +0,0 @@ -package common - -import ( - "context" - "fmt" - - aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - ClusterIdName = "id.k8s.io" - ClusterSetIdName = "clusterset.k8s.io" -) - -// ClusterUtils provides utility functions for working with clusters -type ClusterUtils struct { - client client.Client - clusterId string - clusterSetId string -} - -// constructor -func NewClusterUtils(client client.Client) ClusterUtils { - return ClusterUtils{ - client: client, - } -} - -// constructor for tests -func NewClusterUtilsForTest(clusterId string, clusterSetId string) ClusterUtils { - return ClusterUtils{ - clusterId: clusterId, - clusterSetId: clusterSetId, - } -} - -// retrieve the clusterId from the local field. If not set, retrieve from client -func (r *ClusterUtils) GetClusterId(ctx context.Context) (string, error) { - if r.clusterId != "" { - return r.clusterId, nil - } - clusterPropertyForClusterId := &aboutv1alpha1.ClusterProperty{} - err := r.client.Get(ctx, client.ObjectKey{Name: ClusterIdName}, clusterPropertyForClusterId) - if err != nil { - return "", err - } - if clusterPropertyForClusterId.Spec.Value == "" { - err := fmt.Errorf("ClusterId not found") - return "", err - } - r.clusterId = clusterPropertyForClusterId.Spec.Value - return r.clusterId, nil -} - -// retrieve the clusterSetId from the local field. If not set, retrieve from client -func (r *ClusterUtils) GetClusterSetId(ctx context.Context) (string, error) { - if r.clusterSetId != "" { - return r.clusterSetId, nil - } - clusterPropertyForClusterSetId := &aboutv1alpha1.ClusterProperty{} - err := r.client.Get(ctx, client.ObjectKey{Name: ClusterSetIdName}, clusterPropertyForClusterSetId) - if err != nil { - return "", err - } - if clusterPropertyForClusterSetId.Spec.Value == "" { - err := fmt.Errorf("ClusterSetId not found") - return "", err - } - r.clusterSetId = clusterPropertyForClusterSetId.Spec.Value - return r.clusterSetId, nil -} diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index 7dfd61ee..a235b2fa 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -10,7 +10,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" v1 "k8s.io/api/core/v1" - discovery "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -26,7 +26,7 @@ type CloudMapReconciler struct { Client client.Client Cloudmap cloudmap.ServiceDiscoveryClient Log common.Logger - ClusterUtils common.ClusterUtils + ClusterUtils model.ClusterUtils } // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch @@ -55,18 +55,12 @@ func (r *CloudMapReconciler) Start(ctx context.Context) error { // Reconcile triggers a single reconciliation round func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { - var err error - clusterId, err := r.ClusterUtils.GetClusterId(ctx) + clusterProperties, err := r.ClusterUtils.GetClusterProperties(ctx) if err != nil { - r.Log.Error(err, "unable to retrieve clusterId") + r.Log.Error(err, "unable to retrieve ClusterId and ClusterSetId") return err } - clusterSetId, err := r.ClusterUtils.GetClusterSetId(ctx) - if err != nil { - r.Log.Error(err, "unable to retrieve clusterSetId") - return err - } - r.Log.Debug("ClusterId and ClusterSetId found", "ClusterId", clusterId, "ClusterSetId", clusterSetId) + r.Log.Debug("clusterProperties found", "ClusterId", clusterProperties.ClusterId(), "ClusterSetId", clusterProperties.ClusterSetId()) namespaces := v1.NamespaceList{} if err := r.Client.List(ctx, &namespaces); err != nil { diff --git a/pkg/controllers/multicluster/cloudmap_controller_test.go b/pkg/controllers/multicluster/cloudmap_controller_test.go index 16b06ac4..b21a29c8 100644 --- a/pkg/controllers/multicluster/cloudmap_controller_test.go +++ b/pkg/controllers/multicluster/cloudmap_controller_test.go @@ -5,8 +5,10 @@ import ( "strings" "testing" - cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" + + cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -15,8 +17,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" - "k8s.io/api/discovery/v1beta1" - "k8s.io/apimachinery/pkg/runtime" + discovery "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" @@ -57,7 +58,7 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { assertDerivedService(t, &derivedService, test.ServicePort1, test.Port1) // assert endpoint slices are created - endpointSliceList := &v1beta1.EndpointSliceList{} + endpointSliceList := &discovery.EndpointSliceList{} err = fakeClient.List(context.TODO(), endpointSliceList, client.InNamespace(test.HttpNsName)) assert.NoError(t, err) endpointSlice := endpointSliceList.Items[0] @@ -112,12 +113,12 @@ func TestCloudMapReconciler_Reconcile_MulticlusterService(t *testing.T) { assertDerivedService(t, &derivedService2, test.ServicePort2, test.Port2) // assert endpoint slices are created for each derived service - endpointSliceList := &v1beta1.EndpointSliceList{} + endpointSliceList := &discovery.EndpointSliceList{} err = fakeClient.List(context.TODO(), endpointSliceList, client.InNamespace(test.HttpNsName)) assert.NoError(t, err) assert.Equal(t, 2, len(endpointSliceList.Items)) - endpointSliceMap := make(map[string]v1beta1.EndpointSlice) + endpointSliceMap := make(map[string]discovery.EndpointSlice) for _, endpointSlice := range endpointSliceList.Items { endpointSliceName := endpointSlice.ObjectMeta.Name derivedServiceName := endpointSliceName[:strings.LastIndex(endpointSliceName, "-")] @@ -131,10 +132,10 @@ func TestCloudMapReconciler_Reconcile_MulticlusterService(t *testing.T) { } func getCloudMapReconcilerScheme() *runtime.Scheme { - scheme := scheme.Scheme - scheme.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceImportList{}, &multiclusterv1alpha1.ServiceImport{}) - scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) - return scheme + s := scheme.Scheme + s.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceImportList{}, &multiclusterv1alpha1.ServiceImport{}) + s.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}, &aboutv1alpha1.ClusterPropertyList{}) + return s } func getReconciler(t *testing.T, mockSDClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *CloudMapReconciler { @@ -142,7 +143,7 @@ func getReconciler(t *testing.T, mockSDClient *cloudmapMock.MockServiceDiscovery Client: client, Cloudmap: mockSDClient, Log: common.NewLoggerWithLogr(testr.New(t)), - ClusterUtils: common.NewClusterUtils(client), + ClusterUtils: model.NewClusterUtils(client), } } @@ -153,7 +154,7 @@ func assertDerivedService(t *testing.T, derivedService *v1.Service, servicePort assert.Equal(t, int32(port), derivedService.Spec.Ports[0].TargetPort.IntVal) } -func assertEndpointSlice(t *testing.T, endpointSlice *v1beta1.EndpointSlice, port int, endptIp string, clusterId string) { +func assertEndpointSlice(t *testing.T, endpointSlice *discovery.EndpointSlice, port int, endptIp string, clusterId string) { assert.NotNil(t, endpointSlice) assert.Equal(t, test.SvcName, endpointSlice.Labels["multicluster.kubernetes.io/service-name"], "Endpoint slice is created") assert.Equal(t, clusterId, endpointSlice.Labels["multicluster.kubernetes.io/source-cluster"], "Endpoint slice is created") diff --git a/pkg/controllers/multicluster/controllers_common_test.go b/pkg/controllers/multicluster/controllers_common_test.go index edb2d1cb..5775246d 100644 --- a/pkg/controllers/multicluster/controllers_common_test.go +++ b/pkg/controllers/multicluster/controllers_common_test.go @@ -1,13 +1,16 @@ package controllers import ( + "strconv" "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" + multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" - discovery "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -56,6 +59,9 @@ func serviceExportForTest() *multiclusterv1alpha1.ServiceExport { func endpointSliceForTest() *discovery.EndpointSlice { port := int32(test.Port1) protocol := v1.ProtocolTCP + nodename := test.Nodename + hostname := test.Hostname + ready, _ := strconv.ParseBool(test.EndptReadyTrue) return &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Namespace: test.HttpNsName, @@ -65,6 +71,11 @@ func endpointSliceForTest() *discovery.EndpointSlice { AddressType: discovery.AddressTypeIPv4, Endpoints: []discovery.Endpoint{{ Addresses: []string{test.EndptIp1}, + Conditions: discovery.EndpointConditions{ + Ready: aws.Bool(ready), + }, + NodeName: &nodename, + Hostname: &hostname, }}, Ports: []discovery.EndpointPort{{ Name: aws.String(test.PortName1), @@ -74,14 +85,14 @@ func endpointSliceForTest() *discovery.EndpointSlice { } } -func endpointSliceWithIpsAndPortsForTest(ips []string, ports []discovery.EndpointPort) *discovery.EndpointSlice { +func endpointSliceFromEndpointsForTest(endpts []*model.Endpoint, ports []discovery.EndpointPort) *discovery.EndpointSlice { svc := k8sServiceForTest() slice := CreateEndpointSliceStruct(svc, test.SvcName, test.ClusterId1) slice.Ports = ports testEndpoints := make([]discovery.Endpoint, 0) - for _, ip := range ips { - testEndpoints = append(testEndpoints, CreateEndpointForSlice(svc, ip)) + for _, endpt := range endpts { + testEndpoints = append(testEndpoints, CreateEndpointForSlice(svc, endpt)) } slice.Endpoints = testEndpoints diff --git a/pkg/controllers/multicluster/endpointslice_plan.go b/pkg/controllers/multicluster/endpointslice_plan.go index fc0f1af4..fe05fb82 100644 --- a/pkg/controllers/multicluster/endpointslice_plan.go +++ b/pkg/controllers/multicluster/endpointslice_plan.go @@ -3,7 +3,7 @@ package controllers import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" v1 "k8s.io/api/core/v1" - discovery "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" ) const defaultMaxEndpointsPerSlice = 100 @@ -62,7 +62,7 @@ func (p *EndpointSlicePlan) CalculateChanges() EndpointSliceChanges { // stop adding to slice once it is full break } - sliceWithRoom.Endpoints = append(sliceWithRoom.Endpoints, CreateEndpointForSlice(p.Service, endpointToAdd.IP)) + sliceWithRoom.Endpoints = append(sliceWithRoom.Endpoints, CreateEndpointForSlice(p.Service, endpointToAdd)) delete(desiredEndpoints, key) } diff --git a/pkg/controllers/multicluster/endpointslice_plan_test.go b/pkg/controllers/multicluster/endpointslice_plan_test.go index b4805bdf..d058ab86 100644 --- a/pkg/controllers/multicluster/endpointslice_plan_test.go +++ b/pkg/controllers/multicluster/endpointslice_plan_test.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/stretchr/testify/assert" - discovery "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" ) func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { @@ -49,12 +49,9 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { }, want: EndpointSliceChanges{ Create: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp1}, - []discovery.EndpointPort{ - PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), - }, - ), + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint1()}, []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), + }), }, }, }, @@ -62,23 +59,19 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { name: "removed endpoint needs slice update", fields: fields{ Current: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp1, test.EndptIp2}, + endpointSliceFromEndpointsForTest( + []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, []discovery.EndpointPort{ PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), - }, - ), + }), }, Desired: []*model.Endpoint{test.GetTestEndpoint2()}, }, want: EndpointSliceChanges{ Update: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp2}, - []discovery.EndpointPort{ - PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), - }, - ), + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint2()}, []discovery.EndpointPort{ + PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), + }), }, }, }, @@ -86,18 +79,19 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { name: "added endpoint needs slice update", fields: fields{ Current: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp1}, + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint1()}, []discovery.EndpointPort{ PortToEndpointPort(model.Port{Name: test.PortName1, Port: test.Port1, Protocol: test.Protocol1}), - }, - ), + }), }, Desired: []*model.Endpoint{ test.GetTestEndpoint1(), { - Id: test.EndptId2, - IP: test.EndptIp2, + Id: test.EndptId2, + IP: test.EndptIp2, + Ready: true, + Hostname: test.Hostname, + Nodename: test.Nodename, EndpointPort: model.Port{ Name: test.PortName1, Port: test.Port1, @@ -108,12 +102,10 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { }, want: EndpointSliceChanges{ Update: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp1, test.EndptIp2}, + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}, []discovery.EndpointPort{ PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), - }, - ), + }), }, Unmodified: []*discovery.EndpointSlice{}, }, @@ -122,12 +114,10 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { name: "swapped endpoints need slice update", fields: fields{ Current: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp1}, + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint1()}, []discovery.EndpointPort{ PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), - }, - ), + }), }, Desired: []*model.Endpoint{ test.GetTestEndpoint2(), @@ -135,12 +125,10 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { }, want: EndpointSliceChanges{ Update: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp2}, + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint2()}, []discovery.EndpointPort{ PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), - }, - ), + }), }, Delete: []*discovery.EndpointSlice{}, }, @@ -149,12 +137,10 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { name: "changed ports need slice update", fields: fields{ Current: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp2}, + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint2()}, []discovery.EndpointPort{ PortToEndpointPort(test.GetTestEndpoint1().EndpointPort), - }, - ), + }), }, Desired: []*model.Endpoint{ test.GetTestEndpoint2(), @@ -162,12 +148,10 @@ func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { }, want: EndpointSliceChanges{ Update: []*discovery.EndpointSlice{ - endpointSliceWithIpsAndPortsForTest( - []string{test.EndptIp2}, + endpointSliceFromEndpointsForTest([]*model.Endpoint{test.GetTestEndpoint2()}, []discovery.EndpointPort{ PortToEndpointPort(test.GetTestEndpoint2().EndpointPort), - }, - ), + }), }, }, }, diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index 0ec1cc17..4d53bc64 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -9,8 +9,9 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" + "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" - discovery "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -28,19 +29,13 @@ import ( multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" ) -const ( - K8sVersionAttr = "K8S_CONTROLLER" - ServiceExportFinalizer = "multicluster.k8s.aws/service-export-finalizer" - EndpointSliceServiceLabel = "kubernetes.io/service-name" -) - // ServiceExportReconciler reconciles a ServiceExport object type ServiceExportReconciler struct { Client client.Client Log common.Logger Scheme *runtime.Scheme CloudMap cloudmap.ServiceDiscoveryClient - ClusterUtils common.ClusterUtils + ClusterUtils model.ClusterUtils } // +kubebuilder:rbac:groups="",resources=services,verbs=get @@ -57,19 +52,6 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques name := req.NamespacedName r.Log.Debug("reconciling ServiceExport", "Namespace", namespace, "Name", name) - var err error - clusterId, err := r.ClusterUtils.GetClusterId(ctx) - if err != nil { - r.Log.Error(err, "unable to retrieve clusterId") - return ctrl.Result{}, err - } - clusterSetId, err := r.ClusterUtils.GetClusterSetId(ctx) - if err != nil { - r.Log.Error(err, "unable to retrieve clusterSetId") - return ctrl.Result{}, err - } - r.Log.Debug("ClusterId and ClusterSetId found", "ClusterId", clusterId, "ClusterSetId", clusterSetId) - serviceExport := multiclusterv1alpha1.ServiceExport{} if err := r.Client.Get(ctx, name, &serviceExport); err != nil { if errors.IsNotFound(err) { @@ -83,7 +65,7 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Mark ServiceExport to be deleted, which is indicated by the deletion timestamp being set. - isServiceExportMarkedForDelete := serviceExport.GetDeletionTimestamp() != nil + isServiceExportMarkedForDelete := !serviceExport.GetDeletionTimestamp().IsZero() service := v1.Service{} namespacedName := types.NamespacedName{Namespace: serviceExport.Namespace, Name: serviceExport.Name} @@ -109,40 +91,21 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques } func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExport *multiclusterv1alpha1.ServiceExport, service *v1.Service) (ctrl.Result, error) { - // Add the finalizer to the service export if not present, ensures the ServiceExport won't be deleted - if !controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { - controllerutil.AddFinalizer(serviceExport, ServiceExportFinalizer) - if err := r.Client.Update(ctx, serviceExport); err != nil { - r.Log.Error(err, "error adding finalizer", - "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) - return ctrl.Result{}, err - } - } - - if len(serviceExport.GetOwnerReferences()) == 0 { - err := controllerutil.SetControllerReference(service, serviceExport, r.Scheme) - if err == nil { - err = r.Client.Update(ctx, serviceExport) - } - if err != nil { - r.Log.Error(err, "error setting Service as an owner of the ServiceExport", - "namespace", service.Namespace, "name", service.Name) - return ctrl.Result{}, err - } + err := r.addFinalizerAndOwnerRef(ctx, serviceExport, service) + if err != nil { + return ctrl.Result{}, err } r.Log.Info("updating Cloud Map service", "namespace", service.Namespace, "name", service.Name) cmService, err := r.createOrGetCloudMapService(ctx, service) if err != nil { - r.Log.Error(err, "error fetching Service from Cloud Map", - "namespace", service.Namespace, "name", service.Name) + r.Log.Error(err, "error fetching Service from Cloud Map", "namespace", service.Namespace, "name", service.Name) return ctrl.Result{}, err } endpoints, err := r.extractEndpoints(ctx, service, serviceExport) if err != nil { - r.Log.Error(err, "error extracting Endpoints", - "namespace", serviceExport.Namespace, "name", serviceExport.Name) + r.Log.Error(err, "error extracting Endpoints", "namespace", serviceExport.Namespace, "name", serviceExport.Name) return ctrl.Result{}, err } @@ -159,16 +122,14 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor upserts = append(upserts, changes.Update...) if err := r.CloudMap.RegisterEndpoints(ctx, service.Namespace, service.Name, upserts); err != nil { - r.Log.Error(err, "error registering Endpoints to Cloud Map", - "namespace", service.Namespace, "name", service.Name) + r.Log.Error(err, "error registering Endpoints to Cloud Map", "namespace", service.Namespace, "name", service.Name) return ctrl.Result{}, err } } if changes.HasDeletes() { if err := r.CloudMap.DeleteEndpoints(ctx, service.Namespace, service.Name, changes.Delete); err != nil { - r.Log.Error(err, "error deleting Endpoints from Cloud Map", - "namespace", cmService.Namespace, "name", cmService.Name) + r.Log.Error(err, "error deleting Endpoints from Cloud Map", "namespace", cmService.Namespace, "name", cmService.Name) return ctrl.Result{}, err } } @@ -180,6 +141,28 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor return ctrl.Result{}, nil } +func (r *ServiceExportReconciler) addFinalizerAndOwnerRef(ctx context.Context, serviceExport *multiclusterv1alpha1.ServiceExport, service *v1.Service) error { + // Add the finalizer to the service export if not present, ensures the ServiceExport won't be deleted + if !controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { + controllerutil.AddFinalizer(serviceExport, ServiceExportFinalizer) + if err := r.Client.Update(ctx, serviceExport); err != nil { + r.Log.Error(err, "error adding finalizer", "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) + return err + } + } + if len(serviceExport.GetOwnerReferences()) == 0 { + err := controllerutil.SetControllerReference(service, serviceExport, r.Scheme) + if err == nil { + err = r.Client.Update(ctx, serviceExport) + } + if err != nil { + r.Log.Error(err, "error setting Service as an owner of the ServiceExport", "namespace", service.Namespace, "name", service.Name) + return err + } + } + return nil +} + func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context, service *v1.Service) (*model.Service, error) { cmService, err := r.CloudMap.GetService(ctx, service.Namespace, service.Name) if err != nil { @@ -189,8 +172,7 @@ func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context if cmService == nil { err = r.CloudMap.CreateService(ctx, service.Namespace, service.Name) if err != nil { - r.Log.Error(err, "error creating a new Service in Cloud Map", - "namespace", service.Namespace, "name", service.Name) + r.Log.Error(err, "error creating a new Service in Cloud Map", "namespace", service.Namespace, "name", service.Name) return nil, err } if cmService, err = r.CloudMap.GetService(ctx, service.Namespace, service.Name); err != nil { @@ -207,14 +189,12 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor cmService, err := r.CloudMap.GetService(ctx, serviceExport.Namespace, serviceExport.Name) if err != nil { - r.Log.Error(err, "error fetching Service from Cloud Map", - "namespace", serviceExport.Namespace, "name", serviceExport.Name) + r.Log.Error(err, "error fetching Service from Cloud Map", "namespace", serviceExport.Namespace, "name", serviceExport.Name) return ctrl.Result{}, err } if cmService != nil { if err := r.CloudMap.DeleteEndpoints(ctx, cmService.Namespace, cmService.Name, cmService.Endpoints); err != nil { - r.Log.Error(err, "error deleting Endpoints from Cloud Map", - "namespace", cmService.Namespace, "name", cmService.Name) + r.Log.Error(err, "error deleting Endpoints from Cloud Map", "namespace", cmService.Namespace, "name", cmService.Name) return ctrl.Result{}, err } } @@ -231,10 +211,14 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor } func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1.Service, svcExport *multiclusterv1alpha1.ServiceExport) ([]*model.Endpoint, error) { - result := make([]*model.Endpoint, 0) + clusterProperties, err := r.ClusterUtils.GetClusterProperties(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve ClusterId and ClusterSetId") + return nil, err + } endpointSlices := discovery.EndpointSliceList{} - err := r.Client.List(ctx, &endpointSlices, + err = r.Client.List(ctx, &endpointSlices, client.InNamespace(svc.Namespace), client.MatchingLabels{discovery.LabelServiceName: svc.Name}) if err != nil { @@ -248,80 +232,80 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. servicePortMap[svcPort.Name] = ServicePortToPort(svcPort) } - clusterId, err := r.ClusterUtils.GetClusterId(ctx) - if err != nil { - r.Log.Error(err, "unable to retrieve clusterId") - return nil, err - } - clusterSetId, err := r.ClusterUtils.GetClusterSetId(ctx) - if err != nil { - r.Log.Error(err, "unable to retrieve clusterSetId") - return nil, err - } - var svcExportCreationTimestamp int64 = 0 if !svcExport.ObjectMeta.CreationTimestamp.IsZero() { svcExportCreationTimestamp = svcExport.ObjectMeta.CreationTimestamp.Time.UnixMilli() } + attributes := make(map[string]string) + if version.GetVersion() != "" { + attributes[model.K8sVersionAttr] = version.PackageName + " " + version.GetVersion() + } + + endpoints := make([]*model.Endpoint, 0) for _, slice := range endpointSlices.Items { if slice.AddressType != discovery.AddressTypeIPv4 { return nil, fmt.Errorf("unsupported address type %s for service %s", slice.AddressType, svc.Name) } for _, endpointPort := range slice.Ports { for _, endpoint := range slice.Endpoints { + port := EndpointPortToPort(endpointPort) + readyCondition := aws.ToBool(endpoint.Conditions.Ready) + for _, IP := range endpoint.Addresses { - attributes := make(map[string]string) - if version.GetVersion() != "" { - attributes[K8sVersionAttr] = version.PackageName + " " + version.GetVersion() - } - - // TODO extract attributes - pod, node and other useful details if possible - - port := EndpointPortToPort(endpointPort) - result = append(result, &model.Endpoint{ - Id: model.EndpointIdFromIPAddressAndPort(IP, port), - IP: IP, - EndpointPort: port, - ServicePort: servicePortMap[*endpointPort.Name], - ClusterId: clusterId, - ClusterSetId: clusterSetId, - ServiceType: serviceType, - SvcExportCreationTimestamp: svcExportCreationTimestamp, - Attributes: attributes, + endpoints = append(endpoints, &model.Endpoint{ + Id: model.EndpointIdFromIPAddressAndPort(IP, port), + IP: IP, + EndpointPort: port, + ServicePort: servicePortMap[*endpointPort.Name], + ClusterId: clusterProperties.ClusterId(), + ClusterSetId: clusterProperties.ClusterSetId(), + ServiceType: serviceType, + ServiceExportCreationTimestamp: svcExportCreationTimestamp, + Ready: readyCondition, + Hostname: aws.ToString(endpoint.Hostname), + Nodename: aws.ToString(endpoint.NodeName), + Attributes: attributes, }) } } } } - return result, nil + return endpoints, nil } func (r *ServiceExportReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&multiclusterv1alpha1.ServiceExport{}). - // Watch for the changes to the EndpointSlice object. This object is bound to be - // updated when Service or Deployment are updated. There is also a filtering logic - // to enqueue those EndpointSlice event which have corresponding ServiceExport + // Filter-out all the events if the cluster-properties are not found + WithEventFilter(r.clusterPropertyFilter()). + // Watch for the changes to Service which have corresponding ServiceExport + Watches( + &source.Kind{Type: &v1.Service{}}, + &handler.EnqueueRequestForObject{}, + builder.WithPredicates(r.serviceExportPredicates()), + ). + // Watch for the changes to the EndpointSlice object which have corresponding ServiceExport. + // This object is bound to be updated when Deployment are updated. Watches( &source.Kind{Type: &discovery.EndpointSlice{}}, - handler.EnqueueRequestsFromMapFunc(r.endpointSliceEventHandler()), - builder.WithPredicates(r.endpointSliceFilter()), + handler.EnqueueRequestsFromMapFunc(r.endpointSliceMappingFunction()), + builder.WithPredicates(r.serviceExportPredicates()), ). // Watch for changes to ClusterProperty objects. If a ClusterProperty object is // created, updated or deleted, the controller will reconcile all service exports Watches( &source.Kind{Type: &aboutv1alpha1.ClusterProperty{}}, - handler.EnqueueRequestsFromMapFunc(r.clusterPropertyEventHandler()), + handler.EnqueueRequestsFromMapFunc(r.clusterPropertyMappingFunction()), ). Complete(r) } -func (r *ServiceExportReconciler) endpointSliceEventHandler() handler.MapFunc { +func (r *ServiceExportReconciler) endpointSliceMappingFunction() handler.MapFunc { return func(object client.Object) []reconcile.Request { labels := object.GetLabels() - serviceName := labels[EndpointSliceServiceLabel] + serviceName := labels[discovery.LabelServiceName] return []reconcile.Request{ {NamespacedName: types.NamespacedName{ Name: serviceName, @@ -331,12 +315,18 @@ func (r *ServiceExportReconciler) endpointSliceEventHandler() handler.MapFunc { } } -func (r *ServiceExportReconciler) clusterPropertyEventHandler() handler.MapFunc { +func (r *ServiceExportReconciler) clusterPropertyMappingFunction() handler.MapFunc { // Return reconcile requests for all service exports return func(object client.Object) []reconcile.Request { + // Reset clusterproperties if there is an &aboutv1alpha1.ClusterProperty{} event + err := r.ClusterUtils.LoadClusterProperties(context.TODO()) + if err != nil { + return nil + } + serviceExports := &multiclusterv1alpha1.ServiceExportList{} if err := r.Client.List(context.TODO(), serviceExports); err != nil { - r.Log.Error(err, "error listing services") + r.Log.Error(err, "error listing ServiceExports") return nil } @@ -351,26 +341,28 @@ func (r *ServiceExportReconciler) clusterPropertyEventHandler() handler.MapFunc } } -func (r *ServiceExportReconciler) endpointSliceFilter() predicate.Funcs { +func (r *ServiceExportReconciler) serviceExportPredicates() predicate.Funcs { return predicate.Funcs{ GenericFunc: func(e event.GenericEvent) bool { - return r.doesEndpointSliceHaveServiceExport(e.Object) + return r.doesObjectHaveServiceExport(e.Object) }, CreateFunc: func(e event.CreateEvent) bool { - return r.doesEndpointSliceHaveServiceExport(e.Object) + return r.doesObjectHaveServiceExport(e.Object) }, UpdateFunc: func(e event.UpdateEvent) bool { - return r.doesEndpointSliceHaveServiceExport(e.ObjectNew) + return r.doesObjectHaveServiceExport(e.ObjectNew) }, DeleteFunc: func(e event.DeleteEvent) bool { - return r.doesEndpointSliceHaveServiceExport(e.Object) + return r.doesObjectHaveServiceExport(e.Object) }, } } -func (r *ServiceExportReconciler) doesEndpointSliceHaveServiceExport(object client.Object) bool { - labels := object.GetLabels() - serviceName := labels[EndpointSliceServiceLabel] +func (r *ServiceExportReconciler) doesObjectHaveServiceExport(object client.Object) bool { + serviceName, ok := object.GetLabels()[discovery.LabelServiceName] + if !ok { + serviceName = object.GetName() + } ns := types.NamespacedName{ Name: serviceName, Namespace: object.GetNamespace(), @@ -381,3 +373,30 @@ func (r *ServiceExportReconciler) doesEndpointSliceHaveServiceExport(object clie } return true } + +func (r *ServiceExportReconciler) clusterPropertyFilter() predicate.Funcs { + return predicate.Funcs{ + GenericFunc: func(e event.GenericEvent) bool { + return r.doesClusterPropertyExists() + }, + CreateFunc: func(e event.CreateEvent) bool { + return r.doesClusterPropertyExists() + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return r.doesClusterPropertyExists() + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return r.doesClusterPropertyExists() + }, + } +} + +func (r *ServiceExportReconciler) doesClusterPropertyExists() bool { + clusterProperties, err := r.ClusterUtils.GetClusterProperties(context.TODO()) + if err != nil { + r.Log.Error(err, "unable to retrieve ClusterId and ClusterSetId") + return false + } + r.Log.Debug("clusterProperties found", "ClusterId", clusterProperties.ClusterId(), "ClusterSetId", clusterProperties.ClusterSetId()) + return clusterProperties.IsValid() +} diff --git a/pkg/controllers/multicluster/serviceexport_controller_test.go b/pkg/controllers/multicluster/serviceexport_controller_test.go index 45eaaa16..b78f8d5e 100644 --- a/pkg/controllers/multicluster/serviceexport_controller_test.go +++ b/pkg/controllers/multicluster/serviceexport_controller_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" - discovery "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -161,55 +161,25 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { assert.Empty(t, serviceExport.Finalizers, "Finalizer removed from the service export") } -func TestServiceExportReconciler_Reconcile_NoClusterId(t *testing.T) { +func TestServiceExportReconciler_Reconcile_NoClusterProperty(t *testing.T) { // create a fake controller client and add some objects fakeClient := fake.NewClientBuilder(). WithScheme(getServiceExportScheme()). + // do-not add clusterId WithObjects(k8sServiceForTest(), serviceExportForTest(), test.ClusterSetIdForTest()). WithLists(&discovery.EndpointSliceList{ Items: []discovery.EndpointSlice{*endpointSliceForTest()}, - }). - Build() + }).Build() // create a mock cloudmap service discovery client mockController := gomock.NewController(t) defer mockController.Finish() - mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) - - reconciler := getServiceExportReconciler(t, mock, fakeClient) - - request := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Namespace: test.HttpNsName, - Name: test.SvcName, - }, - } - - // Reconciling should throw an error - got, err := reconciler.Reconcile(context.Background(), request) - expectedError := fmt.Errorf("clusterproperties.about.k8s.io \"id.k8s.io\" not found") - assert.ErrorContains(t, err, expectedError.Error()) - assert.Equal(t, ctrl.Result{}, got, "Result should be empty") -} - -func TestServiceExportReconciler_Reconcile_NoClustersetId(t *testing.T) { - // create a fake controller client and add some objects - fakeClient := fake.NewClientBuilder(). - WithScheme(getServiceExportScheme()). - WithObjects(k8sServiceForTest(), serviceExportForTest(), test.ClusterIdForTest()). - WithLists(&discovery.EndpointSliceList{ - Items: []discovery.EndpointSlice{*endpointSliceForTest()}, - }). - Build() - - // create a mock cloudmap service discovery client - mockController := gomock.NewController(t) - defer mockController.Finish() + mockSDClient := cloudmapMock.NewMockServiceDiscoveryClient(mockController) - mock := cloudmapMock.NewMockServiceDiscoveryClient(mockController) + mockSDClient.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName).Return(test.GetTestService(), nil) - reconciler := getServiceExportReconciler(t, mock, fakeClient) + reconciler := getServiceExportReconciler(t, mockSDClient, fakeClient) request := ctrl.Request{ NamespacedName: types.NamespacedName{ @@ -220,14 +190,14 @@ func TestServiceExportReconciler_Reconcile_NoClustersetId(t *testing.T) { // Reconciling should throw an error got, err := reconciler.Reconcile(context.Background(), request) - expectedError := fmt.Errorf("clusterproperties.about.k8s.io \"clusterset.k8s.io\" not found") + expectedError := fmt.Errorf("ClusterProperty not found") assert.ErrorContains(t, err, expectedError.Error()) assert.Equal(t, ctrl.Result{}, got, "Result should be empty") } func getServiceExportScheme() *runtime.Scheme { scheme := runtime.NewScheme() - scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) + scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}, &aboutv1alpha1.ClusterPropertyList{}) scheme.AddKnownTypes(multiclusterv1alpha1.GroupVersion, &multiclusterv1alpha1.ServiceExport{}) scheme.AddKnownTypes(v1.SchemeGroupVersion, &v1.Service{}) scheme.AddKnownTypes(discovery.SchemeGroupVersion, &discovery.EndpointSlice{}, &discovery.EndpointSliceList{}) @@ -240,6 +210,6 @@ func getServiceExportReconciler(t *testing.T, mockClient *cloudmapMock.MockServi Log: common.NewLoggerWithLogr(testr.New(t)), Scheme: client.Scheme(), CloudMap: mockClient, - ClusterUtils: common.NewClusterUtils(client), + ClusterUtils: model.NewClusterUtils(client), } } diff --git a/pkg/controllers/multicluster/utils.go b/pkg/controllers/multicluster/utils.go index a434d2d2..8bcc0b08 100644 --- a/pkg/controllers/multicluster/utils.go +++ b/pkg/controllers/multicluster/utils.go @@ -11,7 +11,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" - discovery "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,6 +21,9 @@ const ( // DerivedServiceAnnotation annotates a ServiceImport with derived Service name DerivedServiceAnnotation = "multicluster.k8s.aws/derived-service" + // ServiceExportFinalizer finalizer to perform cloudmap resource cleanup on delete + ServiceExportFinalizer = "multicluster.k8s.aws/service-export-finalizer" + // LabelServiceImportName indicates the name of the multi-cluster service that an EndpointSlice belongs to. LabelServiceImportName = "multicluster.kubernetes.io/service-name" @@ -225,14 +228,8 @@ func CreateDerivedServiceStruct(svcImport *multiclusterv1alpha1.ServiceImport, i return svc } -func CreateEndpointForSlice(svc *v1.Service, ip string) discovery.Endpoint { - t := true - - return discovery.Endpoint{ - Addresses: []string{ip}, - Conditions: discovery.EndpointConditions{ - Ready: &t, - }, +func CreateEndpointForSlice(svc *v1.Service, endpoint *model.Endpoint) discovery.Endpoint { + ep := discovery.Endpoint{ TargetRef: &v1.ObjectReference{ Kind: "Service", Namespace: svc.Namespace, @@ -240,7 +237,18 @@ func CreateEndpointForSlice(svc *v1.Service, ip string) discovery.Endpoint { UID: svc.ObjectMeta.UID, ResourceVersion: svc.ObjectMeta.ResourceVersion, }, + Addresses: []string{endpoint.IP}, + Conditions: discovery.EndpointConditions{ + Ready: &endpoint.Ready, + }, + } + if endpoint.Hostname != "" { + ep.Hostname = &endpoint.Hostname + } + if endpoint.Nodename != "" { + ep.NodeName = &endpoint.Nodename } + return ep } func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string, clusterId string) *discovery.EndpointSlice { diff --git a/pkg/controllers/multicluster/utils_test.go b/pkg/controllers/multicluster/utils_test.go index 7c86383f..56157af4 100644 --- a/pkg/controllers/multicluster/utils_test.go +++ b/pkg/controllers/multicluster/utils_test.go @@ -10,7 +10,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" - "k8s.io/api/discovery/v1beta1" + discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -90,7 +90,7 @@ func TestServiceImportPortToPort(t *testing.T) { func TestEndpointPortToPort(t *testing.T) { type args struct { - port v1beta1.EndpointPort + port discovery.EndpointPort } name := "http" protocolTCP := v1.ProtocolTCP @@ -103,7 +103,7 @@ func TestEndpointPortToPort(t *testing.T) { { name: "happy case", args: args{ - port: v1beta1.EndpointPort{ + port: discovery.EndpointPort{ Name: &name, Protocol: &protocolTCP, Port: &port, @@ -230,7 +230,7 @@ func TestPortToEndpointPort(t *testing.T) { tests := []struct { name string args args - want v1beta1.EndpointPort + want discovery.EndpointPort }{ { name: "happy case", @@ -241,7 +241,7 @@ func TestPortToEndpointPort(t *testing.T) { Protocol: "TCP", }, }, - want: v1beta1.EndpointPort{ + want: discovery.EndpointPort{ Name: &name, Protocol: &protocolTCP, Port: &port, diff --git a/pkg/model/cluster.go b/pkg/model/cluster.go new file mode 100644 index 00000000..97f057bc --- /dev/null +++ b/pkg/model/cluster.go @@ -0,0 +1,86 @@ +package model + +import ( + "context" + "fmt" + + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ClusterIdName = "id.k8s.io" + ClusterSetIdName = "clusterset.k8s.io" +) + +// Non-exported type, accessible via read-only func +type clusterProperties struct { + clusterId string + clusterSetId string +} + +func (r clusterProperties) ClusterId() string { + return r.clusterId +} + +func (r clusterProperties) ClusterSetId() string { + return r.clusterSetId +} + +func (r clusterProperties) IsValid() bool { + return r.clusterSetId != "" && r.clusterId != "" +} + +func (r clusterProperties) String() string { + return fmt.Sprintf("ClusterId: %s, ClusterSetId: %s", r.clusterId, r.clusterSetId) +} + +// ClusterUtils provides utility functions for working with clusters +type ClusterUtils struct { + client client.Client + clusterProperties clusterProperties +} + +func NewClusterUtils(client client.Client) ClusterUtils { + return ClusterUtils{ + client: client, + clusterProperties: clusterProperties{}, + } +} + +func NewClusterUtilsWithValues(clusterId string, clusterSetId string) ClusterUtils { + return ClusterUtils{ + clusterProperties: clusterProperties{clusterId: clusterId, clusterSetId: clusterSetId}, + } +} + +func (r *ClusterUtils) GetClusterProperties(ctx context.Context) (*clusterProperties, error) { + if !r.clusterProperties.IsValid() { + err := r.LoadClusterProperties(ctx) + if err != nil { + return nil, err + } + } + return &r.clusterProperties, nil +} + +func (r *ClusterUtils) LoadClusterProperties(ctx context.Context) error { + clusterPropertyList := &aboutv1alpha1.ClusterPropertyList{} + err := r.client.List(ctx, clusterPropertyList) + if err != nil { + return err + } + for _, clusterProperty := range clusterPropertyList.Items { + switch clusterProperty.Name { + case ClusterIdName: + r.clusterProperties.clusterId = clusterProperty.Spec.Value + case ClusterSetIdName: + r.clusterProperties.clusterSetId = clusterProperty.Spec.Value + } + } + if !r.clusterProperties.IsValid() { + return fmt.Errorf("ClusterProperty not found: %s", r.clusterProperties) + } + return nil +} diff --git a/pkg/model/cluster_test.go b/pkg/model/cluster_test.go new file mode 100644 index 00000000..fad36427 --- /dev/null +++ b/pkg/model/cluster_test.go @@ -0,0 +1,108 @@ +package model + +import ( + "context" + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestClusterUtils_GetClusterProperties(t *testing.T) { + type fields struct { + client client.Client + clusterProperties clusterProperties + } + type args struct { + ctx context.Context + } + clusterId := "cluster1" + clusterSetId := "clusterset1" + tests := []struct { + name string + fields fields + args args + want *clusterProperties + wantErr bool + }{ + { + name: "happy case fetch from client", + fields: fields{ + client: fake.NewClientBuilder().WithScheme(GetScheme()).WithObjects(ClusterIdForTest(clusterId), ClusterSetIdForTest(clusterSetId)).Build(), + clusterProperties: clusterProperties{}, + }, + args: args{ctx: context.TODO()}, + want: &clusterProperties{clusterId: clusterId, clusterSetId: clusterSetId}, + wantErr: false, + }, + { + name: "happy case already set", + fields: fields{ + client: nil, + clusterProperties: clusterProperties{clusterId: clusterId, clusterSetId: clusterSetId}, + }, + args: args{ctx: context.TODO()}, + want: &clusterProperties{clusterId: clusterId, clusterSetId: clusterSetId}, + wantErr: false, + }, + { + name: "error cluster properties not present", + fields: fields{ + client: fake.NewClientBuilder().WithScheme(GetScheme()).Build(), + clusterProperties: clusterProperties{}, + }, + args: args{ctx: context.TODO()}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &ClusterUtils{ + client: tt.fields.client, + clusterProperties: tt.fields.clusterProperties, + } + got, err := r.GetClusterProperties(tt.args.ctx) + if (err != nil) != tt.wantErr { + t.Errorf("GetClusterProperties() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetClusterProperties() got = %v, want %v", got, tt.want) + } + }) + } +} + +func ClusterIdForTest(clusterId string) *aboutv1alpha1.ClusterProperty { + return &aboutv1alpha1.ClusterProperty{ + ObjectMeta: metav1.ObjectMeta{ + Name: ClusterIdName, + }, + Spec: aboutv1alpha1.ClusterPropertySpec{ + Value: clusterId, + }, + } +} + +func ClusterSetIdForTest(clusterSetId string) *aboutv1alpha1.ClusterProperty { + return &aboutv1alpha1.ClusterProperty{ + ObjectMeta: metav1.ObjectMeta{ + Name: ClusterSetIdName, + }, + Spec: aboutv1alpha1.ClusterPropertySpec{ + Value: clusterSetId, + }, + } +} + +func GetScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}, &aboutv1alpha1.ClusterPropertyList{}) + return scheme +} diff --git a/pkg/model/types.go b/pkg/model/types.go index 92355775..3a78a3e5 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -42,15 +42,18 @@ type ServiceType string // Endpoint holds basic values and attributes for an endpoint. type Endpoint struct { - Id string - IP string - EndpointPort Port - ServicePort Port - ClusterId string - ClusterSetId string - ServiceType ServiceType - SvcExportCreationTimestamp int64 - Attributes map[string]string + Id string + IP string + EndpointPort Port + ServicePort Port + ClusterId string + ClusterSetId string + ServiceType ServiceType + ServiceExportCreationTimestamp int64 + Ready bool + Hostname string + Nodename string + Attributes map[string]string } type Port struct { @@ -63,18 +66,22 @@ type Port struct { // Cloudmap Instances IP and Port is supposed to be AWS_INSTANCE_IPV4 and AWS_INSTANCE_PORT // Rest are custom attributes const ( - SvcExportCreationTimestampAttr = "SVC_EXPORT_CREATION_TIMESTAMP" - EndpointIpv4Attr = "AWS_INSTANCE_IPV4" - EndpointPortAttr = "AWS_INSTANCE_PORT" - EndpointPortNameAttr = "ENDPOINT_PORT_NAME" - EndpointProtocolAttr = "ENDPOINT_PROTOCOL" - ClusterIdAttr = "CLUSTER_ID" - ClusterSetIdAttr = "CLUSTERSET_ID" - ServicePortNameAttr = "SERVICE_PORT_NAME" - ServicePortAttr = "SERVICE_PORT" - ServiceTargetPortAttr = "SERVICE_TARGET_PORT" - ServiceProtocolAttr = "SERVICE_PROTOCOL" - ServiceTypeAttr = "SERVICE_TYPE" + EndpointIpv4Attr = "AWS_INSTANCE_IPV4" + EndpointPortAttr = "AWS_INSTANCE_PORT" + EndpointPortNameAttr = "ENDPOINT_PORT_NAME" + EndpointProtocolAttr = "ENDPOINT_PROTOCOL" + EndpointReadyAttr = "READY" + EndpointHostnameAttr = "HOSTNAME" + EndpointNodeNameAttr = "NODENAME" + ClusterIdAttr = "CLUSTER_ID" + ClusterSetIdAttr = "CLUSTERSET_ID" + ServicePortNameAttr = "SERVICE_PORT_NAME" + ServicePortAttr = "SERVICE_PORT" + ServiceTargetPortAttr = "SERVICE_TARGET_PORT" + ServiceProtocolAttr = "SERVICE_PROTOCOL" + ServiceTypeAttr = "SERVICE_TYPE" + ServiceExportCreationAttr = "SERVICE_EXPORT_CREATION_TIMESTAMP" + K8sVersionAttr = "K8S_CONTROLLER" ) // NewEndpointFromInstance converts a Cloud Map HttpInstanceSummary to an endpoint. @@ -121,10 +128,18 @@ func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) return nil, err } - if endpoint.SvcExportCreationTimestamp, err = removeTimestampAttr(attributes, SvcExportCreationTimestampAttr); err != nil { - endpoint.SvcExportCreationTimestamp = 0 + if endpoint.Ready, err = removeBoolAttr(attributes, EndpointReadyAttr); err != nil { + return nil, err + } + + if endpoint.ServiceExportCreationTimestamp, err = removeTimestampAttr(attributes, ServiceExportCreationAttr); err != nil { + return nil, err } + // Hostname and Nodename are Optional attributes + endpoint.Hostname, _ = removeStringAttr(attributes, EndpointHostnameAttr) + endpoint.Nodename, _ = removeStringAttr(attributes, EndpointNodeNameAttr) + // Add the remaining attributes endpoint.Attributes = attributes @@ -174,8 +189,7 @@ func removeIntAttr(attributes map[string]string, attr string) (int32, error) { if value, hasValue := attributes[attr]; hasValue { parsedValue, parseError := strconv.ParseUint(value, 10, 16) if parseError != nil { - return 0, fmt.Errorf("failed to parse the %s as int with error %s", - attr, parseError.Error()) + return 0, fmt.Errorf("failed to parse the %s as int with error %s", attr, parseError.Error()) } delete(attributes, attr) return int32(parsedValue), nil @@ -183,6 +197,18 @@ func removeIntAttr(attributes map[string]string, attr string) (int32, error) { return 0, fmt.Errorf("cannot find the attribute %s", attr) } +func removeBoolAttr(attributes map[string]string, attr string) (bool, error) { + if value, hasValue := attributes[attr]; hasValue { + parsedValue, parseError := strconv.ParseBool(value) + if parseError != nil { + return false, fmt.Errorf("failed to parse the %s as bool with error %s", attr, parseError.Error()) + } + delete(attributes, attr) + return parsedValue, nil + } + return false, fmt.Errorf("cannot find the attribute %s", attr) +} + func removeTimestampAttr(attributes map[string]string, attr string) (int64, error) { if value, hasValue := attributes[attr]; hasValue { parsedValue, parseError := strconv.ParseInt(value, 10, 64) @@ -211,7 +237,10 @@ func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs[ServiceTargetPortAttr] = e.ServicePort.TargetPort attrs[ServiceProtocolAttr] = e.ServicePort.Protocol attrs[ServiceTypeAttr] = e.ServiceType.String() - attrs[SvcExportCreationTimestampAttr] = strconv.FormatInt(e.SvcExportCreationTimestamp, 10) + attrs[ServiceExportCreationAttr] = strconv.FormatInt(e.ServiceExportCreationTimestamp, 10) + attrs[EndpointReadyAttr] = strconv.FormatBool(e.Ready) + attrs[EndpointHostnameAttr] = e.Hostname + attrs[EndpointNodeNameAttr] = e.Nodename for key, val := range e.Attributes { attrs[key] = val diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index 30bd48fb..bf3bfd8e 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -27,19 +27,20 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - ClusterIdAttr: clusterId, - ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, - EndpointPortAttr: "80", - EndpointProtocolAttr: "TCP", - EndpointPortNameAttr: "http", - ServicePortNameAttr: "http", - ServiceProtocolAttr: "TCP", - ServicePortAttr: "65535", - ServiceTargetPortAttr: "80", - ServiceTypeAttr: serviceType, - SvcExportCreationTimestampAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), - "custom-attr": "custom-val", + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, + ServiceExportCreationAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), + "custom-attr": "custom-val", }, }, want: &Endpoint{ @@ -56,10 +57,11 @@ func TestNewEndpointFromInstance(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, - ClusterId: clusterId, - ClusterSetId: clusterSetId, - ServiceType: ServiceType(serviceType), - SvcExportCreationTimestamp: svcExportCreationTimestamp, + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), + ServiceExportCreationTimestamp: svcExportCreationTimestamp, + Ready: true, Attributes: map[string]string{ "custom-attr": "custom-val", }, @@ -74,6 +76,7 @@ func TestNewEndpointFromInstance(t *testing.T) { EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", ServicePortNameAttr: "http", ServiceProtocolAttr: "TCP", ServicePortAttr: "99999", @@ -116,6 +119,7 @@ func TestNewEndpointFromInstance(t *testing.T) { EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", ServicePortNameAttr: "http", ServiceProtocolAttr: "TCP", ServicePortAttr: "65535", @@ -135,6 +139,7 @@ func TestNewEndpointFromInstance(t *testing.T) { EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", ServicePortNameAttr: "http", ServiceProtocolAttr: "TCP", ServicePortAttr: "65535", @@ -155,6 +160,7 @@ func TestNewEndpointFromInstance(t *testing.T) { EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", ServicePortNameAttr: "http", ServiceProtocolAttr: "TCP", ServicePortAttr: "65535", @@ -180,25 +186,14 @@ func TestNewEndpointFromInstance(t *testing.T) { } func TestEndpoint_GetAttributes(t *testing.T) { - type fields struct { - Id string - IP string - EndpointPort Port - ServicePort Port - ClusterId string - ClusterSetId string - ServiceType ServiceType - SvcExportCreationTimestamp int64 - Attributes map[string]string - } tests := []struct { - name string - fields fields - want map[string]string + name string + endpoint Endpoint + want map[string]string }{ { name: "happy case", - fields: fields{ + endpoint: Endpoint{ IP: ip, EndpointPort: Port{ Name: "http", @@ -211,45 +206,38 @@ func TestEndpoint_GetAttributes(t *testing.T) { TargetPort: "80", Protocol: "TCP", }, - ClusterId: clusterId, - ClusterSetId: clusterSetId, - ServiceType: ServiceType(serviceType), - SvcExportCreationTimestamp: svcExportCreationTimestamp, + Ready: true, + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), + ServiceExportCreationTimestamp: svcExportCreationTimestamp, Attributes: map[string]string{ "custom-attr": "custom-val", }, }, want: map[string]string{ - ClusterIdAttr: clusterId, - ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, - EndpointPortAttr: "80", - EndpointProtocolAttr: "TCP", - EndpointPortNameAttr: "http", - ServicePortNameAttr: "http", - ServiceProtocolAttr: "TCP", - ServicePortAttr: "30", - ServiceTargetPortAttr: "80", - ServiceTypeAttr: serviceType, - SvcExportCreationTimestampAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), - "custom-attr": "custom-val", + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, + EndpointIpv4Attr: ip, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", + EndpointHostnameAttr: "", + EndpointNodeNameAttr: "", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "30", + ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, + ServiceExportCreationAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), + "custom-attr": "custom-val", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := &Endpoint{ - Id: tt.fields.Id, - IP: tt.fields.IP, - EndpointPort: tt.fields.EndpointPort, - ServicePort: tt.fields.ServicePort, - ClusterId: tt.fields.ClusterId, - ClusterSetId: tt.fields.ClusterSetId, - ServiceType: tt.fields.ServiceType, - SvcExportCreationTimestamp: tt.fields.SvcExportCreationTimestamp, - Attributes: tt.fields.Attributes, - } - if got := e.GetCloudMapAttributes(); !reflect.DeepEqual(got, tt.want) { + if got := tt.endpoint.GetCloudMapAttributes(); !reflect.DeepEqual(got, tt.want) { t.Errorf("GetAttributes() = %v, want %v", got, tt.want) } }) diff --git a/samples/coredns-deployment.yaml b/samples/coredns-deployment.yaml index 5039a0f8..d5e3949b 100644 --- a/samples/coredns-deployment.yaml +++ b/samples/coredns-deployment.yaml @@ -58,7 +58,7 @@ spec: - args: - -conf - /etc/coredns/Corefile - image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.4 + image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.6 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -134,4 +134,4 @@ spec: - key: Corefile path: Corefile name: coredns - name: config-volume \ No newline at end of file + name: config-volume diff --git a/test/test-constants.go b/test/test-constants.go index a689be86..bd30053c 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -6,8 +6,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" ) @@ -19,13 +17,14 @@ const ( SvcName = "svc-name" SvcId = "svc-id" ClusterId1 = "test-mcs-clusterid-1" - ClusterSetId1 = "test-mcs-clustersetid-1" + ClusterSet = "test-mcs-clustersetid" ClusterId2 = "test-mcs-clusterid-2" - ClusterSetId2 = "test-mcs-clustersetid-2" EndptId1 = "tcp-192_168_0_1-1" EndptId2 = "tcp-192_168_0_2-2" EndptIp1 = "192.168.0.1" EndptIp2 = "192.168.0.2" + EndptReadyTrue = "true" + EndptReadyFalse = "false" Port1 = 1 PortStr1 = "1" PortName1 = "http" @@ -45,6 +44,8 @@ const ( OpStart = 1 SvcType = "ClusterSetIP" SvcExportCreationTimestamp int64 = 1640995200000 + Hostname = "host" + Nodename = "node" ) func GetTestHttpNamespace() *model.Namespace { @@ -103,11 +104,14 @@ func GetTestEndpoint1() *model.Endpoint { TargetPort: PortStr1, Protocol: Protocol1, }, - ClusterId: ClusterId1, - ClusterSetId: ClusterSetId1, - ServiceType: model.ClusterSetIPType, - SvcExportCreationTimestamp: SvcExportCreationTimestamp, - Attributes: make(map[string]string), + Ready: true, + Hostname: Hostname, + Nodename: Nodename, + ClusterId: ClusterId1, + ClusterSetId: ClusterSet, + ServiceType: model.ClusterSetIPType, + ServiceExportCreationTimestamp: SvcExportCreationTimestamp, + Attributes: make(map[string]string), } } @@ -126,11 +130,14 @@ func GetTestEndpoint2() *model.Endpoint { TargetPort: PortStr2, Protocol: Protocol2, }, - ClusterId: ClusterId1, - ClusterSetId: ClusterSetId1, - ServiceType: model.ClusterSetIPType, - SvcExportCreationTimestamp: SvcExportCreationTimestamp, - Attributes: make(map[string]string), + Ready: true, + Hostname: Hostname, + Nodename: Nodename, + ClusterId: ClusterId1, + ClusterSetId: ClusterSet, + ServiceType: model.ClusterSetIPType, + ServiceExportCreationTimestamp: SvcExportCreationTimestamp, + Attributes: make(map[string]string), } } @@ -157,7 +164,7 @@ func GetTestEndpoints(count int) (endpts []*model.Endpoint) { func ClusterIdForTest() *aboutv1alpha1.ClusterProperty { return &aboutv1alpha1.ClusterProperty{ ObjectMeta: metav1.ObjectMeta{ - Name: common.ClusterIdName, + Name: model.ClusterIdName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ Value: ClusterId1, @@ -168,10 +175,10 @@ func ClusterIdForTest() *aboutv1alpha1.ClusterProperty { func ClusterSetIdForTest() *aboutv1alpha1.ClusterProperty { return &aboutv1alpha1.ClusterProperty{ ObjectMeta: metav1.ObjectMeta{ - Name: common.ClusterSetIdName, + Name: model.ClusterSetIdName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ - Value: ClusterSetId1, + Value: ClusterSet, }, } } From 343244a1d48e25f21d233a6930c88541177c9eff Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Fri, 2 Sep 2022 12:22:52 -0700 Subject: [PATCH 110/163] Add User-Agent (#215) * Append aws-cloud-map-mcs-controller-for-k8s specific User-Agent to AWS Cloudmap Requests. * Add K8S_CONTROLLER attributes unit tests * Running goimports and lint --- Makefile | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 2 +- pkg/cloudmap/aws_facade.go | 9 ++++++++- pkg/cloudmap/client_test.go | 5 +++++ .../multicluster/serviceexport_controller.go | 10 +++------- .../multicluster/serviceexport_controller_test.go | 1 + pkg/version/version.go | 15 +++++++++++++++ test/test-constants.go | 11 +++++++++-- 8 files changed, 43 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index c141f0bd..302c42f4 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,7 @@ eks-test: ##@ Build .DEFAULT: build -build: manifests generate generate-mocks fmt vet lint ## Build manager binary. +build: manifests generate generate-mocks fmt vet goimports lint ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go run: manifests generate generate-mocks fmt vet ## Run a controller from your host. diff --git a/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go index f74564df..1eca515f 100644 --- a/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/cloudmap/aws_facade.go b/pkg/cloudmap/aws_facade.go index 5edf4312..76750c41 100644 --- a/pkg/cloudmap/aws_facade.go +++ b/pkg/cloudmap/aws_facade.go @@ -3,6 +3,9 @@ package cloudmap import ( "context" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" + "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" ) @@ -44,5 +47,9 @@ type awsFacade struct { // NewAwsFacadeFromConfig creates a new AWS facade from an AWS client config. func NewAwsFacadeFromConfig(cfg *aws.Config) AwsFacade { - return &awsFacade{sd.NewFromConfig(*cfg)} + sdClient := sd.NewFromConfig(*cfg, func(options *sd.Options) { + // Append User-Agent to all the request, the format is going to be aws-cloud-map-mcs-controller-for-k8s/0.0.0-abc + options.APIOptions = append(options.APIOptions, middleware.AddUserAgentKeyValue(version.GetUserAgentKey(), version.GetUserAgentValue())) + }) + return &awsFacade{sdClient} } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 7c2f8fc8..45b831c7 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -310,6 +310,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { model.EndpointHostnameAttr: test.Hostname, model.EndpointNodeNameAttr: test.Nodename, model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.K8sVersionAttr: test.PackageVersion, } attrs2 := map[string]string{ model.ClusterIdAttr: test.ClusterId1, @@ -327,6 +328,7 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { model.EndpointHostnameAttr: test.Hostname, model.EndpointNodeNameAttr: test.Nodename, model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.K8sVersionAttr: test.PackageVersion, } tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). @@ -370,6 +372,7 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { } func getTestSdClient(t *testing.T) *testSdClient { + test.SetTestVersion() mockController := gomock.NewController(t) mockCache := cloudmapMock.NewMockServiceDiscoveryClientCache(mockController) mockApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) @@ -409,6 +412,7 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { model.EndpointHostnameAttr: test.Hostname, model.EndpointNodeNameAttr: test.Nodename, model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.K8sVersionAttr: test.PackageVersion, }, }, { @@ -429,6 +433,7 @@ func getHttpInstanceSummaryForTest() []types.HttpInstanceSummary { model.EndpointHostnameAttr: test.Hostname, model.EndpointNodeNameAttr: test.Nodename, model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.K8sVersionAttr: test.PackageVersion, }, }, } diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index 4d53bc64..5bea631a 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -71,13 +71,11 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques namespacedName := types.NamespacedName{Namespace: serviceExport.Namespace, Name: serviceExport.Name} if err := r.Client.Get(ctx, namespacedName, &service); err != nil { if errors.IsNotFound(err) { - r.Log.Info("no Service found, deleting the ServiceExport", - "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) + r.Log.Info("no Service found, deleting the ServiceExport", "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) // Mark ServiceExport to be deleted, if the corresponding Service is not found isServiceExportMarkedForDelete = true } else { - r.Log.Error(err, "error fetching Service", - "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) + r.Log.Error(err, "error fetching Service", "Namespace", serviceExport.Namespace, "Name", serviceExport.Name) return ctrl.Result{}, nil } } @@ -238,9 +236,7 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. } attributes := make(map[string]string) - if version.GetVersion() != "" { - attributes[model.K8sVersionAttr] = version.PackageName + " " + version.GetVersion() - } + attributes[model.K8sVersionAttr] = version.GetPackageVersion() endpoints := make([]*model.Endpoint, 0) for _, slice := range endpointSlices.Items { diff --git a/pkg/controllers/multicluster/serviceexport_controller_test.go b/pkg/controllers/multicluster/serviceexport_controller_test.go index b78f8d5e..1386dcc2 100644 --- a/pkg/controllers/multicluster/serviceexport_controller_test.go +++ b/pkg/controllers/multicluster/serviceexport_controller_test.go @@ -205,6 +205,7 @@ func getServiceExportScheme() *runtime.Scheme { } func getServiceExportReconciler(t *testing.T, mockClient *cloudmapMock.MockServiceDiscoveryClient, client client.Client) *ServiceExportReconciler { + test.SetTestVersion() return &ServiceExportReconciler{ Client: client, Log: common.NewLoggerWithLogr(testr.New(t)), diff --git a/pkg/version/version.go b/pkg/version/version.go index b446912d..96a31e99 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -22,3 +22,18 @@ func GetVersion() string { return "" } + +func GetPackageVersion() string { + return PackageName + " " + GetVersion() +} + +func GetUserAgentKey() string { + return PackageName +} + +func GetUserAgentValue() string { + if GitVersion != "" { + return strings.TrimPrefix(GitVersion, "v") + } + return "" +} diff --git a/test/test-constants.go b/test/test-constants.go index bd30053c..a15cf02d 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -3,6 +3,7 @@ package test import ( "fmt" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" @@ -46,8 +47,14 @@ const ( SvcExportCreationTimestamp int64 = 1640995200000 Hostname = "host" Nodename = "node" + PackageVersion = "aws-cloud-map-mcs-controller-for-k8s 0.0.1 (abcd)" ) +func SetTestVersion() { + version.GitVersion = "v0.0.1" + version.GitCommit = "abcd" +} + func GetTestHttpNamespace() *model.Namespace { return &model.Namespace{ Id: HttpNsId, @@ -111,7 +118,7 @@ func GetTestEndpoint1() *model.Endpoint { ClusterSetId: ClusterSet, ServiceType: model.ClusterSetIPType, ServiceExportCreationTimestamp: SvcExportCreationTimestamp, - Attributes: make(map[string]string), + Attributes: map[string]string{model.K8sVersionAttr: PackageVersion}, } } @@ -137,7 +144,7 @@ func GetTestEndpoint2() *model.Endpoint { ClusterSetId: ClusterSet, ServiceType: model.ClusterSetIPType, ServiceExportCreationTimestamp: SvcExportCreationTimestamp, - Attributes: make(map[string]string), + Attributes: map[string]string{model.K8sVersionAttr: PackageVersion}, } } From 69d2a527df1233454492d3de05a34f26dfb0cb4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 12:25:39 -0700 Subject: [PATCH 111/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.17.2 to 1.17.4 (#214) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.17.2 to 1.17.4. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.17.2...config/v1.17.4) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 24 ++++++++++++------------ go.sum | 44 ++++++++++++++++++++++++-------------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 8b594ab2..2f63162e 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.12 - github.com/aws/aws-sdk-go-v2/config v1.17.2 + github.com/aws/aws-sdk-go-v2 v1.16.13 + github.com/aws/aws-sdk-go-v2/config v1.17.4 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -29,16 +29,16 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.15 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.18 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.14 // indirect - github.com/aws/smithy-go v1.13.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.17 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 // indirect + github.com/aws/smithy-go v1.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 39f5fd5a..30ae3901 100644 --- a/go.sum +++ b/go.sum @@ -74,32 +74,36 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA= github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= -github.com/aws/aws-sdk-go-v2/config v1.17.2 h1:V96WPd2a1H/MXGZjk4zto+KpYnwZI2kdIdy/cI8kYnQ= -github.com/aws/aws-sdk-go-v2/config v1.17.2/go.mod h1:jumS/AMwul4WaG8vyXsF6kUndG9zndR+yfYBwl4i9ds= -github.com/aws/aws-sdk-go-v2/credentials v1.12.15 h1:6DONxG9cR3pAuISj1Irh5u2SRqCfIJwyHNyDDes7SZw= -github.com/aws/aws-sdk-go-v2/credentials v1.12.15/go.mod h1:41zTC6U/78fUD7ZCa5NymTJANDjfqySg5YEAYVFl2Ic= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 h1:+uferi8SUDZtMloCDt24Zenyy/i71C/ua5mjUCpbpN0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13/go.mod h1:y0eXmsNBFIVjUE8ZBjES8myOHlMsXDz7qGT93+MVdjk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 h1:gC5mudiFrWGhzcdoWj1iCGUfrzCpQG0MQIQf0CXFFQQ= +github.com/aws/aws-sdk-go-v2 v1.16.13 h1:HgF7OX2q0gSZtcXoo9DMEA8A2Qk/GCxmWyM0RI7Yz2Y= +github.com/aws/aws-sdk-go-v2 v1.16.13/go.mod h1:xSyvSnzh0KLs5H4HJGeIEsNYemUWdNIl0b/rP6SIsLU= +github.com/aws/aws-sdk-go-v2/config v1.17.4 h1:9HY1wbShqObySCHP2Z07blfrSWVX+nVxCZmUuLZKcG8= +github.com/aws/aws-sdk-go-v2/config v1.17.4/go.mod h1:ul+ru+huVpfduF9XRmGUq82T8T3K+nIFQuF6F+L+548= +github.com/aws/aws-sdk-go-v2/credentials v1.12.17 h1:htUjIJOQcvIUR0jC4eLkdis1DfaLL4EUbIKUFqh2WFA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.17/go.mod h1:jd1mvJulXY7ccHvcSiJceYhv06yWIIRkJnwWEA4IX+g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 h1:NZwZFtxXGOEIiCd8jWN55lexakug543CaO68bTpoLwg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14/go.mod h1:5CU57SyF5jZLSIw4OOll0PG83ThXwNdkRFOc0EltD/0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19/go.mod h1:llxE6bwUZhuCas0K7qGiu5OgMis3N7kdWtFSxoHmJ7E= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 h1:qezY57na06d6kSE7uuB0N7XEflu914AXx/hg2L8Ykcw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20 h1:Rk8eqZSdFovt8Id+O+i2qT0c3CY13DPn2SfGOEVlxNs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20/go.mod h1:gdZ5gRUaxThXIZyZQ8MTtgYBk2jbHgp05BO3GcD9Cwc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13/go.mod h1:lB12mkZqCSo5PsdBFLNqc2M/OOYgNAy8UtaktyuWvE8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 h1:GvszACAU8GSV3+Tant5GutW6smY8WavrP8ZuRS9Ku4Q= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20/go.mod h1:bfTcsThj5a9P5pIGRy0QudJ8k4+issxXX+O6Djnd5Cs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 h1:ObfthqDyhe7rMAOa7pqft6974VHIk8BAJB7kYdoIfTA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13/go.mod h1:V390DK4MQxLpDdXxFqizyz8KUxuWImkW/xzgXMz0yyk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14 h1:6Yxuq9yrkoLYab5JXqJnto9tdRuIcYVdR+eiKjsJYWU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14/go.mod h1:GEV9jaDPIgayiU+uevxwozcvUOjc+P4aHE2BeSjm2vE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 h1:lpwSbLKYTuABo6SyUoC25xAmfO3/TehGS2SmD1EtOL0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21/go.mod h1:Q0pktZjvRZk77TBto6yAvUAi7fcse1bdcMctBDVGgBw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 h1:c5hJNN2DkK1gAytcKp7LkiKNDJeevFSboPezEHAM4Ro= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14/go.mod h1:8qOLjqMzY/S1kh3myDXA1yxK5eD4uN8aOJgKpgvc4OM= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 h1:pGdf7v3gzWs0zaocsz1Pshya2gqMIx8FUsnsN5YlWhQ= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14/go.mod h1:kJHF1cJv3Ua31h5UpHKO+9KJsrg9TTMiHCattw4KJpc= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.18 h1:gTn1a/FbcOXK5LQS88dD5k+PKwyjVvhAEEwyN4c6eW8= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.18/go.mod h1:ytmEi5+qwcSNcV2pVA8PIb1DnKT/0Bu/K4nfJHwoM6c= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 h1:p48IfndYbRk3iDsoQAmVXdCKEM5+7Y50JAPikjwk8gI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1/go.mod h1:NY+G+8PW0ISyJ7/6t5mgOe6qpJiwZa9Jix05WPscJjg= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.14 h1:7kxso8VZLQ86Jg27QRBw6fjrQhQ8CMNMZ7SB0w7RQiA= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.14/go.mod h1:Y+BUV19q3OmQVqNUlbZ40zVi3NM6Biuxwkx/qdSD/CY= -github.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 h1:3raP0UC9rvRyY4/cc4o4F3jTrNo94AYiarNUGNnq6dU= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.20/go.mod h1:hPsROgDdgY/NQ1gPt7VJWG0GjSnalDC0DkkMfGEw2gc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 h1:/SYpdjjAtraymql+/r719OgjxezdanAQiLb/NMxDb04= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2/go.mod h1:5cxfDYtY2mDOlmesy4yycb6lwyy1U/iAUOHKhQLKw/E= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 h1:otZvq9r+xjPL7qU/luX2QdBamiN+oSZURRi4sAKymO8= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.16/go.mod h1:Y9iBgT1w2vHtYzJEkwD6FqILjDSsvbxcW/+wIYxyse4= github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.13.1 h1:q09BdpUiaqpothcv393ACfWJJHzlzjB5HaNL1XHKmoQ= +github.com/aws/smithy-go v1.13.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From e6571cff95bccb2f9643d740c414f662beab87a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 13:20:14 -0700 Subject: [PATCH 112/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#218) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.14 to 1.17.16. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.14...service/ecr/v1.17.16) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 2f63162e..0cc16ebc 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.13 + github.com/aws/aws-sdk-go-v2 v1.16.14 github.com/aws/aws-sdk-go-v2/config v1.17.4 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.16 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 @@ -31,14 +31,14 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.17 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 // indirect - github.com/aws/smithy-go v1.13.1 // indirect + github.com/aws/smithy-go v1.13.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 30ae3901..6ae797ea 100644 --- a/go.sum +++ b/go.sum @@ -74,36 +74,36 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= -github.com/aws/aws-sdk-go-v2 v1.16.13 h1:HgF7OX2q0gSZtcXoo9DMEA8A2Qk/GCxmWyM0RI7Yz2Y= github.com/aws/aws-sdk-go-v2 v1.16.13/go.mod h1:xSyvSnzh0KLs5H4HJGeIEsNYemUWdNIl0b/rP6SIsLU= +github.com/aws/aws-sdk-go-v2 v1.16.14 h1:db6GvO4Z2UqHt5gvT0lr6J5x5P+oQ7bdRzczVaRekMU= +github.com/aws/aws-sdk-go-v2 v1.16.14/go.mod h1:s/G+UV29dECbF5rf+RNj1xhlmvoNurGSr+McVSRj59w= github.com/aws/aws-sdk-go-v2/config v1.17.4 h1:9HY1wbShqObySCHP2Z07blfrSWVX+nVxCZmUuLZKcG8= github.com/aws/aws-sdk-go-v2/config v1.17.4/go.mod h1:ul+ru+huVpfduF9XRmGUq82T8T3K+nIFQuF6F+L+548= github.com/aws/aws-sdk-go-v2/credentials v1.12.17 h1:htUjIJOQcvIUR0jC4eLkdis1DfaLL4EUbIKUFqh2WFA= github.com/aws/aws-sdk-go-v2/credentials v1.12.17/go.mod h1:jd1mvJulXY7ccHvcSiJceYhv06yWIIRkJnwWEA4IX+g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 h1:NZwZFtxXGOEIiCd8jWN55lexakug543CaO68bTpoLwg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14/go.mod h1:5CU57SyF5jZLSIw4OOll0PG83ThXwNdkRFOc0EltD/0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19/go.mod h1:llxE6bwUZhuCas0K7qGiu5OgMis3N7kdWtFSxoHmJ7E= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20 h1:Rk8eqZSdFovt8Id+O+i2qT0c3CY13DPn2SfGOEVlxNs= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20/go.mod h1:gdZ5gRUaxThXIZyZQ8MTtgYBk2jbHgp05BO3GcD9Cwc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13/go.mod h1:lB12mkZqCSo5PsdBFLNqc2M/OOYgNAy8UtaktyuWvE8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14 h1:6Yxuq9yrkoLYab5JXqJnto9tdRuIcYVdR+eiKjsJYWU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 h1:gRIXnmAVNyoRQywdNtpAkgY+f30QNzgF53Q5OobNZZs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21/go.mod h1:XsmHMV9c512xgsW01q7H0ut+UQQQpWX8QsFbdLHDwaU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14/go.mod h1:GEV9jaDPIgayiU+uevxwozcvUOjc+P4aHE2BeSjm2vE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 h1:noAhOo2mMDyYhTx99aYPvQw16T3fQ/DiKAv9fzpIKH8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15/go.mod h1:kjJ4CyD9M3Wq88GYg3IPfj67Rs0Uvz8aXK7MJ8BvE4I= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 h1:lpwSbLKYTuABo6SyUoC25xAmfO3/TehGS2SmD1EtOL0= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21/go.mod h1:Q0pktZjvRZk77TBto6yAvUAi7fcse1bdcMctBDVGgBw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 h1:c5hJNN2DkK1gAytcKp7LkiKNDJeevFSboPezEHAM4Ro= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14/go.mod h1:8qOLjqMzY/S1kh3myDXA1yxK5eD4uN8aOJgKpgvc4OM= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14 h1:pGdf7v3gzWs0zaocsz1Pshya2gqMIx8FUsnsN5YlWhQ= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.14/go.mod h1:kJHF1cJv3Ua31h5UpHKO+9KJsrg9TTMiHCattw4KJpc= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.16 h1:5lQdDB46maKPDfN1M93VnFBzUDdUL2tzYFztD6Uuc94= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.16/go.mod h1:FGLudsvFePDe6/KU8xnQLtNXML7DgUyd0OQfM27P0fM= github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 h1:3raP0UC9rvRyY4/cc4o4F3jTrNo94AYiarNUGNnq6dU= github.com/aws/aws-sdk-go-v2/service/sso v1.11.20/go.mod h1:hPsROgDdgY/NQ1gPt7VJWG0GjSnalDC0DkkMfGEw2gc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 h1:/SYpdjjAtraymql+/r719OgjxezdanAQiLb/NMxDb04= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2/go.mod h1:5cxfDYtY2mDOlmesy4yycb6lwyy1U/iAUOHKhQLKw/E= github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 h1:otZvq9r+xjPL7qU/luX2QdBamiN+oSZURRi4sAKymO8= github.com/aws/aws-sdk-go-v2/service/sts v1.16.16/go.mod h1:Y9iBgT1w2vHtYzJEkwD6FqILjDSsvbxcW/+wIYxyse4= -github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.13.1 h1:q09BdpUiaqpothcv393ACfWJJHzlzjB5HaNL1XHKmoQ= github.com/aws/smithy-go v1.13.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.13.2 h1:TBLKyeJfXTrTXRHmsv4qWt9IQGYyWThLYaJWSahTOGE= +github.com/aws/smithy-go v1.13.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 9fbb77cf3f8b037f8cae798075d5ee029858fe94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 13:45:18 -0700 Subject: [PATCH 113/163] Bump github.com/onsi/gomega from 1.20.1 to 1.20.2 (#213) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.20.1 to 1.20.2. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.20.1...v1.20.2) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 0cc16ebc..cd011d2e 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.20.1 + github.com/onsi/gomega v1.20.2 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.0 k8s.io/api v0.24.3 @@ -75,9 +75,9 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect diff --git a/go.sum b/go.sum index 6ae797ea..0363ccaf 100644 --- a/go.sum +++ b/go.sum @@ -408,16 +408,18 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= +github.com/onsi/gomega v1.20.2/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -510,6 +512,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -604,6 +607,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -654,8 +658,9 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -681,6 +686,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -748,8 +754,10 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -830,6 +838,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From abf360c204aa3fd2dc328422390a5f1c2374e168 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Wed, 7 Sep 2022 10:50:45 -0700 Subject: [PATCH 114/163] Add a command to download goimports binary, this is to fix the failing integration tests (#220) --- Makefile | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 302c42f4..a9c1c127 100644 --- a/Makefile +++ b/Makefile @@ -60,24 +60,19 @@ mod: tidy: go mod tidy -GOLANGCI_LINT=$(shell pwd)/bin/golangci-lint -golangci-lint: ## Download golangci-lint -ifneq ($(shell test -f $(GOLANGCI_LINT); echo $$?), 0) - @echo Getting golangci-lint... - @mkdir -p $(shell pwd)/bin - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.46.2 -endif - .PHONY: lint lint: golangci-lint ## Run linter $(GOLANGCI_LINT) run .PHONY: goimports -goimports: ## run goimports updating files in place - goimports -w . +goimports: goimports-bin ## run goimports updating files in place + $(GOIMPORTS) -w . +# Run tests ENVTEST_ASSETS_DIR=$(shell pwd)/testbin KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)" + +.PHONY: test test: manifests generate generate-mocks fmt vet test-setup ## Run tests KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out -covermode=atomic @@ -122,10 +117,10 @@ eks-test: ##@ Build .DEFAULT: build -build: manifests generate generate-mocks fmt vet goimports lint ## Build manager binary. +build: test goimports lint ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go -run: manifests generate generate-mocks fmt vet ## Run a controller from your host. +run: test ## Run a controller from your host. go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go --zap-devel=true $(ARGS) docker-build: test ## Build docker image with the manager. @@ -186,6 +181,18 @@ MOCKGEN = $(shell pwd)/bin/mockgen mockgen: ## Download mockgen $(call go-get-tool,$(MOCKGEN),github.com/golang/mock/mockgen@v1.6.0) +GOLANGCI_LINT=$(shell pwd)/bin/golangci-lint +golangci-lint: ## Download golangci-lint +ifneq ($(shell test -f $(GOLANGCI_LINT); echo $$?), 0) + @echo Getting golangci-lint... + @mkdir -p $(shell pwd)/bin + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.46.2 +endif + +GOIMPORTS = $(shell pwd)/bin/goimports +goimports-bin: ## Download mockgen + $(call go-get-tool,$(GOIMPORTS),golang.org/x/tools/cmd/goimports@v0.1.12) + KIND = $(shell pwd)/bin/kind kind: ## Download kind $(call go-get-tool,$(KIND),sigs.k8s.io/kind@v0.14.0) From 5a8f1751d1a337bb824d2e19bbbb61b7dab871d4 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Thu, 8 Sep 2022 15:39:49 -0700 Subject: [PATCH 115/163] Update the ClusterProperty id.k8s.io to cluster.clusterset.k8s.io (#221) --- CONTRIBUTING.md | 6 ++--- README.md | 24 ++++++++----------- ...ut_v1alpha1_clusterproperty_clusterId.yaml | 8 ------- ...v1alpha1_clusterproperty_clustersetId.yaml | 8 ------- .../multicluster_v1alpha1_serviceexport.yaml | 7 ------ .../multicluster_v1alpha1_serviceimport.yaml | 7 ------ .../configs/e2e-clusterproperty-1.yaml | 4 ++-- .../configs/e2e-clusterproperty-2.yaml | 4 ++-- .../kind-test/configs/dnsutils-pod.yaml | 2 +- .../configs/e2e-clusterproperty.yaml | 4 ++-- integration/kind-test/scripts/common.sh | 1 - integration/kind-test/scripts/dns-test.sh | 6 ++--- .../v1alpha1/zz_generated.deepcopy.go | 2 +- pkg/model/cluster.go | 8 +++---- pkg/model/cluster_test.go | 4 ++-- samples/example-clusterproperty.yaml | 6 ++--- test/test-constants.go | 4 ++-- 17 files changed, 35 insertions(+), 70 deletions(-) delete mode 100644 config/samples/about_v1alpha1_clusterproperty_clusterId.yaml delete mode 100644 config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml delete mode 100644 config/samples/multicluster_v1alpha1_serviceexport.yaml delete mode 100644 config/samples/multicluster_v1alpha1_serviceimport.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index beae2a57..dd948880 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,13 +101,13 @@ make install # customresourcedefinition.apiextensions.k8s.io/serviceimports.multicluster.x-k8s.io created ``` -Register a unique `id.k8s.io` and `clusterset.k8s.io` in your cluster: +Register a unique `cluster.clusterset.k8s.io` and `clusterset.k8s.io` in your cluster: ```bash kubectl apply -f samples/example-clusterproperty.yaml -# clusterproperty.about.k8s.io/id.k8s.io created +# clusterproperty.about.k8s.io/cluster.clusterset.k8s.io created # clusterproperty.about.k8s.io/clusterset.k8s.io created ``` -> ⚠ **Note:** If you are creating multiple clusters, ensure you create unique `id.k8s.io` identifiers for each cluster. +> ⚠ **Note:** If you are creating multiple clusters, ensure you create unique `cluster.clusterset.k8s.io` identifiers for each cluster. To run the controller, run the following command. The controller runs in an infinite loop so open another terminal to create CRDs. (Ctrl+C to exit) diff --git a/README.md b/README.md index 6d4f8066..aa516ab4 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/aws/aws-cloud-map-mcs-controller-for-k8s)](https://goreportcard.com/report/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) ## Introduction -The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [multi-cluster services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) specification, which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. - -See the demo from AWS Container Day x KubeCon! - -[![Watch the video](https://img.youtube.com/vi/3f0Tv7IiQQw/0.jpg)](https://youtu.be/3f0Tv7IiQQw?t=24458) +The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. ## Installation @@ -37,7 +33,7 @@ Perform the following installation steps on each participating cluster. #### Configure CoreDNS -Install the The CoreDNS multicluster plugin into each participating cluster. The multicluster plugin enables CoreDNS to lifecycle manage DNS records for `ServiceImport` objects. +Install the CoreDNS multicluster plugin into each participating cluster. The multicluster plugin enables CoreDNS to lifecycle manage DNS records for `ServiceImport` objects. To install the plugin, run the following commands. @@ -63,26 +59,26 @@ The controller must have sufficient IAM permissions to perform required Cloud Ma ## Usage -### Configure `id.k8s.io` and `clusterset.k8s.io` +### Configure `cluster.clusterset.k8s.io` and `clusterset.k8s.io` -`id.k8s.io` is a unique identifier that uniquely identifies a cluster. +`cluster.clusterset.k8s.io` is a unique identifier for the cluster. -`clusterset.k8s.io` is a unique identifier that uniquely identifies the set of clusters in your multicluster. This will be the same across all clusters in the set. +`clusterset.k8s.io` is an identifier that relates to the `ClusterSet` in which the cluster belongs. ```yaml apiVersion: about.k8s.io/v1alpha1 kind: ClusterProperty metadata: - name: id.k8s.io + name: cluster.clusterset.k8s.io spec: - value: [Your cluster identifier] + value: [Your Cluster identifier] --- apiVersion: about.k8s.io/v1alpha1 kind: ClusterProperty metadata: name: clusterset.k8s.io spec: - value: [Your cluster set identifier] + value: [Your ClusterSet identifier] ``` **Example:** @@ -90,7 +86,7 @@ spec: apiVersion: about.k8s.io/v1alpha1 kind: ClusterProperty metadata: - name: id.k8s.io + name: cluster.clusterset.k8s.io spec: value: my-first-cluster --- @@ -178,4 +174,4 @@ Join the channel with this [invite](https://join.slack.com/t/awsappmesh/shared_i This project is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0), -see [LICENSE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/LICENSE) and [NOTICE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/NOTICE) for more information. \ No newline at end of file +see [LICENSE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/LICENSE) and [NOTICE](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/blob/main/NOTICE) for more information. diff --git a/config/samples/about_v1alpha1_clusterproperty_clusterId.yaml b/config/samples/about_v1alpha1_clusterproperty_clusterId.yaml deleted file mode 100644 index 76bc3b02..00000000 --- a/config/samples/about_v1alpha1_clusterproperty_clusterId.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# An example object of `id.k8s.io ClusterProperty` - -apiVersion: about.k8s.io/v1alpha1 -kind: ClusterProperty -metadata: - name: id.k8s.io -spec: - value: sample-mcs-clusterid diff --git a/config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml b/config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml deleted file mode 100644 index 10d3e82c..00000000 --- a/config/samples/about_v1alpha1_clusterproperty_clustersetId.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# An example object of `clusterset.k8s.io ClusterProperty`: - -apiVersion: about.k8s.io/v1alpha1 -kind: ClusterProperty -metadata: - name: clusterset.k8s.io -spec: - value: sample-mcs-clustersetid \ No newline at end of file diff --git a/config/samples/multicluster_v1alpha1_serviceexport.yaml b/config/samples/multicluster_v1alpha1_serviceexport.yaml deleted file mode 100644 index e53b0abf..00000000 --- a/config/samples/multicluster_v1alpha1_serviceexport.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: multicluster.x-k8s.io/v1alpha1 -kind: ServiceExport -metadata: - name: serviceexport-sample -spec: - # Add fields here - foo: bar diff --git a/config/samples/multicluster_v1alpha1_serviceimport.yaml b/config/samples/multicluster_v1alpha1_serviceimport.yaml deleted file mode 100644 index 57515e17..00000000 --- a/config/samples/multicluster_v1alpha1_serviceimport.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: multicluster.x-k8s.io/v1alpha1 -kind: ServiceImport -metadata: - name: serviceimport-sample -spec: - # Add fields here - foo: bar diff --git a/integration/eks-test/configs/e2e-clusterproperty-1.yaml b/integration/eks-test/configs/e2e-clusterproperty-1.yaml index 8371963f..51b6f49a 100644 --- a/integration/eks-test/configs/e2e-clusterproperty-1.yaml +++ b/integration/eks-test/configs/e2e-clusterproperty-1.yaml @@ -1,7 +1,7 @@ apiVersion: about.k8s.io/v1alpha1 kind: ClusterProperty metadata: - name: id.k8s.io + name: cluster.clusterset.k8s.io spec: value: eks-e2e-clusterid-1 --- @@ -10,4 +10,4 @@ kind: ClusterProperty metadata: name: clusterset.k8s.io spec: - value: eks-e2e-clustersetid-1 \ No newline at end of file + value: eks-e2e-clustersetid-1 diff --git a/integration/eks-test/configs/e2e-clusterproperty-2.yaml b/integration/eks-test/configs/e2e-clusterproperty-2.yaml index 239aa8fa..dc4a45bf 100644 --- a/integration/eks-test/configs/e2e-clusterproperty-2.yaml +++ b/integration/eks-test/configs/e2e-clusterproperty-2.yaml @@ -1,7 +1,7 @@ apiVersion: about.k8s.io/v1alpha1 kind: ClusterProperty metadata: - name: id.k8s.io + name: cluster.clusterset.k8s.io spec: value: eks-e2e-clusterid-2 --- @@ -10,4 +10,4 @@ kind: ClusterProperty metadata: name: clusterset.k8s.io spec: - value: eks-e2e-clustersetid-1 \ No newline at end of file + value: eks-e2e-clustersetid-1 diff --git a/integration/kind-test/configs/dnsutils-pod.yaml b/integration/kind-test/configs/dnsutils-pod.yaml index 9d408d78..beb5e89b 100644 --- a/integration/kind-test/configs/dnsutils-pod.yaml +++ b/integration/kind-test/configs/dnsutils-pod.yaml @@ -8,7 +8,7 @@ spec: - command: - sleep - "3600" - image: registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3 + image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3 name: dnsutils imagePullPolicy: IfNotPresent restartPolicy: Always diff --git a/integration/kind-test/configs/e2e-clusterproperty.yaml b/integration/kind-test/configs/e2e-clusterproperty.yaml index bfc24e13..64f700ae 100644 --- a/integration/kind-test/configs/e2e-clusterproperty.yaml +++ b/integration/kind-test/configs/e2e-clusterproperty.yaml @@ -1,7 +1,7 @@ apiVersion: about.k8s.io/v1alpha1 kind: ClusterProperty metadata: - name: id.k8s.io + name: cluster.clusterset.k8s.io spec: value: kind-e2e-clusterid-1 --- @@ -10,4 +10,4 @@ kind: ClusterProperty metadata: name: clusterset.k8s.io spec: - value: kind-e2e-clustersetid-1 \ No newline at end of file + value: kind-e2e-clustersetid-1 diff --git a/integration/kind-test/scripts/common.sh b/integration/kind-test/scripts/common.sh index fd079b5e..c038c3a6 100755 --- a/integration/kind-test/scripts/common.sh +++ b/integration/kind-test/scripts/common.sh @@ -15,7 +15,6 @@ export KIND_SHORT='cloud-map-e2e' export CLUSTER='kind-cloud-map-e2e' export CLUSTERID1='kind-e2e-clusterid-1' export CLUSTERSETID1='kind-e2e-clustersetid-1' -export DNS_POD='dnsutils' export IMAGE='kindest/node:v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207' export EXPECTED_ENDPOINT_COUNT=5 export UPDATED_ENDPOINT_COUNT=6 diff --git a/integration/kind-test/scripts/dns-test.sh b/integration/kind-test/scripts/dns-test.sh index eeed7946..95463b54 100755 --- a/integration/kind-test/scripts/dns-test.sh +++ b/integration/kind-test/scripts/dns-test.sh @@ -27,12 +27,12 @@ expected_endpoint_count=$1 # Install dnsutils pod $KUBECTL_BIN apply -f "$KIND_CONFIGS/dnsutils-pod.yaml" -$KUBECTL_BIN wait --for=condition=ready pod/$DNS_POD # wait until pod is deployed +$KUBECTL_BIN wait --for=condition=ready pod/dnsutils # wait until pod is deployed # Perform a dig to cluster-local CoreDNS # TODO: parse dig outputs for more precise verification - check specifics IPs? echo "performing dig for A/AAAA records..." -addresses=$($KUBECTL_BIN exec $DNS_POD -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) +addresses=$($KUBECTL_BIN exec dnsutils -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) exit_code=$? echo "$addresses" @@ -45,7 +45,7 @@ fi checkDNS "$addresses" echo "performing dig for SRV records..." -addresses=$($KUBECTL_BIN exec $DNS_POD -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local. SRV +short) +addresses=$($KUBECTL_BIN exec dnsutils -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local. SRV +short) exit_code=$? echo "$addresses" diff --git a/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go index 1eca515f..f74564df 100644 --- a/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/model/cluster.go b/pkg/model/cluster.go index 97f057bc..1b008733 100644 --- a/pkg/model/cluster.go +++ b/pkg/model/cluster.go @@ -10,8 +10,8 @@ import ( ) const ( - ClusterIdName = "id.k8s.io" - ClusterSetIdName = "clusterset.k8s.io" + ClusterIdPropertyName = "cluster.clusterset.k8s.io" + ClusterSetIdPropertyName = "clusterset.k8s.io" ) // Non-exported type, accessible via read-only func @@ -73,9 +73,9 @@ func (r *ClusterUtils) LoadClusterProperties(ctx context.Context) error { } for _, clusterProperty := range clusterPropertyList.Items { switch clusterProperty.Name { - case ClusterIdName: + case ClusterIdPropertyName: r.clusterProperties.clusterId = clusterProperty.Spec.Value - case ClusterSetIdName: + case ClusterSetIdPropertyName: r.clusterProperties.clusterSetId = clusterProperty.Spec.Value } } diff --git a/pkg/model/cluster_test.go b/pkg/model/cluster_test.go index fad36427..2dbb6db1 100644 --- a/pkg/model/cluster_test.go +++ b/pkg/model/cluster_test.go @@ -82,7 +82,7 @@ func TestClusterUtils_GetClusterProperties(t *testing.T) { func ClusterIdForTest(clusterId string) *aboutv1alpha1.ClusterProperty { return &aboutv1alpha1.ClusterProperty{ ObjectMeta: metav1.ObjectMeta{ - Name: ClusterIdName, + Name: ClusterIdPropertyName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ Value: clusterId, @@ -93,7 +93,7 @@ func ClusterIdForTest(clusterId string) *aboutv1alpha1.ClusterProperty { func ClusterSetIdForTest(clusterSetId string) *aboutv1alpha1.ClusterProperty { return &aboutv1alpha1.ClusterProperty{ ObjectMeta: metav1.ObjectMeta{ - Name: ClusterSetIdName, + Name: ClusterSetIdPropertyName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ Value: clusterSetId, diff --git a/samples/example-clusterproperty.yaml b/samples/example-clusterproperty.yaml index 295b6756..4d058187 100644 --- a/samples/example-clusterproperty.yaml +++ b/samples/example-clusterproperty.yaml @@ -1,9 +1,9 @@ -# An example object of `id.k8s.io ClusterProperty` +# An example object of `cluster.clusterset.k8s.io ClusterProperty` apiVersion: about.k8s.io/v1alpha1 kind: ClusterProperty metadata: - name: id.k8s.io + name: cluster.clusterset.k8s.io spec: value: sample-mcs-clusterid --- @@ -14,4 +14,4 @@ kind: ClusterProperty metadata: name: clusterset.k8s.io spec: - value: sample-mcs-clustersetid \ No newline at end of file + value: sample-mcs-clustersetid diff --git a/test/test-constants.go b/test/test-constants.go index a15cf02d..ecf50f8c 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -171,7 +171,7 @@ func GetTestEndpoints(count int) (endpts []*model.Endpoint) { func ClusterIdForTest() *aboutv1alpha1.ClusterProperty { return &aboutv1alpha1.ClusterProperty{ ObjectMeta: metav1.ObjectMeta{ - Name: model.ClusterIdName, + Name: model.ClusterIdPropertyName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ Value: ClusterId1, @@ -182,7 +182,7 @@ func ClusterIdForTest() *aboutv1alpha1.ClusterProperty { func ClusterSetIdForTest() *aboutv1alpha1.ClusterProperty { return &aboutv1alpha1.ClusterProperty{ ObjectMeta: metav1.ObjectMeta{ - Name: model.ClusterSetIdName, + Name: model.ClusterSetIdPropertyName, }, Spec: aboutv1alpha1.ClusterPropertySpec{ Value: ClusterSet, From d07e6802ac50ec35fe52113c7bd6699869107521 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Thu, 15 Sep 2022 12:46:58 -0700 Subject: [PATCH 116/163] v0.3.0 release (#222) --- config/controller_install_release/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index ff9558c1..b870ff22 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -4,4 +4,4 @@ bases: images: - name: controller newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s - newTag: v0.2.3 + newTag: v0.3.0 From e77daac19f352727b8a371911002953d1208d168 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 11:14:13 -0700 Subject: [PATCH 117/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#230) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.16 to 1.17.18. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.16...service/ecr/v1.17.18) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index cd011d2e..3af3cf3e 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.14 + github.com/aws/aws-sdk-go-v2 v1.16.16 github.com/aws/aws-sdk-go-v2/config v1.17.4 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.16 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 @@ -31,14 +31,14 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.17 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 // indirect - github.com/aws/smithy-go v1.13.2 // indirect + github.com/aws/smithy-go v1.13.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 0363ccaf..57666443 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.16.13/go.mod h1:xSyvSnzh0KLs5H4HJGeIEsNYemUWdNIl0b/rP6SIsLU= -github.com/aws/aws-sdk-go-v2 v1.16.14 h1:db6GvO4Z2UqHt5gvT0lr6J5x5P+oQ7bdRzczVaRekMU= -github.com/aws/aws-sdk-go-v2 v1.16.14/go.mod h1:s/G+UV29dECbF5rf+RNj1xhlmvoNurGSr+McVSRj59w= +github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= +github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= github.com/aws/aws-sdk-go-v2/config v1.17.4 h1:9HY1wbShqObySCHP2Z07blfrSWVX+nVxCZmUuLZKcG8= github.com/aws/aws-sdk-go-v2/config v1.17.4/go.mod h1:ul+ru+huVpfduF9XRmGUq82T8T3K+nIFQuF6F+L+548= github.com/aws/aws-sdk-go-v2/credentials v1.12.17 h1:htUjIJOQcvIUR0jC4eLkdis1DfaLL4EUbIKUFqh2WFA= @@ -84,17 +84,17 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.17/go.mod h1:jd1mvJulXY7ccHvcSiJc github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 h1:NZwZFtxXGOEIiCd8jWN55lexakug543CaO68bTpoLwg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14/go.mod h1:5CU57SyF5jZLSIw4OOll0PG83ThXwNdkRFOc0EltD/0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20/go.mod h1:gdZ5gRUaxThXIZyZQ8MTtgYBk2jbHgp05BO3GcD9Cwc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 h1:gRIXnmAVNyoRQywdNtpAkgY+f30QNzgF53Q5OobNZZs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21/go.mod h1:XsmHMV9c512xgsW01q7H0ut+UQQQpWX8QsFbdLHDwaU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14/go.mod h1:GEV9jaDPIgayiU+uevxwozcvUOjc+P4aHE2BeSjm2vE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 h1:noAhOo2mMDyYhTx99aYPvQw16T3fQ/DiKAv9fzpIKH8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15/go.mod h1:kjJ4CyD9M3Wq88GYg3IPfj67Rs0Uvz8aXK7MJ8BvE4I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 h1:lpwSbLKYTuABo6SyUoC25xAmfO3/TehGS2SmD1EtOL0= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21/go.mod h1:Q0pktZjvRZk77TBto6yAvUAi7fcse1bdcMctBDVGgBw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 h1:c5hJNN2DkK1gAytcKp7LkiKNDJeevFSboPezEHAM4Ro= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14/go.mod h1:8qOLjqMzY/S1kh3myDXA1yxK5eD4uN8aOJgKpgvc4OM= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.16 h1:5lQdDB46maKPDfN1M93VnFBzUDdUL2tzYFztD6Uuc94= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.16/go.mod h1:FGLudsvFePDe6/KU8xnQLtNXML7DgUyd0OQfM27P0fM= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 h1:5/CGPA7fMLZwdXJBpx4lkueY92pOqI5B+qc/iWki7xI= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18/go.mod h1:v+R9xFTquOHSVwn36gO9MOwgbINQNh2m0/Y14CsB1bI= github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 h1:3raP0UC9rvRyY4/cc4o4F3jTrNo94AYiarNUGNnq6dU= github.com/aws/aws-sdk-go-v2/service/sso v1.11.20/go.mod h1:hPsROgDdgY/NQ1gPt7VJWG0GjSnalDC0DkkMfGEw2gc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 h1:/SYpdjjAtraymql+/r719OgjxezdanAQiLb/NMxDb04= @@ -102,8 +102,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2/go.mod h1:5cxfDYtY2mDOlmesy github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 h1:otZvq9r+xjPL7qU/luX2QdBamiN+oSZURRi4sAKymO8= github.com/aws/aws-sdk-go-v2/service/sts v1.16.16/go.mod h1:Y9iBgT1w2vHtYzJEkwD6FqILjDSsvbxcW/+wIYxyse4= github.com/aws/smithy-go v1.13.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.13.2 h1:TBLKyeJfXTrTXRHmsv4qWt9IQGYyWThLYaJWSahTOGE= -github.com/aws/smithy-go v1.13.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= +github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 78c6982233b0d484a6b5c47fbfe65be936012ff3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 12:20:26 -0700 Subject: [PATCH 118/163] Bump github.com/google/go-cmp from 0.5.8 to 0.5.9 (#231) Bumps [github.com/google/go-cmp](https://github.com/google/go-cmp) from 0.5.8 to 0.5.9. - [Release notes](https://github.com/google/go-cmp/releases) - [Commits](https://github.com/google/go-cmp/compare/v0.5.8...v0.5.9) --- updated-dependencies: - dependency-name: github.com/google/go-cmp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3af3cf3e..8227258d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.5.8 + github.com/google/go-cmp v0.5.9 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.20.2 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 57666443..48fe9c5b 100644 --- a/go.sum +++ b/go.sum @@ -265,8 +265,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= From 2c267ab57b14cd052dbcefbc439d9d1bc669179f Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Wed, 19 Oct 2022 13:19:17 -0700 Subject: [PATCH 119/163] Generate mkdocs, and copy the blog from https://blog.bytequalia.com/kubernetes-multi-cluster-service-discovery-using-the-aws-cloud-map-mcs-controller/ (#234) --- .github/workflows/mkdocs.yml | 22 + docs/images/cloudmap.svg | 18 + docs/images/service-consumption.png | Bin 0 -> 93455 bytes docs/images/service-provisioning.png | Bin 0 -> 95858 bytes docs/images/solution-baseline.png | Bin 0 -> 75802 bytes docs/images/solution-overview.png | Bin 0 -> 97582 bytes docs/index.md | 588 +++++++++++++++++++++++++++ mkdocs.yml | 12 + samples/client-hello.yaml | 12 + samples/eksctl-cluster.yaml | 18 + samples/mcsapi-clusterproperty.yaml | 13 + samples/nginx-deployment.yaml | 22 + samples/nginx-service.yaml | 10 + samples/nginx-serviceexport.yaml | 5 + 14 files changed, 720 insertions(+) create mode 100644 .github/workflows/mkdocs.yml create mode 100644 docs/images/cloudmap.svg create mode 100644 docs/images/service-consumption.png create mode 100644 docs/images/service-provisioning.png create mode 100644 docs/images/solution-baseline.png create mode 100644 docs/images/solution-overview.png create mode 100644 docs/index.md create mode 100644 mkdocs.yml create mode 100644 samples/client-hello.yaml create mode 100644 samples/eksctl-cluster.yaml create mode 100644 samples/mcsapi-clusterproperty.yaml create mode 100644 samples/nginx-deployment.yaml create mode 100644 samples/nginx-service.yaml create mode 100644 samples/nginx-serviceexport.yaml diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml new file mode 100644 index 00000000..9cec4626 --- /dev/null +++ b/.github/workflows/mkdocs.yml @@ -0,0 +1,22 @@ +name: mkdocs +on: + push: + branches: + - main +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout main + uses: actions/checkout@v2 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install mkdocs + run: pip install mkdocs-material + + - name: Publish mkdocs + run: mkdocs gh-deploy --force diff --git a/docs/images/cloudmap.svg b/docs/images/cloudmap.svg new file mode 100644 index 00000000..f4af2aaf --- /dev/null +++ b/docs/images/cloudmap.svg @@ -0,0 +1,18 @@ + + + + Icon-Architecture/64/Arch_AWS-CloudMap_64 + Created with Sketch. + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/images/service-consumption.png b/docs/images/service-consumption.png new file mode 100644 index 0000000000000000000000000000000000000000..8c9840ac9021317745b28c6ac73017f9cba92a25 GIT binary patch literal 93455 zcmagG1yEj1ur2!GZow^B&|txx-~@MfcXxLPPJ+9;yL%u3g1Zyk-R*9Y|GabSRlRyu zB-rfP(=#orSNHH$PDTs~9uFP_0wH}77ghj)pnxyI$6>)iATV|uk0jtbFh>P3K~Uu= z!9MT}q^W?k00>kQgYcyPHWbcIT-^}_LhgI}2R3M1Yy<+m7JLyFP;%2b%7AfGR-O$( zgMk(ysAhWQ5I@xeAxL+JP-VmBQqf%nQGhErp<-YXNIgoa&$$qWT{d~E*%NXJPm_Yv*WHzLc@%z zTBB7;#f~8UT=cr6$Mr#QJhW<#gq1W7&DXFn;w*)c)*%|PG;l4UR_HBIXc__^bYE$q zdI8H7qy&Xj@@M1((4D@%{+3dejyC3jLUx+oMMJcWBS}z?Q5_|005N2G%^{b0l;kiv z#VMcj(XS@@E4%JTx;VZEk0~0}*B`U}TXKW=Yxep9(yZXq9r7RPrR>}d$Gx5*)8JE8 zjkPo~sdiVeAq7w&NvfaD2Z^@gmsCHFW%0jWP09APT+eEj^z-bkipVXLDwlmTd}zY; zxIK}UDUe8rWwTjMt_=C4I1Nps5Vg#OIe%98Ky{csg z?-(*I=X9^{XB^p5lWmvh?TOFxLrRXMaS+A4K~4L^LD^3>^U0tgY4Jd4r1_6kURM)h zM{pE*HY=@1Z~??P->O_b$lMdhFmHb|LHmjn^22e!i1vZvHgO`c`cq=7kkn$I!RGXrKHIumwMU?CJh_+sPU%~? z0GFO=O?DmIN>^dV{$5Y^@%3mo6w5kd#m`FKUSCH)x>R|f4HnAGS^iHAeRC?{!kyO@*PSjuwK8a>}zhbyTTRZCUI89MXCm)LQ)%~v&?fv$NdzdN` zo-y0LXO9PqmnY*c-3{mT0a4u8u{UL&T;phwpPzH=tJw_c)(F*ej>&Mf$Kx%cT=k9Y z65XMGsDW25e#3lO2zvO)g`&nT=lmJJhg$tS>*a;5R?e*S$IeQtbKWyfx^x;_jVTP5 zVA9u7Rm@75efJ6iHMIYO%s?9>)8bS`Q+*y9DMxpclatXy zZM|m^`RGcl{|Po0b}K(@I|5lA@5){f-6E|Nevh4a;N0Lyk%rZKVd-nX`T6-q>m`Rm zjDv_;ISgvmz0qbVU{CFxGHAY^(?j-6R_O+x+1)_ZJU-_^h9}G@v=sGUEExC7_J=(|0NM;L0dhBO^vx?8N&`TK@Oi6&Zqaz{JK_ zw8eIejVL<}3wKm2kh|CzAAL7(gcB^HZViCZzeRs^J$=14J!!2XRPsC_blzza(if2T zk^hpDDoOiY&wSx)DI_qB^tdhv=5ktp5&VuFDJ?L52dAzqO^-=gP_&JSCAp}*H+Ki^ zysXh^MsqP|6^8JT{v-982FBjdVduZQ6N#8%_B2t>kd)ESq?^&o zJ-G6I9q#>fQa}2F_|*4Hs>g*t=v&QftHE&-nJ4&$H{TVtl-@0vA0D0Y z8lJ93TiK|f?Ys;&V7+PcS-|m9li`@&?^M>GlfJmcC>M0{;gJ#+CBv z=^S$#+SbDmW)QhW@+RrM@;_fqq3}O}LL&kouZ{meaCbhOQl}$QV5+k1VDkN_i7*@3 z5u(Js$h%sE8y%)rFm1P=2p%Q2v)?XMb11rbfPy9phPwZ8DihU%+FiuEm;A|=&NJ!H z&5~^>wCln2{wVk9R<533bUQQoPUy_Jjq1frTm}N`Fu&ncQN~=`IfZRHAXpjF` z8zXTm?9GT73fFR$USRg2QVMV|pN#@ATV)b{xnV;KOIMHKvb;du2!eSutp(lL1`JF5 z2*-T9J+10=?6BJqXqw5{1t%knv0&b(6ih1O0oRnFpiINS2Rt zJx=Vw>2!7K5seXY5l%E|{b`ekoz;9YUZRoK-TjB~VU<1B&aQ&2%<2pmOd2zaA~g|5R- zGZax_?w&eso$I5%3NevLpPb8v#kxVL$Kz${wGLiFWbNTr;zHg%szbCx5~X{u~$Ye}gmePL0qs{CcaJL|whEi0Y*rq0OPMSvwB z9OAy5E&FHtYc7MY*%t2DJv)h-Hu_^6)5Y?E9obAK153$8 z+?1NBG2xU#1Q4nG+Z@cRlGU6pM>w7X`bYCMM^N~wr6y=_K&-LBbEC1cpR$b zbi?iueSXRFr=q4N8GNWq%M|oHd$*GQrah}Y8RxKz5Q%DAIA~n*#d?AIvukA94XjBL zVF3Z)Hv~O^ODJa8dql6IAmdHS5F_tTq|=;tgaizz*qL$k;%K2R>m}Wpq7ElD^!;Q> zAmEnu1NiT^&{IvO6!v+cykyU|L6Rgtj3=)>+|2@BBiAQuQRj==H5b-LXlKuc5q88Gmtse=%ifmCxyNq73ZvxVF$b zTpZ2-9NL~RyVbm$k`iKGdD^XDQ5? zf1ck5N#=>-BHs_Gclq$9!=Y2i&8mz&1%vUoYjf#pA zvFK`9H{8>`>Fw=x65_F5WXWF?tf+c9{VI{0o2!&iF(ZPGNXf&ar5ScVJ8Qv??G+hL zvZ9^jr7eLDVe&h*0MBwMwhlmt4zPC?0cf30UrK1KX@b~TEoY5nQ6U9jp$+Gp6$V7W zNE}wIkDxHEZ+~-w->g^?y^Q4yYsQ`ONAmO(;(^UH30zjJSbE($@4YARC0eaUw`xQi zcV^JTaS4D35fJUx#mV;+;6X}R|18%@F<(_ByYE7ePK!#nEA+)(bJQX#E`?n!wtr`1uX*_Og;OM*sFJe z!|4g4KP1m#?GZLHq44w}zC%CK6toX`mSCi8mQ14GxI!40K4~23{I48zt~ggNa-U%&k#vpDDKj1JI&dKE{;N5;;%zcGuH8K z+iUj2&^>PmciK-G`yn4=Q8V;j^~2zF8eZvQ0RPK)0l7K-6a1?`h1*krl7m8R`D^Og z{;BOQMtY!7d_0C?#i$tye7(_F$S2CXKaN)nA<64`C6DLn39%|crcu7gkyFw+mU+BX zheuB9-?c4*tyc%;b*udj@C`SvhQn&Q(U<+BIR#vp0Y&N)ry1r&wV<|;( zrIO1zm>AoK68|3Gq@5_!m#2=>^)kP3W#FsMWtSijWvm^d1Oy3S>PhNiR2;_M;OXy-Pr} zUi7Z@NOlT6r*7_Xp5@yEcIf3J!O(rV=#8<8Z7$Zw1b2Y8M&GbFnTpHl{%ox}nDS1z ze!o+APM@x~4CgWhZ2G2bwa`Ccn64KFcSDlH(NoYByN+!X0O21o)qVf%9!^M#amB*m zn@j0Z^dqnZL9>vQ6c^Kz1Y#9@2mT8;TNrs#jG6CiMr>JUCry?`Ed3<$js?(K(X@`g zrWm!}x8Bh$Lte049Tb8C3G)mn|<&S;l765_-bv zh#SM(_Os)X-#;OltI?6M;B=Jsr;Y3sPDt<5WsH^_zHA4v>&(M2|v5;gkb}5lJ*|Bi5z;rhx zBwRf&WXb49C`JL!KOtejQ}nV((98Q!(Rp%Luz&~TCBkH?D;3|6CORTZkFJlMLdKP% zwr?h3XXuk_(ZJR&R*{;6a>cA0c0s&z0Jrmj)1Q7gD^}wn4EhU*6REQ+zBW6&x`$I0 z?cv`vq(#JCcejM*IOxg+tiX(b$El*d$D_{J``CAL^7L>OAL{G>dg%Hqd7bQW)JDtc z4*)z<*GI0vElOAyV{)a?wYq$~^U2A{J`=rUyDP?GBX{BUjbXpw9 zOKGtV*~1cP)dd?))<+@x?D`!Gyn|%X(=)%t!0GUw;!xmF&j2~kMiAiM&U-%}8b9td zKY#Z;5~h19=_sFi8fE?zU!a6LlrYvtCDL>$>4I`RL(yujE#zyqM?o)RI9<;k_h)?b zWFeY`y$!bHfUvx|Sh!$l`WQ*{(LRKP0>|`zv3L(R=9R1DQO0?-dJHRT+~gNy8(;dm z%kkm{{Ys6YSiaU4481$vGaZun zetuFe)tb1i@tJ15f(!f*hE=#zx!0pk^Fh%|t3 z15SF@&G66OHg@^<$@71x^P6NQ^fkG|RkMDdSZQTkf-|HOj&T4l+o}lls+8SfO=4mk zoW;&E%WAOXa=P9a;HiVMrOz^XJ*f^gc{24LK2!ST{j)f70i11x0JfgRwRblp^^U=5 zD-n%PLGH?LIo!x{XOMO1cUQ9DA$xoKBWp5NO zzf0*iy9|R=%Wz$q^?8hj&T^U^%>|rq^9N8~dfMTgD|jGVk8+tx0j=5wRC|O1u=Y2H z@Fu?N0GSzz=~yPYnAkXl8p7yBA5k5Nk8`M6mzU@+5}Vb0w+?k0o3%j|2Cyla($!t| zrU69haY3EzxM*umO5o1N-(~qo#3Tg-z&s1i0FN9GC%PtwtfS-3Mn@MV<9&TF10YIq z85w$NYI#M;-#=l1_sjjrj6zM-!}BK$y|7ED(rS4}o~*|czO%K}y;f_CP>4lNDU;r9 z2D03>ew#8IPKIAKvNVR>bcP5|wg}mbp4u7qG#6uK6u>J29VGS)Ra{~n5G+YlMdiHh z2$aAXE=5LJvJN=RDILmoOt=T;=91o6J@=DGeoVatR{3ULK@`m*!r-FWD^qY5kao9= zZLSRWAOXZ9;|Rc00=?Grq>cxrkI) z-owCDl^@OQ(g8s##Fp}|i)`%UCTR?nE;@lY;v{d1p6`bDt*DdwY}s$xQ8)OEfh%_Y zLM!4pIG1pgf)8PNwCG3i0De7QY>4O#oNjftwB6C9nAde%=3V218-m2gfV8o&pb^BC zad2QFs06U#|9&9;aL2a;=*B-wNhe3Ye*abu^5WDh|(u(L_5{#F20 z$;s_{l3yhuheg2WmF%UQJ3N|d?5&8ou-IVv-SDG{le!>gZ;5(cVP%lQ@mjsX*bP#A zprgw64V^Lwpfwv7cQ|-jiE;3>3Y`i{hk0pe!U^(WO&nt1ojI#ZVAHB){}vS!`wpQW zDn#5`ETE=_!wHgcUb00nm#mMjgSN00M@!iX3JooTE)|+q#Ir!RK&@^%7b@W-FFd6m~2Sq{Ng5L zl1H0^_hUh`xoJ*JtwoLq%%orhz2YFt3F+3^=3v}7OX8roUsDQvVnUVvsG=0I5s=m9 za4RSpEqsn}dUWr)3x9~<$R+5LX@Sps^WaC(D3=oOda&^D@BrE#AsBvv)9rjPox)@c zZ%S_h5W~KYOKTj4pbvg4MN)Y+20y|j5@_OxFfmyjw!|;|^-p(@sbj#T)E@*VS6tUB zylCdwLb{B8y?el?yi)=uqILlHJB0WGvZS6eyE<9Hr$%B>cqqkf-buCmN4^$&Y(}xK zV_;!b0!JPF@V#bh%g7->(j8^J?W9%mxh5-BW@s{qL=sG@OXN$5P3TMCwf_gd1~>F> zi}X+={17GlZU`9Upg_=SgC%ytNIK_~iaG3%EeL(9>>H4VIv@s|j5q_^1H9kqcrhW} z_vJE6*GvC7;>-1Ibmic(S^OYK#cVhtB4UC0-Mj9S%>}b`PREe+>S{)zFJ)=*hafgS z;Kz!mN2?Y4eYEvs$RBt0M2C1qg=!??FLt{p4@L`aCkzk zS`TceV+M!q&Tw^XlM8|j%&K5@+)YX?*?Lirj;unxRHN02dQvnBkHmVelCS{ab%i~i zuje#1@6h4&ZqFT94p1V*mM5n*8A2vz<4XbdIt0P*g;rOZOB?ZW=Xah}bu6gy!uR#o z*V<~z<&dNE@nq$dzY>AwO^#=?`mk!5=b+oEJr$oNYZAuxcikbPz@ZQAnu&un)f-+b zvT=x3If1c;{CH{Xs@2a>VJUH-#@vXHb2mU%K=zxOqloTmn4L7EA7QkK#Ssz`67?j~L4SjbshJI|dMByPloT`ORH$G=n2r&}s8C@k(Tfj-;d6#<_Q$NMeoy5*dOD?7Wh#(;Ex zwBH)Rje&*D+gy9==M16DtcweXy#2w?dg!>Ky^R@f(KmM1fB)wQXN;OP-wWsMQn zi;(&}Ubt&QOh-dM2~{l{iE>H}s1;-M;FJNXKP)qcCGyCRO?qSV_>J$Int)yvNr7pN z0_NbPh|aj#=NS%Jjk%Hx{hwlH=|w@$&KN6u5&_y~1{maE-9$IYvO)rr9tHDl=|P|L z20~qG;3fM5JQPMTa=+`)sz)yLq5EVKLl`J&PxqW^uS zP`VAZ{`0-(tC!c-@^6i);B?#HF7%mzIYZUdA5PC9S;17_*#btDDa9EvVQx>z4m1S2K_ka6_0E4ts!3&*7 z6Z)U{GaND7w;xzO>};JU0(#V#&Ry>J&pb^jb^<}IN!)113Tj5gVRfUaY8g`0+Zl~b#DdsZ0Ivk>uZC8rf83k%fzz8J1=D&D{Dv+G zaUG}^K>qm8Eis4*v9p3P|1nP3 z-@>6izuznOuCKT*UM;q{s0H8TayeLIuln?EkAmf{a?dcOS=AV%UWiFmf9#&#IF6V=K03Q!CbfwqATYUBpcbcozwnHu6 zA})6J3wRQs} zS`bTJ;`suf;U@14)s+rW^738*R(~r#!GW+FK9Z~uX@6MgmT$>)?x@GJ`d8Qc*MtzY z71!5DzK9kjbt*be#?}>@KHg*(Snk?WT0|sLZgtx&3&*!ObTGore}==U(Vo)o`+TTd z@5m!KI5@~|y%<0A)fcD@90dcId?*7V`P{x8qIsw&`f92ZD8{a*&d7y|=gi*{k?f03Eb2{5` zcl4}*8%&n}nZ@{7SY6C;S0;MCRJme*Bqg0U$tA@m^~u7E?JbImuouSMct34~675fH z=Dg*>Wfv8I0ijoe*udULhP2Xu`@SFmCoO%t8pC%bS9OANBlmrqauedOvt&)}6N_x! zFZ5L@Rl-fa$vC8J3lw z%^N;X+i4;nQxp-i|n}z5fL69%Xhxzm!#vKAaN#KXo7SpIqGUFI|vnu7O8Df1 z#EBerbNcCT@TlWfIh;N2nXOm~Lcxu`$s3HsXEK`a7T91|y=OnRdz}5xVT_3iJ(qwN zZk|7Kt>ZNx)eeUs#dQC?C1Qk~>jsKe4T?LCd3l~v?udGxPKFOwSvQ6Qe5{8jsjUtf zzgrIRmOXC0YY3Y3m%YEFt4be7XkH|eGi*XfI!s%zPrOChgOTTFw#hNd&T)01j9%Ff z1d>}Qo#+r7i_8=z>!W7Lsx)vrq~^k9sI&9U!F|Ve4JSi9AgpnVWwdnZ_UGD#OV_=0 zQ5*pk@oLXP@?I4D?`NhG_3)f)0Bpg&WAlSB@p8krUaa3=Z1`xpoVN`1MuI2Ot@=EX z>p^n(X)fe8y^M3rqdYqN6!=9KXHT<5Z!Yp9SmPBa4=K&A-ko(4Pcv-ev05xUO-Zva z+d=HEa8`^5g=+j~|@)_^y%Ud3%cj$4i)cG9cIM2y8o^Hz~=7!@UX8F6A?5&22L(-9F|5 zvzwTe^?KFwbUSimBC~~;FP#Td9w|g!ufQY@*xdkvG969dkfQz=010gNB6HIq5!liALHiss{L>Gjti7QQ zcD{CoGY1wo32wVmV z#9&AKZ*L=lJ}b)d;i9F1k|jb4(PUy`YM$ml+pwOl&`jcOqILyCJX(V(%;~S!A5D!4 z7xNUkggwm4tkCI5T|~9C#=yX^TCAsLU_fb@PL>#7@AilI*-HsS{_EGTMrnosAbeO2 zXOjKJezM>kykrPd&p#ucP!h89`yYn9rsT0KK6@EJi~?mc#(jJ?t8U}Z$0d-8vg&tR z=^XX}jWa!=xA8!Jb~7#)ostz@lDi@Eiwf^TWoZA3^o}aMb(*0`A3z6oPB#Q3fE`nU zEt>^hll$?Cl}|Kp-!2z6w%u=un93!`#|v9~A^h`u<8n59a(?a#pQPk|Rq{uaAlV!th!WfsZ8%8FU4siYKy!ET;Vz{UlbP!HdNV-SFBvLNmH^kD3_{v{4x@4DA_vwUrsXD3OSe4YZPH3v1T{8#JHTSig-=W*li{~3(g4HohjHiitW zgDp4wfXsg-H0S*ljnrptERg55I{Lv6DM}yWuB7sCp-gV0)uLgct(m9$&9`4&ISC6= z1u*aOagc_6hneZ9b#QiA^&)i@-Pm6Ts_^ zQ3h+s@u62BQb&%D!5D5Gbu;`UgBO@IkYnQ^h4S+oschlF>Ibm815n^#rVNO(ZF{Fp`ZrjxHZQ19XN1ni)m4XFe?(lRnK($dN> z^+9N&8{fz!iKnO4{+BKJznPPNUkhe&I_``&p@U(!AhDHbKlZwIuJ|KEau9t8@Bt54 z$Db~+LiqTqGdurUZ$7myEGG8&sD=zTBmmVDqn3B`bbl~Y!QcBC=>JNA2kK8>Y2SDa z5|Ja40=6w;a;OlYaUTxHa~n#jWWRT#X4#3Kjc064%iiT`Hrd20Z*OeOW?9xM>p(uE zAQ7}Gf~=V@;tt@c%eVoWg|JmeKv)>kSedh+^_v!cya-54L`0qw{Eg>ko#OI4K}e^D zBCn~bu>-ovL>k0Rfr_Fd9=i=_(w}zonroM;qY%{CbXM(^4WO323Lj5la1 zCHJR&PR1uFRX~2`Axtn666bZBD_2)!?Iz>pOhVM#URT#(mHSI8adQy}FtUUxA1Ye~ zMo3gyUGtP9=v_bC-Z`~Qk>bj6iewB~$0z-(_};|=Am7_I9)_mjxMVTj8egnovigZB`ow=pg$9FQ?5JW#@}AkGhL{&M+%}-AitV zZ^2Ro7HVokXyidVAmL!A7E!L%7T!j-GCDF^*tr)B{0*9BKHbWqif)wyxX^=8XUEW&d9U;s2;SBR{jrNcD@-(fws# z2+}5LPP+d>*Ky-g^n2bRcM>HIyLFHPydFo9Y*r&C!glbcRuK`ABjw`pN^S0kHU_)nhPlr30sJilQB#2K+LFrTY0>Ws*;-Om zQ~Q~z2S4o_B9+Rre^6FM!Ozd1m5#0J6-OX4Q>wgB=YNd8E0O8Zp(t^M(={f*M3k&G{U zc`?NjjGr{13N>&1Az&(74`?`mf>rzdPMS4qfP4$k_t<2!GK7mn+O-{u%WUGbKN-x7 z2RK69R@YNL{a(n6uDd@G7Bi)B4Dm#{d3iI6^+7h`wQlDIm;-|MkU5$y4#uPZbvC>; zgS~y+g<{cZ)DvVV`I{v24@ z4CM!#9~!L6lkbjYi9H-=5hcW9RK=vof<@9IDrddCxH$DDw)Yaue4CWSBhoperZIp) zE+4Nm4Nmw-?)N)7ytsKf-6XF(D~GJ@bRmc++AG(>+s4bSr>GbdBAu>A!kOYdw^v93 z2(K>`3@a@0li&R#8zKF7>kbH zNINerbq40;d$PNr>K;H$BCLKE`m(WT5oz2PeH6uA_u8>To8Z;bCXUoJ`XBL_SsX{2 zmW~-SXwIYBH%847s#K9EAM7waNueYzHun4XO1t|dExxvuT+2GHzCN~y)%K7v!n*Ct zQ}v_Np{cn(KttvmnUvA^;sDY@M{d6KTeFy+dZ$eAkiNt%6xBpqqnXcaLHGTsCqtg1 zw?~V@=sPo4j-(3CO7LZ)<3>X;p^b4?ZUsRDJ*tLd2vm3bqJG@q-R|HQCFzT2&^y7T zDwfKnN@h!qbgJ9|Ci$BmUR3rwrqJ~>tk%A?8Uwq%nWX@ch5zsEvY&#JOIwBhRC*s7 z^=+K9^Vye9oZW@={gIouunKVL_crsoL-h~HvVn9`WIRG5rKPp%4MLbGQvuW_A66L2GR(y9z5V#4Fs!V=2y`2Qu=H0c9C1|St8BNu%9_V421h~T4Ojj6%Hmo=47QZ@LFS_D3 zbPD}DANRvMMW;3-eCr-X7J?vpdg6pq0o5G?B*ebHI*(S-8K8LtpapqTUSMW=_x2hr z=P1O^U_w!dOvcEQqrd-ZNxq^W+tj!77;E%lpkwcXq|VHpDenmnNBA@>LLS@qr(X_=t7TXQBM*k0e|$pQLgm%^+~pw+%Cu;w`XS zOI4If$g_0T0wyLho@WZy(U-k~iKZq_l`WSiorri!eBlF2U_SLzQed3J1;&_*NZeXZ zE3kchwfgeqlh<;Cetk{{U^=#Pc`fJbk0^XZbKWLAxKMU0aKulI6D6Ys+IIUig!|cvj6LBTXC;wzqUv>^Be)PB4K4s? zcjijKN3r&g@*78ou$qmpXk2BLq4H>bSb5z( zQnNOX%v7Y;^O*H})?hk2bb72bjkS6|OJB;&A!c?8%ClURg< zFVI>SgDW*46hURT{^#W)3EMiO&mQ zk260D!EuUs@X$ezz64n-g`Ekl%abV3TmcN^N2Rc+C}UaJcdaFG0O=?h2fBg$M;`}( zjY~@dgrP3LC{ykYyfHx7f&@GTb-S$Zx&!1)szqH4Q;w2>JciUSiB5CC{WKPY6~vUy z$51f>A?y!D0$Z-@E5G+mM9i_`Q5*z=yE6HAj({Ne#g#UOCp#Egi|YwL+$@Q=6=p|# zRCDnO`O64k2Ly%BFKfZA%ZcJERaI`h7Fpt^*wKDc)!cRjPEtTnq@3PQY;>iZRr}}t z?46Rt+oIu%=IbvpTCXe-Z>8#9DkbKVgLGxA6-aggjvPS%d{`T^eu$~Is^X?!Dq}x) zG<2Xy;Y~0(j)_y-?$IQ{1>IL#xqqX3gs?jpwfDaL9kwg~->|Wg@fgOZY6h`qJfUIn zXIq1o>VQdG=kHaWBkSXLXM)u^<02dl1pHZ~SWTzMQ&?UvOZwQKp3YtE7mq^M(kC%{ z#~7$o)I$eM<=$o_PGoyN?aFVu(j?G59TmV!GI-9&*qP!lYr5){Js3}l+B3ykpNe7b z-b46u(1geL{A`Wctz>Kq=p0Cn!#oe08XSijbM#3sE1Zj|-_$^3bu_uYH}_pWMj1x; zQo1NXF;pU{s>y-8+3B@F+k2Ruyq;lz`Ha6HJZ1PQj48vE2$L74!w_oL4{~xl z)B*`BYCrx*&Nwd&o8`F)ZJe-#FW2EQ+w5Myx~7&e)-h>fbR;s zPLjnNrPhF=cN&GlfCIvmZ*3-5Y~W-6r{&qC@DVd61^hoikAvNc&RFyBb9PtlhkHG_ zz0ACWZgK!u1CfnHqkXl?XJ6Ae3pi~CpwxiwWM5FHoR~?$&wxa zp$u36@TPy!zd#)189-kN4sc98puImdfJp!|sG$nt09h)m{s%uOlAt&c*8kx0zsad0 zQ|?M4K&0w0g8rEpl7lh7V|16ThEzsq2%%D*wXNK#`ll$GLu5@+Q+v3uP>o%i&>cnNBFJC^` zNYV2-!0B=KYZe_}4xOvD*B!C_)!u|m2G@$`X-9gM+y>BM4D|eqt#RsRj*lXUuCGvk zazc_TMXZatDyihRL`e9JycT!`^mHu(HzrZJa{IhcN+}zA zSr`Glp0xzn)5&!5%w=>>PY6p=WfTh0hnwT2!OWve}`8j~le= zwMIalU`r9p2S^pD%L)JjUi=dV4J(s{LOt5!e%qia(tL+akG*p4$x5MN285C7+Ex z5`_rS&gN+0^9_LO$%+&TBv4Ih-r9#h$G_+IA+SdvB41co2s7oNvDr9UY>-S?=Omzp zIs@*e#FGA`!$KG;3BNI+w|=3ku72k{kmq}AargxwKXbb_GWogD*8XcRps0&LJq z^rb^NiWY9ky|}MN(q|+un>UZA#RQtBq%Y%oG?q-HS=>4nPwDgZqnZ@z;mBF47;rsx%rTZsm?qsQe&) zvCh3_@KQb%e7iR0Ic^`{qRGm(66n0f>&C;0`JLZdDzW>$34gdxStva3oIpCDcdpd{np0`ES+oTCUyN-M9k-NExf47}7K_ z7+AmZ#1S|L4m?c?lP%hpbQhHgLRGsszQGojcv=2x!`m2YyKkG06i!QEP(J2i*Q^z% zd|!*8!v=`vG#Ed9=)KA=5X2UBHRcI7ETuBgKbS#g^`&{aKkLb}UTOqdb}ZLAy^8>g zS!})><+g3!fUabNGKOF8|IP{JKD$&us>E28>52&U6*iwZe_?Qhv>>UL%8a@IH?*Wo zP`ZQZZ6U3nkbtlXper@}ohq@&9`wupdx^1@sSo z!LfPikkx7Mw6G+ioN#(`Rf0{{nj7sqYaNf*n+^huT&*Ae?oYx!Pa2+z5StQ1G{` z6DZ8-jy4Pxro5CySTWzpWlWa?@~M5z24uh8ySJ-?X)`Yx*)`#AUME}x^Dfk^qs-iU z-7|bX>9~EN*FHH}d7P-(%nUGJx_eXnTwrq#o8qVp8baK0nVj6Zd}MFjuWEpMJ6k;` zyi4^~Y&K6pU%nSv?2f7q?2@q*~+RXCsc&Hnbhk!hDL|MsAy&p2$ z?Ez*yBkvl?j#!N&BkK!A6uQ4%d$rS@vw6HB^7CDBo+ArsaNPY^ml3c|$P&-6tj+9j zrAD(qp3{fAM9X)5^&vmuqpAwo%iFacN5H-3cX>Z_Ug$fXv_iXByhaDERn-tIN(YaQ zxD;mtOm?t|k(JTb-lxr2@ADtnajVXAyp0kcH(RF_(TgH?2={>k%;}>6Ck3JU2)DGi zM=zWf-1KD1`6`rcc8(WUFJ4WySj>BIw0g@~>-={-5+!JJ8ZI6~Z&jtI3E^)fesPmp zz3`WgziQYQ-K%58%;;Fi6pJ*oS^xqg`f32bqHC1X>P|f0b$Q}uVU%3~KUH`BPYz++ zAMno-b0BH1m9{yl(@;=h;^hMi$>3_WV~?g86_Hyvmv#TRJ{cp?777lB8WQ_agQx9h zOah6SC;4z%X+{Ih1a9U<-|ATGn$G}J%E>~8!fW)roekU^t6rQ@G#_FS@d zMfBdIXSHOMqK7I<)um!QnTFM-k%up@#4`P>0xa>d3pOk7g}ms>1lm@`grjjkF_AjR@hfkS3qIg6!nJ-1mQKGK*)pYnZUpPd7Evt7 z;E}>1;1QEoY5NM4tp zbCh?Z;CE7^`#^H55!2AUC@aB{d=*i85qiGyESjPdjNA^mBK8us8Uq0V|C8G&=fAZz z7`RX{E#G8T8(Vza7FQ3_@&vj_TS8O$^Bn+fKSrU#uLITbC!l@m@)vfm)G!c7mx|@{ z01n@=lfaOuTTJ0L?(8Pb5f8;D`&Y3W%|uALEzk_WzU&pm2CnOKPwp9j05Dw20L45S zZ+HaZc3R`rZxIo8_aL-c>3Q@)K`qUG7WBevUc`2=+~f_np%u) z4v}1{6sTR)DG(0gOG-kXnze`;6#xMpxrCn#AoG(i19*Xyyiz_7`WZ%*Qw)mnnG@Au zUv2T!dO!oXbQXF>D#F;8-2uBqmr9LRPfSNYq+#0eR^+YM;FlmxYb!?y!h zs%^WT6M+>zikYeT=zn^j3|x4-^`Nu>2~8wj3Re~y0Ttro@Z1`@w^xtw00cpg+_ z_Rz-J^9gnxxVe2&TDJAfJ_v^B`O?P>TxXjnMaT1+bYCjbb_>dE*+$|KDKqH=;+9;F zppnrQ&=&gs)sTvx0qR``YHVV7RWsLy=$eyJJpsy*F!-6@ z{Q5!}j3;tMfa_}8k?XNQJO_#}AMQzj=lU{(e{iU_sL&t2=Whjf|dFJ3L#af%;b9TTLzVn*tLvGd^-tCm7!!erjMt zw-xxiLz`+$e&gNaBvPKKRjSnh%&&(~dCrnWH5%Zjv;j70CxXLi$b&(2Bdo==7)c=RYmwzNsaYJUWM@St1P;(| zTwB^OqXqc}Ki&^xeq@w&(|dpCF;whcV%NE9onRnEIn)v?yU_;&2Xw}=kWo`pCm*)p zGdWyHY6uUY_>htBOQBVQVHkXodaIWLv)~OW7Rl&mYKZE%d3jv09(JG~>c>wJs`?)R zG;V;#vROhssbqB88IXm2X1+M`t;m9~WutldZ1tkrn4okmCQB?r61iyBvzg#Bd5)p2 zpwNqZyABXH-G(g&BJ((h4={@n>YPb|EgZV0jNXO({+jb@9UiQO@L-5t#l>OZYLB&8 z9X4N;c+D2Vum8Q}9nqL>mR1KV!W>9|JSsFRmrYnl=Y3%&mLgC=1{{K@1a^o=;;gYV zFx+;%Myuw}dQTvWX5Z`gclRy_KU+%TSGs)po2msE_$T{8@OtH*39>+`vU(SiuU!x1 zCZST!kurqFZLhyrUtpTL*on46`zM#1V>EgtET%C)W?JxKYHgqNW}(zmLq_F2AbXKngH2<+py$%z`4h`V1m-ved?hCK}e` z_*q(v1*t1YR01c`3=v}FlC=kvkk984@h6NI6t(9ZgGn~PkOZ0`0@j64JC4llni;Vm z{p~Nae87MCie;)Yh$(&v)FflT^I$BVKVyw3Q@c*i>{QrF`Hwy5~$K~pE57&pP9jyy5NtM9O#yt(SvPBBy5}zSZ~rL z+iaC?m;0qjIUc7igyh!Vo*Bz(i=!z}J8GVW#tS?@(fz&{3mKc!KXoIBQx_iI(9n>Kl$663VkPGWXc3tzl-`_F z*yGtK4#iCWpiC_nyrMuyU|qCc#!FQ!jFdNI0BMIj|Mh@RG(iH3R_!zS=Qi1^A1?Ek zi}!+<|A)4>j;gBb-iA4c2I*89q$Lz31*8>FKpG?rq@=qKjX{@6gLHSp0VI{~K9nFG z5=wn*bKgGC_r8Dp#y7rqykp=P!r5or?G@<*rWRkR;sv<-Vc8&0N3m413B zmKBpZM^;>)qg~TXdtw`(Up}@Bp_nM%l;dSFU($Ie9;EPtLdp8`9ozlLM7b(~6O{E*EA7Mp|Bhe7>Kiwjm4 zOnn*;RH#Tb?e$Xrx=Vxn zMm5ppsu7pA?9skry!an?b*|>ay-R=pJeqawN2Joy*xgg~Y<#%RZrBrD^&agZcITcg(uz^*a5Uj*JoRWU|l6uw5`g7Hn#%E?HTwfxr%n* zyw~KTccuFbqP%~5&`jnuS?$ln_~#=BriN1Dr_gP4Y=Le{hx8gFxT6f!to4n*x2Q|C zA9j7oL8yOsjHtrL#^P+5IG!&s8Int_@=8H9S%LJ_oEUDw=cpnkc!VxS+~VY7J=zQd}5hlGv5RpKpUXKaC)>= z`w#~QyfLl`sXwr*8RsBkL69Q&wveqt~90b}ZJK4gHc27>B(Z5j`?Vy@k| z8*<>sGIQO*^dV^V?nb2=pqm0_f4Jnu#{E@~^*#3b8=r*rqZr58_4ak3WCCP5|~H;AgV!sbFSl%bSlC5H-N8r8=q|51oHi@X2H|3MrjluaLgGyM6BC@*?)(m=5n{^*IegFtfB#H=UYj~41ORBLRjNf~vE96>s^vijr8M(^W3D_2 zyc*!{M>X58^ng7#f)5&6s6mBCfmwiSWQjz;TsI8>BPW)4>gdQL8Q3lmBSoPeeX`Kp z%zdb?iFbM)AUgN!Av8Y57&;z@QN{Jik{2gSmPNHeo*pp3KpF?waFLfXvlyxn_lax7 z*THu0iy>j;DdLeW57v}k657su^W{OmWOhNTgm9%h^oM zja(Nni46VZs{k~mt94vjhyVul4yx>RKgh;)N3&;Oz`B1w?f7urf4`I7;|-YVwrg~$ z@rLeiEimZ-i|QV{t4u=gyua3ihxdqX?h7jm4GaE z8kQf?zg+^Bo~!C;<5$@?6D6=|c{w>dLv&_JO@OUmY_Pq}jP#S-9j;W%enR)G6of;T z8-fI_rc&`BY@mBR*ezL&Vtf2Kbp?R_``yHN>o?0{Vq)gKPeTv< z9A;=8f}w?ggRBG`HoRwzbtxDFq$DK1SjV$246B%zzzjyCe5nD{2UlNw7m2=`t^5hz z416yQ)*658mqm~zA?dONh`*w)uC7#U>l4uR^FEj2nz*nB+vsd9G`ETG%V6m2V%uBA zFuW>y?1y8EB7sq!`97mr!Tw|!%?PJ;h0Frul{uQaS;K<$D-_(V8<)N|@lcPW(}ZG}?x4 zE7_8!Le$=abPoWGCJtuvI=7kct`K;7J=iXdJnNU@2FIv97Dtx>n1=2mg?xfP;b6qW z9Au<3@}>`d&P;`^e2XvFh8}w+Wm_EDU(L=uq#s&Za4K?olznT9osDDT&oky(=FbXj za{vYmfE-|c0fQJWkOmn~-Z^~ReyV=v3S*WZSwW!-*oI};7qKIEhPaU6;I36@*hw?7 zHs9<65LU#lH<2;CfEfN%mPKpymr-yGI0?gl0#HUokSNq&IE$5P;JT91M#Cbe^Cz|A>%vOd?TF+Bc(pyr-;XxxxxZ7_m0ADEB?jwMBtx%W#H3SivK zv!4%y|e`iuFhZz537@V8c$ z?$UVE?g+8h?tIH;Z%UEhxfrAOTp!goT9JRPfT3ecm4d@x3NlpwrK^MnuRa0OwP2A( z4mGN)@;Nb>wJv*UX!zF5VCtaP3SRSr#*P{33%&|XcWMl9r9lHZO7nD{OS@zCh!Faw zaJ^ujb|>rVMB-(66AEfcJ+&@E^89?us+6V+AZ)UG{LN;pcA&}QdtNzMwS`&e^!)sf zokt5hH1S_>!1}nqOX)p*dhKWElL^x33Uqh@Ui0|@3weLJqIF8F}j9|WH74xi-4!tp!r)@N{U8cn#1Az)di~e)8p2-*N=Fh9FY%^nZ z)gO!8uqWN$gUX`5wgMcK^Lb=&61TT#gi`Z6xv~4LQOQZUg3Vu^Gt~o>iOy+j_6i^E zEobw=g73N%W4*$`OXjyVFpV@@MzjDEK*z!ozp^vXuG$I+GJ1hSQz9q6u&9U0m3Oy2 z)z#a`SoA8@-oz{Ij!o)XNp)!OTz}VNhO^g-{?92V2CARlvG0F#|0KCcw+l2H7wS)7 z&Em;X1lUxLK68K6H(wF!O?7>D7@v)~{w^&zfKx)3x^`6GXt-BL$lExEy6)LOaXWC3gi zkGHJlv6ZYi3fr6Yp|1mt!1INCrnVjiq^>GCR6p&O(pzZfFJ*? zaC1gv(*DeG8=#~tstG8Tti`0Dc|Wr;>CnfDNljGySiu0NiXx%BWcoBk$wK%RVPBt& z+hh3-_AeAI*+%bKE$<9PB`2hVRzH69Qc>%y``Jj_cQ%@l^m=SPJ+RjTFg z;=4Ny6eh&~=|0N|sQXUpS-9nU#b-MxpRD^t?C!qLVYzI%{>XK6Yj3|rbf>!bziu?%_*sO9vYcOIRo72@Hru%wzOY$T)@XS}p#yn}xZW;U2li+L|8sNq1+@kz4GlC_!Dz24N@@DeR5pNc7 z5;toJ(Sq_6sn*^_$d!a7eUHV)+3age>=lX-~yXPy!ur(hm*ne}ESyD+9k@YC4 z2E-MLSO8lTOgzV(CNAVqh-g*7ZH;ca?ThtGgoxTA{L6NR-m9O{3=_mFW{-?gOD=^& z68;WK&BawWe7q&jUi;)Euv4F7+r#L!_mPsIwDpRLuFj#L_a1AA^@%AN@U?Ac2)}uZ z8ce|Y(7D|+Jdt(2Q;oI!LrRLXoH=ZL#+t=j$=}psE%Nbb ztjte#?3}2o6i!&YcB3V@24W(gGE0+uAg>}Xyon@F8~O3-4+GH1v|XF2s3afReXbEE zeP3Ul$;y!J?;u-|i?lt4vAusQRxCm2g%5wDbN10i{inG$Ej@aKyqtWSWU-i zH6r3lFnDZn^QU532vuk{v zOd#!Jyz#HKVlG(A_-XGS>ATFfcEK{7K7#ra*H=`$MIVgM zA)x)trDNtgsVhCQXE;*{oIxH&ApJ1$b#eu5!a_qUf5uaEk;Yrk-8b!{wWh;kU~q7? z0!)kYReGR>1|4rqXXI&ErU@wmLu+>RXhqWCnU!^s|Eh+Qf_#@+92ye(aU)o3> z85OAp!8mVJ`JDjj#5XdUwXP<@ER;>naUtT~EZA2J1F|L!!Kv+|p1zcA19tWsd z)o;;3UK)q?xO^;^CI|}qtKtz69iv`Od&@yv6d$I}E*j)eVeC)oG}FhIqH~`T8My}v zr7cTB4=4Hez<~3;QcBw65&ku1q_o*mCfbH!+Py}*>c*?{0<1>x;j`Ez(e{<@?mt{Q zBkLKj@BQ065*Tmv-^(tV1O;uH7>j!B9SSuGF@;$+*pjS2b-Y+(E8=R`Xo;V znO=D4CE&PpzU6bA4hs1DCD6c_nke&JJxwEz^VE2tb2}W^3xI!5CZyYI`k!36yV~%E zAIO=`)h}k1N2S_NUar22iaG%MHt`QO7P7wOo4LI*?Vz812aSt2ug-ekE7N|BR{pFQ z`$XV=PE%MC_-;=}_-fyG`HG~2J?VX7c7M_DPpkEHt6Um-m&fzb^9rlTT(O&B-iZd2 zUjwPmIc=_XP{=JK?y_XLz2`S(!BdQ~Sc`sQk`Nm1ZhLgcAgyegop;Pa{~>j_-{0w= zq92zQbJ(jQ&@>r+Fl(6ab*S&W_=$PHc{ahnP3l9gE>QJV|9PqIF>ob=A^DtUYM4m> z2gMH{!naDblAd^>QlC>pT$v*+axazOd5#>#=+Hd>K=Ff&M8WJ<^67b(cxbNq%9Lw6 zLxC)ey(!h6yQ@nwdUjjS&H*^;FZWUYl;74#mq zrPw*Wpd?53x9a04l_|=vwEZ_ZE8{g;ksLGEqdRC4C5!dEj~o`y2s(3pr+U2pZd=b{ zHbqvW;0+5XzfTNF<^p{~o*HLAf2D6WkWK)?MEJ)9Meirx-Sej@J-yX`fan3pNu#%; zpL6dH^t0)nv(l+%e(6gyAHB-~HPJDyvFs`Wh6BIGB4s5gf%gtS_-UaOqFZfh<{(g| z(nD3yX6rV_dsbE|iI}M6XuS7I1yjoVk5+sVMHP57-&#a`j0=9OnvS7E@$|Ip??!ax zHXD~M7o3PLWUsB5=uA+5i>Q9O?Zf-CYA+5@T2vZ7&-7?nQ=sIuVB6)dWi5_A3DlZ0 z=!##{-MF+jQbhP$gHwpD>GvciC_cvsTBqwYoOSEZU zxN(dKHUE21O2eQ&n&3{^Sk^MFJ7RkE_hdr^6D<8}2;|9($!PxFrbE!KW;tY0(MeE< zzh@0sPNX09->?beKX&wQs0Hh4d6S@!EdS$Kl?1&-09w?4gE1j!*jH!pzX9R@9&Y~= z%emczwsZ}Xp|x$@bUEwURoK{Yj!wQ}`e_kl!{^tu7j<^a270PjYK=FfM@R_0D~=jKa!lAZ@6;b%_5PpdpvT z8$CSUvBu+hS%Oi(F@R~Adbd+(LqgiNsb4kVsz6n1M%&2ia3D>FN52Y)5dOepVx(3_ zupdwS2Eu|bx2LkWU94*2&;os|wIZu}(kPaxqWu$s_63IVlh^+U{BF9p?Yv726U43Q znLNOmb$S#R7jyh3#lSYzkB?wdY-!IZzG1hsr#g%+xy$>ad!Fzt@n~;D;m6}Y2KBg~ zc!wS$!Wq5?sBqp(wU?#x{GtcD5cqQ|5vWTD-+kqJ*zmH8nUV7ris>tL z^&}E`Hpn6t^Hj}S_cj@{I-A9RyY#GozbK_qQ;Af{wLsW^Jvr(pi?qOWo%LI)VMqQ( ztz1eWe#-l@dyo$ah|VfE-2V4fLtsZU#{2o-Cb;EX3|jH<-zh1UXmf|$#n}b27dd$~ z6MB1d4#9#x7H`F(G6mIw7}Zn?zI+b3t4A=bsLhQex38n%9|8!?2x)|xoJSjd3GQ3F zF#AsI`wo&6|W^{aN%W%pXJ8B)&xk#EEp_|Ff)oUpE02m(AiH-SYJ}Q0oTxO3TqaxhC=T5VYPBkuc1ZKD;olG*;oG zpnaD)w`>sd>4J-j(p7*%6VO(6gA`5j#Vf35-PcEQcXJ=-7}LU%Pf&z6l;Nx~t#hA9 z59Z3T0UlRXN44x=V^||c;BcQJmo1&XR5?B08hjwI3F0;nFz?ZF|cNoWz*c}=gO4Ug$-GgMW)>@4&_Nh)l>K8;~b#-;s zvy{`OfKGgb%r*M^u7J)7SjGw55rJdUIB9;u#1-n?rLxx6ydqzcmdS2bgY!fny=qokLP zWI6{`hb>eu-~ak%!X8OKmn4zbHb!K0>Y#O5)+o8sI7rrT+%Y?5+a9a!xfjW`BtLIufXw%vIgC18{aF(@Uw9x9koYGRX=IxY`K9+=S(F*8;cz-;(WF$B#U?qpY>0aVp@ee$u(UT%aAe5NL_SG_BGz|w0XyR z?5fNrZTa*#iQ{_@hv#;w(df6qR~hwhJa%ejxDrzI0X6Obi2MI)jXY}4$*#L7m}~eA z!2gFY?kSHe$8LO|Jm4z~pLG-c0Sv!af9S*$DO<-{nL zY`tP{4C*Q;l511_LQ_H+yBYRTnuRKyjgqpun)v7Pd^o`ht6)QR?lYXk)k4vUIbX{@ z|1Qvf-*V~qYKlVlNUl1d5sR7h#84T!E*#JMC|JFC!R)2D@iRMR4uHD^eO(WDF(lQq zEYb0Gk34APg=fRX7pPbA4LuYmk|G~!ChX_kqPgqbhaJ8zQVFie4%iZ6qx!n z;wf?}Vf!0l-RmP*(V^zsZ9^$SaDI13E)3RAx5!{j+(a?I>?biHHNIoUZ%`knE>asQokRT9Qih7{+v?fd_cL@aylWX7UxkR_@XMu;s>$-+h%Ray@xNmd`}6Zp zI4xnjFy2+0Rx}=-s#W8Iu;JC#3ojBlqFwMsOPB_CelGa~D$dZT1UX3=%Hu&PTF8~J zHK6q%b0dWT%5ne~@ANhUR0)AM5ZcG@7|>`xbEEKRdO*dNB?PQKNVJ38%+*n;&HMWb zt67W3-yIps9A%>6)1%zK8<2fPZVYAOd-ySN4p%Nr$EOCOv49;LT&faMgpflJJV6wP z0t>@x^lD?=Ar@r=Rtc0Tx;>9q1M-+4QOep30DiH27b^wEg8+jH@FR9vw5nN_ZA%ok zt|yr7POsI}W^{JRn3166!n+@BFt5S90c!ClJNqLK`^M6^>xbRlPf_*BfJQYz>Z=v6~>T^Gu!!@};MFwOwx z^jMlAT2biC3_h2~BeFBdc<(+%7_GZ%)jdl z@x>yhUEog)owOTB$Ayk);`FdSwSYgE8+GFzucz=v=P_44((P=Io#aB?U1H|Eg`UX8=Rs_6&Rz({ z)r}=#$rFWg&Fp`Uy$Xy{6c{{~BP9T0J0xmhRn#CDOFipJ%3ZgfFMCTB`qyW`%v%6+ z0=S}Alg^Z(Rj)a@{V*!-m#G}yCn^8dQ;7@Nl&0|GpXVIO?T>;OJWD!Rh@9oNe$8zq ze-ibY`nCGnx$yCpOslJ~-_-yFqp56-Id%;!pkoIT(TTo5BBVe>{5aF`ltjX>OO+iZ-%m}n*@3<@Kz*oxPpbNvFBl~EXp%{zLwefv!O^c%Cj_21L z7&`Wn7XPa*{uvDVpXX~odL^j4eRLd5l@rjZ^6fO}+J%2B98lmqZE;((|A?!RyWIhZocQxx&KXpMuTm!-n)+F>Z z`i>o|boZepX?dF?8ZDeb0)u-hh=9f}$!Zl6l1R)i%5w%~09tE7zIEKQZcxh{)BW1> zvF(6cV&ZbzejLo$w-2Tog;aI&UI+fJeHe?Tj*CIqZLUgvzZZKP3P#4Nm`_>0~uzkCENtiP_6YT#HmXIKH1GD5Ese`lhe zwai=8FRl-6DI-Q3;aL>MdZ@gd{%<{Bl5#L_n+yInxg&9Ub>{J8Mat~dX6jdCKd+v3 z7SjZUVlc!Q_-$=vm#Kx@g2Dr?5fL(s9bQ;3EVLQ(IXCEmq=~%20qN=Oc+kW-u=BMg zsYoqJJ3-RE)x$H`^zJ!S)^KR;x{o|jR-UPI(g*8V%>LIJ07dV857n1CrWZez7$dp} znHvj#5tyhp?iT`iu#BavA$JN9rcP%xaH56r^3^snkE-jJBf~&(EO@O|KJ4As13tJ- zSH$yyB`UGe0qZrgwv0W$i^9E!`8;Y8*Ke#T_E-Qnzl7mH)d%;_Dw$CX_mVZuwG{jk z830OD1qB28V$wq+n~q(wO!o{8c9OyJo57Hm`*2;@3}+jyWPwGV;JP6EI3R@G!?Jom zHgLcTNdrw3AAPtDaY5*gFTSoOQ(7`f&})7hruU z2wtFUy3!+VWCTn#IPVZlSd5M|nc421-rH5#nt#p^w+*F^Y@yT7HF1*3TO;qXjO$1IvwjLm|VRaGMAh*4Onoq3TxS+`!4$OCG|lxIx^)PF-)w$V-KN`q?U` z=QCFRYFU;n5!}WH*ImS1L(md1f?uctWYWyc44`yqm$zj4h4b&>xvq8>)2BqIk4|+O ziiBQpDJu2}(K$(dO!%NI9Pgubm38~4QGZ^!FIi9TwGl}I;wtTZ*IjT}4!vP;BAwMw z0v&wW%d~p{)Tykj?5VoCfPg@Mq6irc%{L0aaA=S(Z$v%K`raTAs_;zJXaX=fKvu2j zr^o*38m%zM_E!#$w6gJ%!;T2Rp5|E(r?hU^>)70UGb2DUYx@8Y2Wt9=pm*@Njiz&& zepc*rZYeAK_449al9$3W;QV9opYe#8JCFZ|h$9JtTD|;F+bzyXTni%6>G+bZub#g{KC3ZdIG;6^GyN|1$-jvh0L9o*m zCmS#6RahN&hAD!Gkv~ zQ8s<|sBcvNN^scYl*4cO{^qhGb9>srvU#`)6ay(ZdNNI^v^__ET00~{&B;oe^CS1y z<(B9nfoCBaC{8n$emxOlu2?$YmoG$f{tY(@QZpl^OaYP!LQA{s08#h7jRco}wzOST zTzq`6{j*2LJ9CZCm`-p`Z?mVrk$jQuw29BT>lirZEXDPOG%C|*sVV9A-Y3DOTfGYH zDNp!%`nY`?ckTX!)aEX)F2|oWUgpkt<{R0V2LthYV~z)nM559NY@0;1=IuqooTpD$ z2P!OXkY^au5Y%#F$15;+-R0)q2&6Kk7O{V2ZEaotdb_E38Fpu!4OV^s=WYvM z9{}Lds1f@SXF;pZcA;L6_{A zqaWX6xUQL!S@t((98WFm`r?#yc0CqZ``X?Px?IXjCHX!lxADRdDy}m*=o@eSn_W~( z#9To^L5laEw@P%S>4`1C8xMZD&6dWDEcy7w=$+C#{P*u1wN!OtVp9NMt7uJqwbA(u ztE4xg(WO0ODv5fyW*UnnWE?A4q1LzFsuS=L?CtF#*i`f7`OcJN@*aSnu(hO7&j1h- z0KS1{EOms|uXAZLjG;D4XOOc;VfM_(?P(e@a6Mh|I$kJb_e#-o_fbc+XwA%eOC3-7>1Rtf+G4>W9shle8(l>^>9P)y%{3zgjJ=>kW;e#6NH za3=nZp~DZCCqn^JH|k|u`M#m(XND|1=Cjqi;4rzCc= z^zMF#a|GH;nVvjE`z)e0u~nC9WO>*CE*AXVa);EBy_BydXx(}V{x>99FzFzxizzHB zT6aNA*wAVCs49?V*yHGDxc7JWrQz}ztK*78ma-C{%LB$wdv1>K9Wml@gaxL~MtsaO zRXZyY6Q+gbe0!_6%zS*|;;XjRjp$y_i#I{^xsN{#&8M+CB)xtw#8tn>2vs?1sxPnC zrE*^#BSp{pE$4FR79%VOJu^VmLT1|Oqk3)6)I4hi>sPP^$sq!fzWm1E&`g0!@*y8u zubRv`@sqy4nhepCKv7EJo*(D606&-bji)Mb?so=mVD*QpD%sPumrq{yn?bFZHT}00 zA!wKT-}nfE_o!Ox#p7d=wC`9LuHW-gSuB%`S06f@{PNpK$A|w@>gy6VmPHUF1Vt_H z{~JdpM&ES9FbWq0yvO<Ym;68|~gZ zlV=RNr{Sx?lTB3gf@}*a@0~PnklOxxQh$LE4@Yeb${QLQ&ZZSKl>pM@KE4vicVw@> z%3-3@jSb_Ix8`zS_l#HcO_n;-_>mc2*6ak-8gL23a>P^q&~Tb<~%<%qMWS(~%Qx ze5TCDMph%2Llx0KDFI((|2IMadyY(2*ge$g0dews*4}t8-ui=uxZ&_9d-y%EYqx13 zR*E;yBe~fT)?6dcJxm)L>=E+ZKM{eX#(>{|S#ce%OhQ^#2*}u^VWP2ynO4r*uePg`m<+WKXOY`UVm!f%eiwi_g7l>Bp zes8wUkB(mVXruyIbbxR$uqt- zz%Pl?>`4a_NE{W__~Vt^$XP;q2>lMgG6y(K^mKdY7k=_AqzQ62kpLb#!tk1jVa(RG zHB6HCJc$9l6<7wFj%N_vAPwVViO#LVps)A%2+qJa^kmDzTZ7u3B;9w5N(dr{<}wt4 zk%S4df;L~l31OHM1^>=2CK0MgNOx-pTB;P*S2c1{i0YU3c+#k|_?Q#aU z{=Bub)qY*9#|vP66EO zsOMjRH4tBdlKlVhF?evKkTbs8l@^-r#noU(6!$p#(t0P&W3=uWaB;5s3r7NihUxv* zuvxK~pC4`dnLme{7589zXyM0*@|BDY#0mPp|I_jiC?uKyGWJ6e8%j`?Q5?kcL7>Xc z5PI~rRyVPs;o;goo88F1l+^|X#P?_{7|`e8X9j_fFA4VGxJ#v(Qv#TGTo4$ED)2_05m29e zrjZ7(lL+ z_@B}WtZ!$h>N(w_QdJtrREqBirQya6D7Ji{g1)Ydc289a_%?%UE(|nWkDCHiw6{V7C4;&}KokGtX-2{eP<(!rqc!Rsvz5_s?la$6!tsl=ri%^Ek^XVE@v(jiRKI(X8-v zgxy0=tWGnO%)a#-QHIukeTp>G6R#^9UlU!}$LcMqaSz-8{ednO{2JgYVyljQLW?HK zdB{3FyXhx^-+^~=koV%*oW9*i6^1;Sx%j5B1WgNV|b#`JO-|amSiK} zvJ?3N@nD{ZeGHnDOUejXtc*+d!c!XP#yDsuFj*X$q0>kW{G9bD4w$;0(wYPv!R@iv zxU=780Vxws#T-yR>dpTSPb9;gcJ4V_G%^ zQ$EIcehRGXO9pjLH&bs7ye0lUH8*Ivvp(Zk9G8I~gQ>9?ik44Bk!bbeG7lZMMCA`J zFdD95dbpEqJa^|Ahte@R0@2X(;Ok4;81Zq|ndbq8lzEtm&D66Jl^i0a!TJ-nV_rqh zG65C-8HQ?T>#Sy(#dx#Q!!)q5G%q3F_ixOU9wK~?y{LfUYG}Pkc(^WKJW#3jjSZ!T zFc&d1QVK*IP+dNy7%2ScJH#V>yk91+W4SI^>QhZ)9N9Z62uRS~6oNG)nK*K?vZ`;T z8wr(j9;3y55p6$|M^qm8WvKF`6{a6B8R7EpHeC@>Rl^u5FcP8XVKAYUoV`PXdQzJ4 z`R^->M;!2=td_hF`rH4HpOsJ`vg~FYxnWI)FS5i*g(2n$+s^vQ%U>G7pKRZPv&P&5 zIGvbM{FrrUE`V+*1?WChY7hw#$B%OWvj$><`k}7Zg+N?HDPm<`UgRtwgc4))56Hz_ zeQ1ma-Epr23_=iZx_sf<$Vu1}@hr!6!MXJ{i`cym!HS*r=*zW@$DCrQ^EjW}XCz@3 z0`{6T_Q0AG)}b|<#*F@H8p<6#Pr~YL&QY{%C?$Ycj-LkQG_qT$Op?s`ZQf*fok-?C zJEszk*RI%1fi!)ve*(kPnW``qDTq_BWpRP4bj;>SphZRNOO4OeM0@!}ni|;(#Y#hI=r=B{HNi>=T3JTSk4N3wnc&-$R*o;X@_So% z8N-}P&XZ$m)5RDb1MZl3{qF|nKa^)bgb>`*5J&j_FF)gJK$+w> zAiiRDBq|u;IK9FIS9U%injnJ*eyDYWZqV{l_8L6=|NAdE4QR(efe!?e0Jg+pzDb-4 z*N^zK>n$v()>$18f$qWE1|b!CI42U2H(D|sfPUXUTvysBdU`j;!PS1Qj95VY8>v;n zBm&Uk8C)%NW@{J%#QZQr5Gn_UMo?k}gpSNfp@MPvO3$AugR&|s@_vi~w!@meNoDE&bppVXEZPIcj8UqQ#3-+=t4dI9u< zLJwNo0YZc+W;Yt@Y?q-gj}iwXqNB?JA!1*e-J0gjbRN#Unl3=tTL0Bia zGDAL8`Zs~l3|h(@MF;7pyowqY0gx2}zVq@2I>1+MdjeI+?n}UQk(b9IIJ=2#1j9I6 zFJ$`WLAc#4ltW6a_C2)o_Pzit;vP2|zNV8GAg(5|L_1!J4UBuO)wCe!1{5}+6?^e0 zmLOM@IiuNC9+eO)+dQpOlHsi~-ycq2BW zx0axXfD$&=YO=?63+o}ISKi~rjJ$pO_Cz*L3F8UH_>0uSt%=eMyaVpyhBDAZjUq#f zu&}VX(wEr3kqgP3MUQlS>u@Uop?Jl3rpB%dl-{i@DH$01iq|tKfXTM;q?U~zUeNfW zE1Qo|4fh1)PgGR<P820LMRufv%3l$tEKUO3Kx7Kn4fei*V zisYljUXJE75@V@dAso>o+O)z^+}(@wQ_0IShaRqqEhi7>QuvP#p-?3Yg(BJ}S}Ve< z#$4Mc1p*=YqdP5i-nchl@@0yp;4o-;NTN_k1ng4F{1xx|2~;$(864LDhA&7F$I9bW zeAolIY&IXvT(|{L6pCOu$f<>s&MLjgYVm#*pvtNPpGlW{-_UTMi)qenl}^LJZ6%hk zVb`ydp>;@N7aE#ose#f}9IL`7h+slGJa%^WY@ku4fwPwFx<*4rc)Yo&KH$gz6=p08 z@jB6a)BD)3yH33b_aj=q&2`;dl=jRlEC39tDLf7m;D9ZjL#!5%YBBT$*97eRcZ%8{ z=!=26mxl&lf%50^#(4PIY`x1c&C}w>qZ<@qbYhe=b#o4G^70+`G2{{XT7~X_+3j_D z1Qs_~;(*>kCt|-0D4!jgfhN3oP^c6rhFJh>9cjmdRj2o-g z9>_XlX`-?ctQVOHz`#g)p75Ij&qEOB)e$WT9bHmH1km_KFMZyI^@fImqTP+)7ojz< zA;NsmK5X2klE#(9<-h&1K0U|>FLv_ z-ltnaefWJK=|%)2Hf9zT{R5Whdij7XIOX!P{n?CV#9>nS^feXG%*`Z0?Sa1d{dOuN zaw@7}qjH>hS&WJM7|Mtikf(}_jFcj(-E{zIg3Ak!JW>oM^1TPw)>0lZxT0Vu07mFL zUuC7AM>-<~edj6SEeK^S!h61dFL-mTeFpXxO8WOLi|y}A3(qZdYHl(ke=zMr`>Hwd zZJhgee*6G1WS|&A@_+h^ox-){cxTV|I3-~B6@=RdlS-2V?n1ATKh+HMHqhRVx?Ki` zIOE`%6F6OTia2VoFd?dWuYVebE5Hhh9iO=YbpU%1V0gK7K(PzS%oT_41NA)5#-&$4 zoTgS|fDw!YAdUdRFQ3=ChUaNO&zRKudi1Z+e%#*A_m=AH5e0}q;&y1GQ1jWE z9mj8Pw;S&yBV59LKhyyPb!*Vg3S0QUbtH+@`P|{@2bs=b6y9#o>}tY?iVo^z=fl_U z8+{1h?TqGARSWW!!6~K5Qwk8l%}FwP9^cHAK8}^((3^POwGp1#AJez7+pq^fOE;W! zG$ZFp6PL%5{%aQWB+btHV?f?r|KeOk@wtQ0s|Wu2@@$?*D?|+~?V#>>4v4>iZIOxE zwHaOn%tf@?+Zv(@S}V8x(dE-DcCLC;y~W|3>@>V;KfdaeMFQT}z`k$Bf}DG=3^8U6 zK+%F~(+oDHbB#;60dbKL`c+Wu^^%KhCCA`GM$pFg_G?mO0KQ>*cg&4~G!y20TlbHf zOrud8ZOpFU>vNYcH$cgv)EHoRA9y`N07&+SHz^WcC{S*~7PuQ-hCb(q52q_FgQaJ9 zHT+c-pgF0A7`K`kVZH-P!%W-aA6)0KynememJHu^UR%J;|0&om-jOe0z~oU(a$lG# zBA(t&&jYcz9O-vR84OSc?k7M7R$1}j`1S94W1fvxz5L}JZ3$~~Iy2vzMyl%C6Yqyu zKh?qzlTOZCOGd+<1Zy-8i_O)73M@S-a#Xm_eTB3YJ;Vl4`CYRCL^WsVZF^#+FEwb- zgCa7r4JdrO@*Zf0!6GS2h|@u@CO2}Wq|c?Hh4WCL0O^v!>zCSG)-O(46d^(%mRyTV zu|HMsyU#Sb9B>y}N+Ix>9VbBd$w7`Xumx2F%lva# zQ1ezidP68%qB&+ktftk8x|SBwPYO33tvU&rj-S@}((ghXl{ufN4sX~wmUN?`xmo8r zC3kC3X+UF56+vSEkl=!};%auEQtZZiyyF}KrBGG$J_?>mayOXkYnrW|{q7>W+HkqM$KY)1x(oQK2 zce6()Fm%2Rx?WUzGc=EW4AApmlsMfKuzuwXjt>DG61p(5!Dfa+$z|DL5~_mkU9yb1 z-Y6f_LCE;~@gh?}t)1W(0z~b}>dw0-wU9hzKG9Efca*M*uM7yNv;0 z3nA?vH~xIIRq{lZ)#3U1IZz`d7W-RGzaKI4v!`B4k3nnXuUdoM@$a)a5*zn7yLi1- zCP>QQM?7Gfx6s0`#r{96y=7RH-S&s;q8p_fk&*^MLP|hFx*O^4?vPX|>5`IA8YCqb zASEf?9n#%(=F<1wd;hO1qu<@%qZE=4jk)h$Ccu>NK%b%fMU|a z@oYZ%^W0>(#))s9T}zD_mCOE@)cy)L*!9}ao&p|LLdcIN=vuU6!GW>yvE%E6YlYlH zW*0-2(`1;7FTG%J1>Og{YE&c_KmKVo8IY{nVzOEOfyTjskB0vJZ7H|Fr`~ibCWD1Q z9b9kGuf|-EJL=2YLASN%L==+u@E`E*%v)9|5;|SU_GuIp{fQN9Qq}AaA>GFRSztTPE~P_g(Iznp4j~h6xzi86%#ppJEerGjCwWE?5JZ zly7_ns}M?>xuf58;pgWaz6(dFyFPwRip8o(ctXp9!dc-zvJ-m(>(7#-)7`TNPCcoMM{W5}7L!_owsN=!^7b zYEqm9bRd*cHip@LMSsPzv3R@S;HeBs=d z?}gv#crj4TnXnst(?U&(MWZ#7H9u7MEX2rY9o6fYh>-O)wQuh`pJ(Q3CsbBP1=7}5 zOuIFOisXcyzBMsvY3L9npu3iA3hxK>r^*!{!xT>B5UGePPVv=?2$FDL&^7}bampD1 z@!fb-{+|A!3TvA#>1p{@`?9)S4_|D(!h}#2sIdazj&-WvaGx^Lrv#SU(CSQc^UfCQ zz&s+oPL~!?;j&bkj;FOO>C_a)Dw*aYHSOt^(I9k5>3hSLz*js?(QpXp5_QW<#^UIb z=W{ex3gIiK9LLom+ABgyI3QbVJ`V*QomveEmrnL^E7JBv89hGo=H@2v3r7$)W*tKj z_zg^u{X7{$Eg$q!RP+WZBfaT-pqF=OChy>nXw$=cwO;S++IBiz^nA3r`cBuUQ)xF8 zR^-O^GP#mI_=O1JBAY*xVs=Qv9-IH_(owx>)mqdSoW+;X?ze3AppMdAK~A;bK)3_1 z0DU^!a3;RkLyxL2`)4d)@n_2IEc>Uwcp}D%ewtv_el8xb8zg~=9-(h}Lrw8708Ei} zu91(C@q|)NSvT6+XN!epP~)!|U=56ntt+zK!i*{dbN6&I{)@!$3T! zXK_VDcWhpOSBQGQZSXs#)nP){9TM!~_)dXZ4!2DDuD8IbVF9V{M*--~g-5j8Dgvomv4f!%1 zQF)_(uX3-Q@y1?C7Zg-t-pYT5_;6))MoRv$6>NUvtwlNu+ujsJyk)fzDPFYZrSS}= zaE9p&rjC{9@XUiGd>BZiDal?+1s9j?9jlg?7vd>OFqLVUD53yOg*yk4@)?;3B+z^| zKZn(8ZR$;c+C^M78Tt80MK$9OP_XG@kgwV4?`Qa1@=?zKVc6ol=z;Q_t+&)ht zD0k-o{Sr5jZo_*MSQk5n8C`VwgMkc=!ABs;+8S76!OQ%uV9E_eeSbKOpb-G@7OM0& zct)sj7>?ER(1FYaZ!QHUsGMY{SnLwOV^!(SmYnh1S`Rj~TkTib)_gstzn&j`GaLFG zoOvdQx^`5r=`@RM%R^_fUOujZ8V6{0YZKA%7Z7DN^|MDXvayqIxZuKhu>C3x_ zL2M@9aKa^hfaaaLKdnV$S5|Y%o+9e)Srx!H65_g|)%F(SyZum7+6b$-fSXH3r;;5K z4p!6DRkWA#VVbG+_8t_%^mdTI`5f+}S;R43$gMO8SQ*U&-n(0@TQA{sH|A@;%!rChM+zArgjlqfQENTRggHF_gZPvX2#%iXp_5SmKo~OLknBAl5)7Vl)`dbcX zf(C$sYj|b@JNHOHWs~W?QF&+NhY!mU=|sFYe(CCsyCNzt7)1} z23e&83KVy1{xfb7j}Scc`uJ;p!%AQ!lgH5AvD`YRP9_e&DJf zP%YtQ$(C0*)QqOkfY);}Sp*xW-JZwoD5J$U7+(3~BC$X4NSYFa?Dc`vur6Zg>9k@LAKk3aUCJW8Vr0n@D zz30StH#^v8i9fpulwtDxNeZRA&OF&ogV@5`FFhCq=2^|u|MY**h~KMXr>uq;~{!Ls{sY+ zwULqsVI0`$%Y%VCcPjTgtL$s#0zf^xc4W{ z7`Y0&yJ}V+7V7rW0u)L4jkm4Wb_MFg#U{z)%42DR8SHh@0O#WpA!QVH8e+d7{PZnm zi=<@NtCd2HU&E(W;SoHwG=&@e#8Bh1`C-imQNIcc84dty3XNbIL z-mIhcChoGnLl>%;@j*5`YHNjDFN34CUq;CDl(pRZIrC$;)vv9QJLF4VVa&gd{ zC(noJZL}c3`C@x1If*MuE{0l;Zhi22oST)#XpaWr(lMm(Xk z0up%*$fmhg2)*f;$|8Y#JO7fF^y-5{LJmvCAnpSMO=e|*BZRKAmv~&vdpj`_?Zx`i zs%PEdUm9+iP^}mE>1ZNSlRlhspd~zz^x?zYT-koo#v_E;MK-v+mqqSORfUQV33a_H zZ#a$29Cr6L=ytm*4=YXKh`ioG(1LhlFr{H8@{=+LrSX7zFkBwyCm`cXhR6O*jSVqR z`g56U4b>z5B(w+|?}65@u|aEPm6SxSU%!F4Y4R%m`Y6k-eHF;;O;XQj0+El1HFJrU zIIp#Kdb?_0iC3sFKEjz6Kb-sz4Ru-q2!UwyLmY$2&kJwVNifWR*_Zt$;jLoczr(Kk zTK7@NKwNE0M`F4RrHB#FPT)ew*Gk)ZcH;>7BqA8|<%Z^=S-uara{IPxO9zPoxsE3uTd|xUO-deMU+!4WBG1oW zHt)6p(4y$aAsmBm=KdaD)91J@A*_9XKjVMBA^FhwO0{zDc+lkJk+fRKnbfc3Z$E+c zgmGjq_43*}oY|dQ6UkO{-ysi%#jy&4p@&$4g-V|jO~8*gvyKwhHB-36d|QY53(X`< zc$gR+zeDi0=6Vt%6zocOZj*#fSuON#(EfZaW*>KuHs>Igvh7(QFrywA4`e*t3jX}S z+c>vayF_ih7_G?s>VQUB_iB2Kg`hdUx@;J>Tg**-{N(E;E>b8_K09SVT=f?B@hIh8 z5U8bb9?_}<^Vx&&DDz%aD{JtbqXc~E6J7fxpI^r!8z2 z6?HOm=a({2b?=!gFU{}{_mX#hVzo^005*ZnzP^K+Esug)ekD}OA8I{HzW_g88t^)_ zTK`Vvi-~?7a9}V$rJdOyZ>CY@#xO^7*Hu3z3IbSb-LG|CA~rV6w-f|Eg|2V6!?g$* z?56#}rOz_ua&oq{fhNpD;x}duMgj)zZUt$K(Jo3*09BE+eExm*tN3p;9>7U??6I0d zrgdJOax!+i4>69ExeminNjUR{N;~crxpexnBEl8l{`d(t$UpCiD1`DcJ zsN-n7F0iu}=4JD;iNyWbLX$K8>mi~j1vbE8iT~r*J+-H7LrZkbOn=N|K=mpH-$Co2 zuYr6%5D{<-{3kM0uBiSe^x$9+0hZWzG1Nb?ysVFQL7dsJs3)L1MEv^{)DHZE_2JCP zh{#A<`YI530Rb}u5#TaYy*ji()XkF);d6c*tRDWM@L_l6!roX>(iv&MiS7E3=%0uI zDFhZOl-AOK)(uyLh;h!Nk0FWw?|)L%p(FWvA9cdOYJQHV3k#&cq9A`-dW-MS;zt-f z9wL&9I018-LGjL?S9A17ET%b&rK^L5B|z}dH4~Q`U@FM}G|SN$5h!mU`qJhkt`&)lCHaAz5jmc!V*!yY0r!>Kdx%%-?oDMU~>AF zS$AeO8`&F{9JbguC1a!{!oFHwPU@fB+F_;l|9KJjy~f9-<+d|*AEe})zj8C3*IOy{ zxfo7hdue^K0$!f^pSPEIxF~oiTon;v{X8_Uc8sW3a0GXMY_ep9t=nNPJ25!o*lOng zJT`{Ww<&jyYX>oXd3q*Zyk#?(MKz-H2Q0K4^az1hkiL5JD?0iDNZob}vE1ZidOg+_ z`YNvkI9=S>Fp8n5I#l#s;DN~?--*w6gI2`%u7(`+b`zE*oFk|@Ux^~$^8nhKsD%Ln zWS&GNYiy#k{S@wI2%QksA?J~!aOQ>VU$$T{Z1Ny3^<+3(AY#A}P*qqgF!zG6Y3|6WH42)b_RI9Jneds$X&x-WmL!zyF%oFo$iljRdonCY`_F ze_FR3MP!l5rg!t?WFTmTUrwh~Wh$;`vmU2pvxjm8#2k(mgPzQeyOf^vu~@jf{8clFSuaYIK#1q*s`&2^nQB9r6tjd9 z6F$waR%Q}@>mj6A1UWMsID&e$MV)gP2;@2XsC;v!97O*;JT)*Q#tk7 z$DBn;zF{~aY0Lwq5mYY;Am|0)K8xld8<5dqc5}4#ZQ$A9Poqs|xZ0|fH;9x;0r^4p}>|pn$cUqjXJ+()t?B@$38F>ut6}*2IYgs)(3WC3k$44 zz6L^67WzRtJ!#`v z%~G$Qhf5a=RjH%0zh^377zWT%UX$r*utQg%-T-B0T4>=%Hys$|@6Q|pqcm9-Q2mh^ zM%_|Z2fA>#^u^JQ2f|kjBv?Z=!3?%4Fu;+F^dA3ql>Jmvp*mFc-E>4T9Nv=OqUh_bFy=Npu zriD+#-(nUNdUbqy`mzU}Gd5H_kNNIu7W>+1!?w?c>pHJq<^=!(A`{<0NRb$U+?XTn zSRbIiHscuJ6>`&$+)MR*2orl-QW9bYh9jWE@x^LHvUaj7y_nINH2!!kV)8gj})>3kYC zbv4?-EPrat#P@-lny3D*yBnER#@wS%^!4uRgEr(y3&(F5V~&$Z}~sMphPxha@6eL<7*?MBE5{&4ncEnwb+6*F~9I>zbmn?KHr=`TRqA+D*Je|g72{aS#hY`W9Xmb z)E!~+7N1@A+?0k8se0U&6bCljU}xb&NGQ1s|B%Tl1I}VITjm~cMRV9P?Or_dn+>eu zu4xNpYsARTI}a$K4EFo7J1wRUb>9&DQ>y#-oc{aZ6u|7#?mawq)d>NBmL}NtMkE&HobHv z?bp8>)(3t#-F$rZzB^=wUAfxVFmmy~=9wEe4{b|%d6twt@N|XA^g@u}d9a3ku{{<; zMWzLb0OsTGpRx8sG#9FGr%3xx6!{un)bDCGkBR|Nlj z1g?#OpMsBy1APCz#Q$g1H8a@$^U>dXeOOEX^ZftUPr@$x!GjicOg#R*0s}<(|DXO? z6a$%mj_wbA5If@}$Guyt8eBH;Cda(1&n)ja=Rz)d;&T5C4Av5feiIn-y2rxn>u+96U!gb(X&JVjN z!{XxNf=)mhl+=RHaX+X#X=@Dy=(ATrAh8Lg?q$7t(AhHfLO|?XLFW(rrN*7;zVP8} zM>3qi_pSk+u3bv!32wdb*6psS^PY4_e*eXyaq{~_|A&hv#I;vB<>+QTSz#=BzBkuH z4-xV>cx>95B@#s8x%$Xvror7l7eK24^>exYD6;a93+vyu53EPL^D|tu)62HLwmC#4 zYXW#Tq2<&)B-Wj|0uG>hEb8ZH$8l-AKc8XLlP9_WJUspge>IdnBQ8kByg4Cj*$>>Q zefjrq;lKK*-_IS4#2u^3q!$E;Ac%_eGD2JSrKl;RKv}Lq)a1xPQp-&q$hI5c-`^Q6 zz#YE1@xcQGG*BSz4hn-on$waoHn_g|=>85BP{f~-Wn^D}DM80I?J|D05^a{wmwvv6 zOkYvUmXw}*l&nASxWQ^qX-wI=#V@HikdOxf8^Hq*tsES%fr}3J7G$K%HqCGjA1&Ev zWd`6)q)6jNnctVcxZCY%QBgNPi$Q`!?Npt1H&@0NB0cR|VQIrCP!E`3;f>-AJ_fiV zF$`n!2&W%ZXq(?JE?Q-+c2Zyi+pZ&~syYg04)^!?_qg<*6`7zQ`%cL3<^p6blzUE; z>XGl|`WW40J9vcdW!A9Db`QrMW$ad>W2qkGQF)sS+-=7a-f3QrD+weC->=#u`^jHl zoKhdt-ImS;%bril!T4}6KHUe$|AVKqdsMVRE=ItEW)m21M>jgj37hK(?>qrAder1~8D11Ze6*jm55BODd`V>W z$P$G`K%}U|!6l&Nsf8Msml;?{L63|T+6z>^OpatBBy;!l^)=oc^)3kj*%3u+ok(Nt zTD#lb+J*5#HFQ64pMvjqI;H`%p9A7PeTq;`moPM>;tMYQM52$gD-Y7?Ld+45K*nl; zkMNR%0A@%#r^|hnt0A1X>BSoa{>uVXbx4Mw2ev7v@^OVZZ$;QM21eI>rrfVzs+liG zUuu-%8w3lJy?QbBB%GRnFky)9Db>{3Bn+^0QzYt;1VwS!K&EI|ziM&L$D7OJJ8UJd zE6019jdwN>p9}*|jnoIX1;(c9XXcBhD?yV?4$#c z;vN(ow5y}YSmnTt!LJCVk;@W$=K(RZ5$;UFw@z0N39@!T4wvKJEMG!8hjD22z@v{Z zi<4_@XOJ@SU@SXWMI2wFx*5UI$!q8 zfbw%skV!p|&fji7l8;#kE~X{bw55*Y-9%NWKB!`C(&GPMBu*BNV9xRZ#+m+{P zyXAwu6D9H*zl-V2%B!SZm6&-_#M~s<=`pPXAI~5pl);&Ib2eiSlGwtIe@RlVO}+~c z&tW_OJAS2k`<+CP8Z#QI1q}o1Yx3NflVoii&9ve0u>cos^i={Y=~TJTMfe7J3ojrN zAMm+Lf;NYEiMi-DJJ5(2hV0fjCON$&I+loE@b41T&Tn2Z_?U#T5L;fl;0B0TQ$Z_{ zG{rK4Qz)`3{H-Y2saYoQy$oh|!&45HuXnzO9hRZB>Vc8Jlgay8zb)&nw;Zd@yYi4i zyr;(B@Rs=r8jw0KQn1RIhkOuDJRNzl{iRB@yKayhkOIS6TUSs%(Lc#J&L6s~zJ2_; z>3JiN>#3b}LN~WDF*Cy=er5t>iw=N_JN__T%e~#LN4j7TGIqXMu=-f{J;|sO7`j56DAQ`E?*FGUiMM(kdz{OD_Y)s2pmfOc613KXD_}9m7XEt}mxS z8&r&AS-y`@Oo{Wn?Gbd#r_&bhiFp2W51yv!myU9a-lKEf1^q9lHGX=c4W59KtOK7& zyYoHr@5@p{d2SZ^1vB1(S-JIj_DpNJ&by{)C%K_U$-?&PU=(0B6UL3SBE;D*nHFAr z_+oq{)>b!YCh2Vr%PK0X-(nn?EPQMO5*c#1UqIxgq_T<=VWmKkXu=2`O|7D0{6`Fxpp8ub3@il7Eo(ODWH8BBeD>)@9|v9(_LuLg zt0yk3goT0;;G=y#E`p*tRTY9u(U(+~@HO6yxlfvyyycb{)e$w3JTreK=xWe*D{McL z{O(}ds5x(#-ss=u?3s!ZP4IOEXoM_ML3AK9IjqvQcp~p((l4M` zYIkG)3RM05HLM}kL!{g$96RG+5Vs;6zB{_>nb8k~o8}ORq40 zz#hA!_7z-_SF!5VL%)MfUI~liZl`ik=;lD|YO`o^QWFoSn^ACF8Dz@?1t&+kSBgvq znJtVRKH}T}GVCOI>r7ZG;pR379+REAe_MyI# z6Km50YoqNoCk~gg$L)d;8FOCa=PkJ+{i?xR-+q2du#S9#QOi>gUw8Zl-@O43p-kR! zv!LTSufkcdVwWP^&8V+%JLO!62K2w6#0NvSv7viRmoe!z!(l`WGFX2ZjUhjf^a3!i zakyTd)o+*Pv;p<5=BuqDhu=x%UMIibj`VmXL8kN%$b{O3W_yRH?E6pw@SQD1tr%4cYq-a8tn?=7CqGR=GM zef%jJ;M$U#9hfTgxTLwi4AbW?tdaBmsb;|YKdclK2hVn1IDw_ROd08cXq%S2ujRq& zO`4;hKUgu0rPd%ovaV?{uMJ*$qsf;5!k}M6P#ew{5lCpfkW9g~mGGEyBAjN!3;GqI zYe;)(KcIR!LSR8-HN#iG3qzy`DqchXVln*flu?B|tPEs#U1K%M1_OvwAkoBE-pvaz&VXVaky+lfx7{G*6h@iC+s1} zw>awB$`mwwY+YWzX2SYg^~~d=t;x}Kb@x2V^Y$N2p|hu)RWpF@xmwQztjRKI3+#Bc zYF;t8XZHRsz;YDWxX%E7%Nja0 z3OnUUHZ#e zo{!jlhLcSgs)QpM5#va(UJxIr2m~>F|y#52aG5n*Prl{Zz}I4)!sfwcP-$05*_uUB2dg z_akHti=#zX^E!4(JM6&-el*czxiLRl%6tlWh9cRC9gj_W_pcY(9%9*y??-*ADXUfG zL#imJUs=zZ3qQnxWA+F=>>61juJ@m=XMbz>D58H(h;nELtsj~v5loVs$Amq_} zm)UPR_GjhP;&%2<+Mb%Z_C5AFN=dIVb=yc;7?@}6Wxf=xu@E5bSjJ~_xEd^V++upF zTwfp39N1AQr5#_7{{3){81t`?nz!%8=hAWZ`JhUydo-j{fg6wYIaY9heyfGU^a*qyK3M1bK&K1>E2`pli2( zJ@62P%ozLB%E2S=Xh9(jofks<5MBdYcK zQc;`r_5pY=@U%VELfw!s)d%%RX^w=57|1)Se_QB|T$}Yg;wt(JCe~!v7g48YYYe7y z4ach3$XJt=0q}0_2b?svi>zpq17YP6dSxb4Qv%WSf7=0%Y$P_p4p^>$#JFR%`yEm4muYI)KW*mU^FvQ1I{=Q3=42ds`8vLX3s*;yEk$Y2(x0Ak zmxRZv1B!<`xWRo8g;cKhcbtTEphya^i$L9x>s-BSRrj6D>*0G)hpvK&mw(SJ8*}9K z6{#>xeY8wpm;ff}9>VZ?;3JioRFn#4aeVoc6_%+wN9x=Ai_H0O5!@!#jQ|B8UmM=|@+6Bl~l z2ie#lZBs5ZX@AINcV2%KGif`jkF$g=_LUOPyOY^lao$HKgx{15@|w6a<@$_aZ^V$NSW;+C2JA z*_n@)lq(x-q94#PW9GE2Y+@kFLR7Fk`oc?mKta;ptyS5ZGDUm#N*NZ09`gS3I5XI= zscr`#W0TL<+WXg^(NONprGoVbUMD+qmFD|3ENLSl zJbt22r{W7Z2?Xww6?M~L6KK&|YWn4jGxU|kQ?OQA{B=@FlPu@bu*tcXPbb1K0EO<} zi1OyGw?S{lO#ZuEL)J z3L|T~L}B4B4C_{uo~*+E9-F06N4a%tj57I!Ez)soZcqrxsHII(I^>^St%HTDddko+ z-ekaqa4$F9RN~xh6dr?gZ)2a5%jLJj7P8lwSIAZ$P@KFM^g1!CjwB&(^I@NweKUX5 zD_7lFGmGH>%?K$ZJNT+Z1@7eisNpq3zhv%tEkavIM@I?Y?A+}fvt{sk?_*1j{v1uM>%*G zZ7t3}rWT%NwYbD0cE{$}4g0^?oH8;L5kl8A^~tHmsz>Vj+_qwQ%woYeSFgtF_vsrk z^SV_BOJ(7=UK9EA&{Y*l#AweQj-Yc>MO#39^t?T#YL|>?T#4fM(XJ)t5sLo;%xt&W z;2vChX~I+!)enfwEe2Bw@`*|5ukVwDjssLtx1W5qzXSwLn_q3s;_S`8{v1fE%K=>> z!4V|n&&+l_GSsu%+CtC)?2&=pDEJfXszVwT1ytH8% z{M`#?biqKcUa-=^h^N+DZ%S1>xsKDjqAQWZI!~{lnPbauAV)TUuu#Nw;m3;zAUNl@QK*2 zDc*_tiOfNmNt6qAC1PE1VF3vicMKNds}tS1ivjEXO`f8Tw4emq$gm{_lIv2W;cw7e zlg=D^IBgyDmGYxTTc2j(w}zi4qwwXW0c}-isM_)Rn|O&Y0NeD4H8922*s*!;?>urs zuXoD(DNrLC^63*9LDnz3p|6V~W!RSnQ%8#&E1EQistspglRn?izVVaZl+M`>(-Z%I zmaO--Z&iaZjU$$I@lyoXWu8=L<1=&}JD-vC2Y#WshG75Pb%b_BQ*LDAfHs%2_QU!F z00I4Vb;;19pZc*o=lU;B7^b~G`X*%r@U%4kggsUtWl7xPepNGL7W5>;nH2;oCX%sl zGvyX#SjoJ{O41NTLdGBqAR%+upKmT}T<)dwcvseZWd$m@$M#x;c}hL zMf3g2nO@)K0w$0yTaF@BXD5Hxe|A>&WNJ3EN8fhm=yt?W}(ld+$~a3*eOem;QQB| zwewuCZIXnH-E#pbsx)3%QCfPCl9T(ccK`Cb)xX|Y@GsMAkhkM~?4UaMrm5#tnfGhi zlCS2TsZGl#$UmLS;)BvFAdp#%d*EM+?+Rz4Aasc*V>MmHutvis1Mi5Yi2ZWFBUm3A z&gOc4jQ&_>^3t!Mu7POgIVbEqv~qyL>;L{yD!3j5T*JEn=%Q45)IjBk`2bjKJ|-`R z!0NW?BR$c!FUTT=82~}0t!7nR$TmnMI>(9}!-%5ViL3xb35s+RV{asoSha5m#uOrm z`7_AaUEx-P!Z6R#QAnB&Ht*wp*2h#dal^yx?pVG>2%D?vM82aK7!7BBjDA+BMw&=y^pbPAG^BtY0<4_HfVtia zJJRetK7-yzta0M$`{;hnw4O$2gMQVR$E{Sxosn1+Vz&=4f4oL(nK~8md}J-Rb_+A% zfosA%S+&WzCgkA*b)X7TlhD9Quin{;Wo6uK^WL}&o9a#y^r6!A*r|B^^X2o0%stvL zN9Om%!N(4iyVu8^?laG9cryk%2Xuzu+DxBJ?g;I6Dxf6z99*Er&F`2^)S`?nV8doW zp)1s6H3--cymmgnfBd==jm7MgXg)%ZW-ZL^wyx@?aL>&RN3Iz%mEds6`=urC`CB;B zSqtgsek1yz+3c;kRG>f3PVSz8orv43BYdiP-^l%%M-Y9z-Ld2RoEjNb$fI2A z<*%+Qk^8ZW96Z+aYfDkJGkQqxkNBobBU`FYIq;z`6k?5>z;DnG^Su`LgvD@faHl#Z z7w#i}ZnE{sLATn#p_!R~`U zPFS()jtf*{St0#<#tNtWgq1AA1XJzRUA@+JoC>J7L!9fru+s@eYtSZK2pmuT=c|hj z$g^eoEr1rgW0cBm`_nk#p}d?eDQHnviq6mk&R{LtuWE{3J^Ss7s|_=8VYW(ox>>jO z)0Dps!S6Y$_0z_vf^NCwBlq3(JS+KX4Ub`Ym{m!OFlStw-wNB>(=A z{=7^MP8|8>XIdVaLtIvO%Zj{o$6N=P+Dj=4BJiDp=d-xDxb1{L;ktDBd50hq#af)y z)YO84f~>4jci{3exdjoEEYQp^lakc}_1s)1E_68~)lw$&Gzyrk^a6kYuI}==^=pzlh zBCXA0>$v-6<3e*>yy``IAt`-!JXIX_20e+ONz8>=2lTqkqwlfC&ROa2UwHAIDNw~m zbP(dLqaUghFW6EG9Ls42z%FA7+I`BCy8r;A`TD1a2?Fe8W+9X>Tr5Cgwfn52q=-nX zakOBk@!OH1&cTnd-;78Roh)UHlC~}nRmT-N2;x-B&j)1*^1DMw5I&jdDNrPU7{R1d4i(;Q8{PC?whK%?BMHs zcKmpL1A?9-9aGR3LF%nyg{>9Xh7bubNC3wRFu517NFHFi2#h1}u1~ivVO;>4i&z*| z9)})KbFDP1Z7IhTfPev9PrQ5o^OU%E&gwQM=3Zj0no*0UMaQ0ouuI7JB)r{@e=(Hu zYD{PBFr@r0);o^WQ~zqP(acbZDx_yZ8Y`JkV_k03-*3s5gGHf<%suvlB5&Da4JW8F zP0=_b*n1Jy0|Zlm1+5hJ`YkYUz6B6F^i>D7lRE&aFrbb*J3I5(&-dj7a5|Rh3{_Rb z!{r}^7$xg?Z8Tu4t7K-7R68kXd)VC7Qx1b+E4`(d!E!J#kfG|B-rnIFHz4&Dha zBcsLjkh6!Ibd)@6o=H?%KXegPgbri@;KispAZT#Y)0>0q?|-_l0;>*410Z8l6zjM6 zt^G^@eybv&_+&iz6gtxn(J0iVby;F;R$LCaW?A}NbNz{^Fj&c15dnLy{`vClMRd&< z<(HWRD(-4s^_kUW$ClSGGV%Bq*LJr`cV2SfwQL_r^o()rcQ?{)YM_ny8zcKX@uA$* zdp#;GuDNBuc%H{ z7Ig+3eA+E~*&vQvRWDEKvt7dK2KC0k;++hJkCJK@^|X0djK$MDBk>aA%{*#LA?)`l zYISqu3lJCD-rnxTl?^a1B_|U?7I58CjcozG5op||z8SlY-fMN5SWOZN@U;_>{A_LUk%w0x_RxIzUfbc5XCAzc&UpwtW_LcZVuMGcdgIO$`kV z&CJ#T3ZvCn?$0~z1^8cW(%aj^jI*=>oEwauk@9Tohc_y+ULKW7Hv?ZLfhd3kc92%#m^A_u;TK%r#HCcj4SuB}OP;@J zs2ZT()$u)StLPTS92E7Rjha2kyC^D|Xnu8=a~6SXrIH3hlyGR1aRsUc!$8ibv9S>d z5x{h_|MO~;Uca;3!?3@6`9e=em+Ft^l@r$Lt~0{>0}^Mtp0~C}xJvj=9Ry&6u4lQb?T&m0YocpmRF3d6r z<6nenPnEUYeTa^Z4i5*)IFcU4I5$2gL>G3}B$9jvGRy{pM8nNUh>iI<_ZbhV)T)8; zgfgfKy_r6sxyT?D@tc^LX#h0`3=HojQyySx^7F0`aHUfMkdN0+MOWWF#{0}@3it(E zxfN;TzMb#=Klig$OJI!yHbv`sBUf&~rj*31R|3~Nm9V6m3OdDmEeLbR5?1o#r3SNp zk{2?mJoW=Q@rxTdT~P4;lS9w{G2u|28eqTO_q3EM!XV>40I|`OloXIQvQqJ{O+d)@ zCfE7*ocqG*(NR5!B>Z_ZuKwBuzqI&xE>pSX;?wu2sHgyU}m(m4cPWr zSXd0S8c*KWhrYeaie3gcwk_9t{24uSiwz|p0M^U(XSI?#U!81H6FQ|KZE8n9Z_Eg} z;~~%EcM{qF%3uOvq-~&gU^+OU@b>n6VK51V4U5S?IigRCUR-LXrt`b!=bB+@;>^&z zYk2crX*DaaqQH`v-)(0RQTCTWZNDAOTRpP7O%vVa2P^x9|cc zxITls4>VHT6cga(iVgboSt&r9swJ zRD>GQ*tkKt^(!!%XHt>$g8E-VI0rHWug3;cP9^7?=XIjTcrb=orsA1>2~xGdVZ1=m za|g76JYLf#!_)0s3z*dII@NqX|G4OFhg|owD#(lM01UuA|H&DmM9}Qge#b~bXLk?N zRts;`O-1}th2f)(f%KJ-NCT{7v~S{@-?EFM+CXJ^{9B11JHj0*7#)@Dmpl8-*HMgL z!V~qNc=<$b9VA_fA~JjL59h|YT%evPyl~(OgTMBpKIyZXn^bze6<|FfO57c25R`;e ztXW8i-Am!LDrek&2?wqGZAH7!q?a*_DANRK=$l$&ZuvU(G5r(=FH1Z-DeGG&2Q&RG z8*cLk-A5rpGo%+-0tmC@r@V`+qY(tHy4Pn~M7^avZ$E$5$qm)YFv~eDu(WI}f8-J} zRNZ>M`wd zsgnFN)_eW*C#$oky}_(ZxA9Z#Tl2d~oQsUK2jBukF&ci}zc4^gNiNW(gZ|8Re(KZk zvsZ&{zg{Eanek!kcW;D5KAz~c6B)@9B}~3?iVd@L>1zuXc%#kCSTERYc6daD)DU_Z zXbIa|tR7RqMZLW38W~_&osfWOBiboCWnNa-E@Ke-p!yPa_}w~pUhAk#8il>~DW7M5 zd<0hH;##D^aAMYz6Ma9+w3fd9bDSe0)b(Gz!TaA)|N5$n_AR$=-3_QaAFLKBwS3+? zZe0-*_&3BmGdzjzkWvv5>5Xe^YxBU*iy>ndAw+7Zd}RvMTYux!dz~_dXff)2#>?-0 z?bq|mzeeg*k+DJO_5Lug28Gx}HXbW3n7E-)NZ1gjw{Ua-Yn2+wtvhX^dujjK4MhG1{;@HNxK;I(bs#A>F*&o(@&Q!7A|f4YA-GH-uNlqcT}{`hYMdS zu$HV1P3=U*dm`}VJcuPf^cD+jDCZ#+iI9|1tXfHM4onOJ7-8j55(v*vuP!A%IYffR;?u z@FYcV`c#r+&pYNk@1BbwT7e}iIfk-xUpNInt?wGLDJji4o_BP#_h7*p97Em&jE+#k zHp^A(qIYxolr~9T8by$TR!c z3H{GZBLCl+#36ip5_e*OPds)r_p1-PHjf)EV&_f*g9XQ#hh`JE4&ILYgZUY8&P=U6 zWtsTtfgleA7Er)J7P$5YC-H<&)e=>)GY`gj`q^E|m@`GoS>qKc1s|OSGqdTNqmm^m za}`B10s~y`V4qaUK*14aSdSFc`iVJr$&o4&i7kl|DHbgEya31%H8`LLb2o=m^_9qB3 zTE$9cmDwa-KUrR2=0kF-_J7c`9XAFf6MJmPM74X zlQ$2&f7@_}vh;ZO#G3nhIsm6XdAm-1Lo9W6-4eKL&PHIn8ZGq98izhK})tuES0U7UK6fJYCFkfRFbWJ_-BF9U_&Ic?p7=}2#&0Cg2gtDd_L%3_eJz0 za1K{ukVCdHx1#FiG^$8&Pp`*B!2%<37|D}+l3eQb9X{Vq!~wcPi0PA^+m#hKT*RlD ztY-AdiVvJu0W}~r+>nEVEc;QVnBMzdH}2(J$3Y{QXmxl&9W;bwe3>knPd@hn$a-Tr zP9Og_8F42lMk|DK5*z>b=V!Xh9a2e8nd}(HL}63z1+xGG*;{Z=oUYSd7KsDYT>1UP zeP#}CNbLFL@0UL%Pp$_3rkTf9)T}?Y>1KZ;Y;4;Gyebb*>d6P`k!fpK$?UGT-}G5X zN)L>_qHMEjI33tyK9UpW+BT+Wxp~`IwnS`t@oW5VR7*XUx||S+IS; z6`Vd00M0SiH;t2Sv$k(#(qAr09`4dfg!sfB^hNwu>b;zSORmoR$`8WvYz4tI3JeDG z5E);%@Fd|!PiRd|O)IXgJ`@muT>@({pgO3Kj2)9jcrJlXPcJC3GxC<%@aDr0D_^_+ zhq|{8%d%_YMQ<9U6zNoH1Zj{4k?u|fr6mPHKuQo$TDrR%X{5VBK|)ZvK}zBw_j*uY z-*12CI{Vt!+53;7uDhbtD-x_B1S$1cLkO!^2P%VWUYZ`bMe z-!+sO{~*yT|D+CmDJB=DE;suT+siN2^0mK}w*gs}jDkYZLYKQ8Fw}>fa%}h$cgLz3 z4JhOaO)Z#k)ZF#cMTJup;+~e~)i%_?ZG2_1uaC@S>c^;sG{N%YgosX zPaaARvaI=Jg4@R_7Qgc3v?7OY-5{1=;0_tedb$0;HzYL{xa|Xs+}r_#o30IWbAf~{ zDg%K<&y^zqz*NmR!|N?*_n-Ww=xT3vAZdh?IABSRKfrnxmWztu_m18)7%{`+bgE7& zWfW<~K>!}TVtUd1Mlq+4_0~0%3~AnjM&zsW+9%%uy(zKWy;SIxtx%?_tQTO@*gzvd zzz7^&^Wp~fDmc}B@>0z?U=i6he?`EVFuB@GGlCxG??>A;-}Bk#fk**fLl@3l- zzkZd1A9P7?RNi#=f1xc&chnMzAUuIOi=~enQtJf9dc^x9-@}a>UKIKEb#xba!o+4s zCWvOUt{_H&M9wzMlljs`+Akf6Zq}y=<)7E4S_iqW3u)BjH?Y1&`%w6=51vf0hA`Y( zGu&%jH0eROv>h$FO%Yut?OppgAmps%B5AdC5yCXx$?<}*~SUNl(BkGb_yE;%+l7Ac#J zyi}u^q=phW`-Y)N+51opvB&Cb6X^iWI+TI$^|wh_J~)G*hW&&*0}>{HU@hm+*%7?x zhkZi(frAgW7=X)D{&BdIjTZ}oiXnhjA(Kc6NV$yw^-=-qC@ASvCaYKD7?Huxt!Rfy7M#?CnwDVx1K(7ejoss@L<9sis@z?wMhLP) zwQJU$FRDCsL66LK*zF=deo%%R{mS54+bQY7dHaa58v-ZeIl^hIS~^^ zTm-|ru@GwWS~CI?1VKv>I7b@bmu*&x0UY77iY!QlWCQpXcqych0sjy$w0Wlqy4mmY9_E2+XtXiJ%`!5orRzmwqY`c}l8amEb^;w~t$@2Xno26i*EFNg*f^ z75)?;Y2AIY)ftmTYasly>^kWg9e2S=KS0iN7ZZX85dmfKYLQHFqF}n&UW^)_5X8ay z^wG%KAjCz#0g*ZsuscA0h;I{>+W@R&1py7v8j3^$jLcbzR(R}P)J-3X05!h@8?(Mn zMAU_tmzT$E$YR$3zUK8sA1u#5ASFFs(B(r(iLLddt=m+EML8I6M96qf8oQ+KMySi6 zcnNk5-MVx1eo-JRWnb1t;E@@nzwd#cQG(A7lNW#O`985~ zm0PS83FrwabM$6g62m43J72S?4h&OiJcvf1Eo29MU!6{VT8!0rcz6hEqRNGPL8w>1 z+w8>^WcpA_iuF<5XjhkimaB^k?Bs-!|HWJS7`uf|?C2JXv~Ta#3N%@gXK~*6`R(z9 zmD$b+hu6>qTmZXf3?rv--5Z z`z!Ff_vLAofj37eBR7ThuFjsS5j6Jxl1x!{d5cYYaxcT66_gb@$KL=nco!0~3=1eb zt3t76+u|@N#r=vk6(4$yf4ychG><-bNf}iDH{SPD&O5r=oIL<>f@gI2$8T!92V*CN zL+<&}sF|DR3i-@}w$9fadv!pcfgvqY(tG>l9bupM4-SBSRj%|-$Hc)|UXl}6eY3(K z8GE)^MmwDE;*EuOmhGX}2Qq{ePsRMK(OLv)6rEvstJinAZVo?=c)(Mhgw^YGcKrNt z9^Q+If$8npxPaT%lJuc*^c6Ap9_2ebeUiL}OHC~9ul}6#O@#j}n5XE$G5#W@lAshXb6wvA8P) zer@iaFY2UeCaIXXF=;(%Xh$%6ataNJD76u3%>EKy5J760AG_aAO6*DBA`S9MOiO&! zUBN2kG*qLc9ySnu0BMkVZHz;5S1qaDP&G;nZ<$w$x;Lb)3`J-4EfZl%BA>o(u&cF8-s(ay2N7)V3ExjewIw`MaRzD23$y)%0^corq?&WxG>uD1*Fxn!VV zi)K((wO6DXP|l&vi$N*%ZO}PiGE$;v@_XzygOM)IVs zWmN;rG+K`r=P{?(NE5KkP;sy4|Ix$gtZj5prP_2=6>8HN1C+ z?hhDVim~DLusqI2UWijQdLzS!v4CX5M1Q+bH+FKljFuswj>Gr*ySw{G`oyEzol^bI z2F?#G3h|mF7${f4cL+VE^K zGZWYMB~UCC z)_mR}cZBFf=vo)@OfzdzYu>cA8p$b7Q}_8^-=brz1A)~i`-{q)yVD)$^tfp?*i87% z+3h(y%nf$UTQ{2D@J~@(Q3%QrUr-U|U}@<$*CvjhT(H%?V}t# zK<|2J?*7%eBmWKBAjXqE7b^@N^faK@MpUw-`MdCK`NNcf@LnGSYi3Ih#!tq%HG0bn zg?8vV)9p7&1UT81h}wTR%&XvrWG)`pSj?lu;p2MB&dA8um~Yv3r!8g6Pqr2eI*_wA>O2&+2`# zI@a75#&YIg{FG)I86%~-rh3}>64_Z7qjlFf_Qr!wwBSSUpQSGh8E;LyO#+u+E}X4V zK)k>(hvA&a5w9}qt|qfk;s{5Kma z3x_>m%ZkTO+vR7qc(dF5mH4x5=gk-1Jf+IeO|Hi)CbDehE?CJW$P2FpWN_qg-tP3t zV8Dv^cn|bC|i@e~>mi|G&zNm60v3BZ zh^e07y%sDYp5w|^zlSUrHota1LNgQFl>S}Z0=3xI1MC5(2#PyQv=nFa2Q!`hGK#rr z-{9sF{Es=B8Y1zpA0cQM|9bPEp9K-o4gdW7fAjCuN1fIi^BfBW$k6?WFl(i~>ky>v zH(QeQ%q}{J z#=(UQS-hK^@^>{5p+u}sB#42!hBk-PfEH?wYo0@Z3MF>N6}cN_A+LxQ@FWxD>P&MV zk4%p~Vvwj|(J{LEsrp}R+hQjRh@_se3V%1o7eO-loQ z@u3~TC3X?n3lA?^_Sj3GDWqgna!ROK7ul1AcN_~9JqQi5VpFPCtmtoQ z%192%?>IW9nQ9Zt+BqkiBn8s8$_ous))XTsqmH9Oq)0^EfH#M2p;RHeM@Xqy#n=!R>b+-x6sa?i=&RW} zR8bOrGRPcDe-y~ESQCQRVY4H)7e@^bG)!ERivW#zdmuquuQx~`6FX}tmEBGS2`WT6 zM}@v3jdMf2vtJ=mquF#+NE`D68>&DnwuB-XV=NcOiu44Ly_^_J_Q|hw05KLcOVhh6 z*ac^j*?q_k53bl=BKROciO9%oP(G3{E<}%jfCbqRu@OVfGNKStD*ykdFN#kImpotS z1WCfwFE>CDRYq)cvuM>Z7%MQAtKwsV)J!sL(J5Yx@^^i!Tno2juJoJv=aqvW-%bHU zsv;U-j|C4#K{`bY-(ydvE%^inQ>g(FNcWSUz>~?|c~33HfO{9+3+efxi_NtwHZx@N zY5`ju6doG5VomeT70{~S5$NP_kBKS0m)C0I3R9w-)|x33!JztMAB+y-i+%{=1b8~@<6MXV`YVX@EeV$xb%ikZ;Dg*~P-tjQA^^s~bH1S{&7_+mL)J*Y ziO1mG0+mvo4ml)TY4Pd4l}fMxKEAjB+piUQV0wk%7#BRt4A_G4av*{C@^t1>RUY^( zBcDGrn1Z6&=+WWgM<2mklrI3j;S`khEtWyrUIQz&NmthdiVwoRAsb}Xxx7E^aYcZm zJ(tGDM)fMGP)fH+WZ*i(Kpi!snX|KV$>RoRD~r09+a9~~63hMaOYdVrchIHGXRr?x z)33H0oF^HkJW@I*)7on`DDSk6=>b( z>sx<)9QYNs5F%V+JJaY^%LQs-VW77GLmQZr3CD#{>Jh@X3C1u>WW9CSU;YNbI8p)O zGYU$3zN`Dh)a`q&N(O_)>Tnvck?cM2x3{;4Vgi$ZEboL<}DckdD><3VB8Kv z`R<5Q9ii!$=Ma*M#Fw_YF5D6H0R(_`)B$WXiduT+~@KkYgrYXhA zAMtuKlRtc;m{;nHjX12-lMP;O)=PJbsJN3F0dEULxC9HZ#C|Zf+63x_-;Bfrxv)k1 zt1}rm7F4r+VJysg>y}q5C{nrr_1)&r2NU_2HD(iR8`DP7D4o4nV+_-VBFHEFu*m?U zf&oKO00N#u2qFzxtn9=eg_i522R>*iFh%1Tb|D~YFDaAlhFM5L(97P!E|3ImQvy5xEI*YsumFL#Mdg=5LSxx&pCK7biH0!#u{?V*+m^2NFsmZXT(1QX0I?d4gI% zjw|?)Xw>-6`QYxlY?_IH<{LH77f@MLg}|}ER#HOWzD>mfTx--9&EisMA{=_%14*78 zAn1+QvprzcJ*InRSCNeRh6bHX#&?+iwYyDnEQUs8xgQ^bZG975F`_d)bZ$@(7Xqxk`JT06oo#bj7t7XkKH-MLWK;6`PzOT$)}(pYHS$5#wo8? zT8{hSQVkjfHksKC2_3<8BP*E-n05vtM%g(zYC4@DM+L{{s=`OTtbN?yacO2+O`Q8n zRd5R_dJmHP6~5$laasnCD<<(v$N>J{`jmkN)TOMI@X*#TdzcstXtKmNK)0P=nG8Rc_EE8jMNLEiEC_h8 zyA0)A7M~YEK@Anb?M6^#55)Q6WTOOsu&+m zEPe;)?#e&9k`ugP^YtM4pVhyD38n*BV@pd*Sy`ESPe&lpzzh_pXJ&myKpJJ-E^X+Xkxy zYFEv=9cXqSeH}Ng(m@E@w&8%V0QCE|2C9DLe|h~_rqJhrTLy3-^%s40^$IX_abbHr zAsvh@+6>$WiVIu|;5L|0aDfT_`+5ZfJ#PI`JABlVK{YP>bVW-*4~Y9=&6RP09!Bp0 zCk1g_-xw<@P|Lwlf|{QXPj>vh^5MsTob>3AbSRt%z5_EL>(39TI*PZ(zSzVx^>Cu&iFXn)7FjC}GH^3z z?-{bq?vxIMU_Ci)Ll$5suJT=L5(g1vQOBCfAwV}WkmTUS`RRlOd{3{Bz^Z}l!9IpL zY_|5vi?zMzk$}avqYY$56bD#wx}8Ptlr|@bXo&ys|IH==z@rbu#g!5-j*bnq=oJ&; zLk;(kQ!V~HJn{KRHMqOrU`O08M_ad~1y0Qw(?pN&or&j7zD*z}f=$Xb8|Ov%8brID znTfHwcMlIAmp$$HK9inPG-Sa?m;JhF(98;-1VuMEIPSt^@Vh|Hs${ zO{TxD@a4a9UC+ybi&;|Fd{@b!4JG+EHqr4q?GstaKztdQo0?`@{DeTB`I%ZVk&2+- z8(aNaxdF=Dswot3j_+s4D640PKC+x-6J?WqhM#N8!Njx^(O)DL&SQO21?;N&JE<6d zcD7l&_6leuH_%uD;AzOuJY2=?jl|MlaT-O|KJoIH@F)4GXk4sUeF!Ei&PO4>M1>zy zne(Qh7cqx%cSuAvB}KL+|9Vx1*-=a1F>CNO$u%mn?5S5Lnji+^lGQu_C?eViPOm)Y z6=+>;GE1*x|80Y!(2~1z_KTbRIcwE&hNJ3x$=4^l1rFamFWnzA+L?jHL6m`AIH*5r z7D9wXE)M$(X-FZqq3?IJk4~nZ&rl4YpE1l8_^iFA>ae|Uc&s$%6^f@Fd!ckD_#76Z zPKw7yGR;A=o>klP@sM)(t*lM>bn;8-pWlth_ z?vG-C$2A-6_P0Qvz=T3>5#w<(m(wvGYz*o7ssv(*#zvF3bDh<@m!=snzO6hg`I<1s zv3O{kog02h(d{HvJ_fn(Euyr)Pz%oAB8&Q&R^)@EYSJ1d+K0wQtiinCQUj>+cAHT3??nm zD#^|SW}vq0)Av?o0Ey2`FQynNfv1Yytow`5W;m#Ctp$z_9>>m8;L+APlHwt=D6Mkw z1-vQ7XC%fZc=Da6)a)5KQ z+2GPD5TZkawYS%}`?G50srr_ra`nIJIbf<4UG9$!442ciH|&q=Ij2gik#^K9)PCL@ z$@l=<-VvJ_?H5PR8hY&<9$qZ3{5-kHI`b%lLaN0efUz036;6~5i7gTJTbzM?=r)Pzc?zfx7`oc~H+cJx8bPo30r*u866hzfV|HS~!El}CszJ||tIa}enNF`RnX2O1G zC|Jjc#w#Is?9@GSch0}>_?odctrsnr9!T5;6~Hyym27NfZ>N}*R8v|WXQwQd#X?oa zs5O0~xnW8Y2=akj^c~~^&qDw~Iu)f|KlcP_b1yBvk?4m6uEW(}*ZgZAg!vRQ_AT)Y zUE~=`+^|n!hNE2=TMshgk=Jznh&_H-o~3NYk$l?lkS0QAvskm?oF7bV>eW&vg*gmv z;?z_yDBa1V?_eg!QjXQ=Zgocf>hVWCSZySu;!>WS8~7ebNy;h)B7VPJ$LcAY@_~Oh z@Z183UxXIs34_NLuEp!Vz_z{xMqxvpvyd4M`a<~~oJN*#_W+R_ZnIW%1tpEWTB*pe zs`;L893Du_-zcl5S8nugu`fO))<*e{_16&|uJyj||c3 z#q*bDftS_j_p&_l!oP?Rx*gB?*FT#b{G&Q2-A1ZG-v*IOH}lr@ephPhsgRZ8f-l5A z??o~$6?|SONUg<(j=dkV#=p+kQOy9GMIa$4+#up}cOL`U879rb?gCUB%ztEg0gSEV zbDwi9L6Hge`DrbuFUa}1`cbyTkq+@`yR&;&k4i}+$?51r@k~Kn0t@`yjdB{;Xl;R} zJ04;#2{@LDAiYyTfsZ-B?ncm&}0r9wC1)M`6n1wmH(G7rVn zf3diO`jI;D_(oPP4-AsO8!irHzvfLqVI-qf@%xlwa^|?a=YuQU{jV!6%zQNW6BsJ% zxF-3vgKRBEexhUJr^ptt1p&Htsl%)4kMKB*Di}cMSCRnS{{JHe!T-{HNVfhqQf(&k z#^Q@J*%bUfX*Mb=n%UXSyH2Qin=h_Tz@f_hmG6≦sm=1VH=<>zC>yYzC^2faa#W zT<6pT-7n>0 zd3N_dzI<3gGF!AF($gy^;%X!n_ny+SKG+(r;AYDrCv8XbHE#2NO?^aRGCBG4lB49C zFdGsU`ENrBrPTUc#H~1WlP;UhOrN&5L&F&FWpvoX$GAhzcGt%(_}(#2VxUOCKC%|w zev%lGB0(~duB@S=l9m}qK+VERnS6G1(2mKh|Ksn@U>-iBwq=hU>tV+`8WTQkuWX@T zYK6t?kqA;fY+8y2(P)Gk^V&WrFQaTIQ|q2(h*);Ca3rZ^oMJ6x;0#B$dn|TuBny9# zrQ5VsqqmuBYc018pE2%f=2FKO(|`6ByUp8s*0H_VBbXzaz(c^n+0pOq;SSB)Ux~PYFAPC39OFry$NTTMmmiieTHYX+FiAoGYzf9t9Kv^buAbu6^)!2?^zfS&z(#u`I~F|1PoI1@&`>=qW=P8U zy`Kk!Q@x^}TOw@clFWh%X~q7!FD?Pa#VL(nh)q6C-Q&t%$jrr&9eZxC{_$OJR&G_7 z)M|24p3*=q3dttog0(&&h^pm*-n9k{2v;l}yT*${G0#yc4&{T!M9g0qYVM#^*}P zyU`|N49X%0fs3H<@yQo9e==L65oW!r&Ap{Q8ZX#~E5`%APhzgOcQy9u8qhYrzq;Gr z@h;EQYFYmAu&LFL1{B<&ZOfaOQ2?} zHRpkcwhPI>R_WA@wNpkWF!XrHmWTgC<$F7mPUIcmgk|I%DlL0bw0Qhxb@4v-_c7EC zp0$TbrB43DTMr~{dItjwGE=Z4%BXgG4KxUn&v;KO!tvR7$YfOnUr~>Teu!~ql)Oe) z7|8NrtLgfFf*aYJ#DLSjpz5+4!otG5Hq*Bb`eLF0rGVeEd>XtpRjaMR1#2&12Q%G{ z&tI0kHO^7mw>q5lCYjx4EjbnVG1MM3`lx9I%lU5A6GP*OTMcd#qH~>Mt^rgvTu$Tf zHWv#=`<{Dl*4At0qAklV3|QU#NBKD_J-8%}pcFQG^em%!%BFv!yX?zB;3khyor(V54!XJG~K4G`NJQ4=z2hb1L#t*l5b&`6q_Gf4FEy;!|| z4*U`Ulpp{IjdSNw~5HZYqnH49`v&ceg;7_;{86m!&GwjD3~(diST^jcu8^QvQDLTd?;yG0Sj6 zN_=Mji2*kOueraFspC85z;{H2Cx=ev9nyJkr{A17?3HUXK0(#)w&-*wOYlNwThmrs zU~wFTM#eaNnGKm|NoZzE+EW&p^!*qSJOO|!IfDzt-xj!|JIBXkY{e|N<4kr{><5`g^Vmbg}8+J)70nH zfm`5MStK`fk?&5Lx1B%FI8#v{QoHGh$(y})6Z2Bx{@p_SWtx9;%Ha^Sp?aZ%A3b(U z^o5o&_BnNzN3>xWdbLebP2z}EMKMs?8gGBhu5C`A-3gHIVIV($-nu}22NYH3$a>r+ zFJ*32G0apZCWF0)G#?l8NIAUHE@Pd5r*(HLyYSKqvlupIr_#{Ew>^T+Do@GnimArefmT!A418?zXkdrQ2XNwQz<$)!rH!M-yKyfmRFtYQwXc^M>)i_5@3NY{# z`q_!?6JI%Gf2ejcMpN`173^IX5*QJo9?m$XNGS4G5VdG z%}tj{ZN&x>9(*y@%UQ83-UmV}9W-XyTQxPR+5-J^|CSnXcV{N=K0C_`buu13Y7#o{ z6CB%QjS!xQ-n)3oDA%Omvi&gEcm!wsO^)WrVIT zaU!TXlu>^Z&hR~~g4?9?D@}kIYmd3GnOd8Y-UM`Iq>;XHCh*mLI|AuM`ef3BV7?G?LA8p3zr+Aw( zd~RXE;cKs!(d6_iOv*+3O{#>DO}GMv;fIrp_cJNX_lex)|7TKIOtg!S-8u;Jl|b?NKE{nv=bllmvD?0# z^;Yof13vl^(nkA5PDxMpX@<3wY15AFb`B4PpGrhOg9(T7PF?(6E&0b_H58nn$lp>A zt>nV!T?H+h6p5|T)n#eT=3<34J|by%ZjHPV5&YUKrnGbhjCm#FGQVz6O7;wql1Txi z$?`0Yy-}2J?u12kf9nbfyc@we;yM^;al>jYp*S9O4V5K0qk;HXlib%=U2V**!lPMl zH;=DJZ^-k`3}iMDnk~|9SgWJ=E^_9!Wp&rTWB_i1j&j;GmENJ|J@~dAz7Y~+#W^^; z6AKA@a+aj-L{DEFCt%Jx@CJn!+ab6ey50-qBL!3Eix3XRU(%)Nt=uerC2^Wbku_#8 z4qL+(iXl&Cw8raKgc}cBb@R7L;E~R**2OY*eI>G%gX$ZjaSW{y!LwgTTC{ADt@CNm zQP(a(C!t-G4fY?-7DUe7Z$a7!6%B3Q_d{vzQS`^q@=sDtmz*W9p9m_;mQ;~vYzxvp z`-Z!AdeQ&YlOym#u4?6|(14Yc(qGt#-M@8@rnmNJs8Hk82){fB=uMPc{zP<hZ6N>$bD}$);JU%dt`lqWZ(a#s&?KD{ERJ-8D&x<(9&EEoTzgypBctGN+)b-xk(^DWZS6wEf02hFtG4D43VQPzxU-VxTSYadl2rV6q(sau^fhfh0ja+fph<7^%rp+gdbcV}&o5B2+A(=e z2FSee3Sq;EDRVGk(E8}9du&)DMH6h3!-9;NJvBmjF{aH*Rm3y)yG{qJ0O~aI8pUOW|b;VBKyqYr>fY37y1Ix zQ<<@6L33)7)FN*^EfleQU-bu*sF2fP_dh0@G@hG3+2DWQOWPq}KTh*iLGn{ky4X|~ zad?s`h8w>6+(l`PE!8le_WTG%VW>tba7n&g@PN2sP<<+Y;YW zyyUqy`aW`to0X0;XLa%^!u0*HS8mb9*PCngAxuD~PU9X~8AsQluakz88wcZ##UFcj zZfSR6e_Nr7XR?0hLY>2yq83HCbT77pbB4);O(Tp&^*R67R&0|V`6!btWa&3#CF;Z& z=mh*j{~>X_?!Ff)9~wMM=s$5=x<74r&(A}^%)4#)t=<)m;Bnv8 zW#kiCnHDKXMiNo7smaABy~a~shJg~-6R3>Zqo7(D=VFna{}k}t+u`A1Fqm1*05SzJ z9Y~N+?j(35M=4=j0;PA;dEiAjXk(E6!7Yt>GX+6$KygQpaOy?mv0~WUWACRg-4pt5 z)NUeOpDUA;dGDeS=!xgZDe3m8`<|c$Ou>=#%HIy6moX(e*>19Lt`wT(x8S43Mpx(W+=ah__tfSFlMJ zni>RX$dyzEAxiQu|Yfc&dJp z(cB0@9&3iLOiX)xfaqm;#~K>p5f<5CYcYPaJ&Y-o7g+4RF&B;R^GJgfa!2ZQc?<|R zzjx9Zj@$rHdXU_Q>#k7WlVE@nljiC>@%mrXoAS#Pc-UG2 zYXlYo`nepY^pP=#HI$;)D zt+Z-sg=%>GB~4+9^6+*I{XD@O%VbZwe_M8ovB1iCbMa{c z7>W!$LQluH`4%w~f&y%A;C;DMQ{nrNn~XwZ#(ETLgq`pMw~hR4glllmvMn%RxJrt-KpuRtfHinNvJhMXj>$A8 zFz+Y^r}_#hLE|?X)FJ`&Do=qU0#P_2@ zsAWlPQ?wc0to+>i{L$QZ%1+mn@YRk+4w-vA$MFe8T({-h@XHP|5MlTQj1(KM z|0gpCkS21g@yu8Ffb?B)y|;Hg;)y}kfXLbF570f$TGW_Bla)0zqH^c-u1?`_h_c=> zw-kYm40v-_yoo0bW6PsUJ%-H#1c*I>HmRy|VGhA9>mG+lzY2zuGOo-=uywhNGFu|O z30q_|v=aT=aMlTIZZzB|KhpU4C{}=RfjjKo_d1f%Lz@oT-@GL>74aJdrMVKZEaF&W zPW5wAs=`JGEaE>Eycr}}*UVrJPlCVWq@A-cGX)LFGYj9eRXDe}5qyooqFqtg5PgDE zgOuR@LRC%9zg^wz+vvDDKEopX!M)fP?NGvu=NB*D@e2_@0ZL7_fBiS5i_kFV4jyN{@%zqg`dC1N=e*kgbk!?v_Qy zZMB;6Rf;H|2lbPW$CbK--!t%r*Ie|?zf9`Jg3&xVKHW9gR05qE^e&A~!d@=$kV_lv zIPML$5FSSaLttbG$TKpX-X~d@()OR z2CUidnKCyUt=M^vzI&s+pPJjybJ)9P;3<&(1iE~9hZB^W2bLR2pWB>sJIwdCgCMaa zJt3Cc-D2Qg3`lWg7t5rVzI^KqQIvMb|HsB=RwAn572j39jN#;)ZGa#Wn9ZUpG4Co!j*7)?@Le z*7MuRo2^>-30h789}09=DC%18VUWL+p{Yh6q_782 zUsiiWDEKygtG*`SY;~=Djm(1u-*x^Rus(D9yo1J#Bw`L(rJ_Me=@FA3O_5 zQcc**wY+@W^7@I?zGS+h^Yb!V$-2StHCeL+4OLB!0@*SWL#MZyB!>LvqwUP$t$Rdb zeG1C;gwY(f^})eA`};;SbB3x4UJW{Cz**=x2{F1%aAXL-_;@kEe}&#CYnn-I40Np` zObkWCuMB8KTYTMUbyWD<>1Ic|P5Fe!I#4VG8Z$KXuy)nD|6_+dOA znL(heGOB@XLv4i|AguVB)XpOb%!4u#yrUPguv!=A2QCo}ayPa%s0ELa3~H_z3Zf}O*Kv9=;wzARD$ zZm@5fnF8#b)>%sqQUit0q4ytHh@c&tC^{u_Haq+hjz##Lj2Xo$)en(%4IP*#1@6?R zF6m5}zSw89GM?z=a}R_jR&ak;*CngBP}sPHvA5*$_W|NNwGtWv}9XTIc)QVZ5s zF92ag%KOB(kpG02`~N^2j8TcRY5!MWGD0O@9)oKajg_^eFNPHj4Q&L$Y^IH4dN|-IR~Qv|zUB8L zLBgEIAn)lZ3v-ZGF3Bw~7PLR_1&4%#4E_OyN|5;=P*F{pxhv=#{V23yfe@_t{v2T& z7MtBI@DD(4Qev#IldO~8it|&DeXwD0+-@M4pKn)}80ok5x&5yvR}Daz+t_{<{mz_v zxW9Jx5f`<-zBl$%$TiX|x>O&EIJcd_?c=$`bYNXzvHpi)%0U4AC!V|!&~mQ7PG;Jj zt<=%y?+d(V&JLjA3m@9Vz4>B z;FT*tD@4<@Z|}yQ6diAt-uS~Nwa%!=T?GaPa)MM5;bj-1PE8#uNHLP02M#za7YTOD z{&x*9;+`y-)W;RUDF%ghd_zOxwIMdnr!`H0=>B^F`fNyN`%MYdYQ^7!22RY)mPgbw z)?pH^{tqija$Kc%Cqmc|j3J zk>s?$8cgwEiOr16)^r2TpUp7R=Y0+@-LILb`k7K?d(H?v~=H04^@C{a2Xq=M?=!Sub^0Q8u#`MjR# znGtwLt=Qw`52GhpY-Ys&y!2DE;VrJwRCR^8UAI74D%+wP9?0xF6p(3}F<^<{=jv?nps=w^@ z+o{vv?t-@QIF{giNokD?huCVMv7-AJk>grvtfNTg%LK}A9PP%~2j`At9)k_!NB*VH zV%vgw%MajMX|~MVicOTmc}dpiZRNn7g|95%oFm)wZy8XS>^BKgOLNC^2Bl#CxvowH z!F9D=SHXXedjUW*SXf?-5WstHN!>H15>|@mQ_rPT4;cO0tGB+|JZ7TnT7 zu@ruyke)spY#{j zW&lR(oUqH|)qSJOo_|$bWu$loI1ITBIj+@B8)MYm#@vOU8Ll zmvg$O>syn$(tVx#2IR5FH191{dlvlXZ3@x@o-cnaEcP#cx9zMzC(QAbDer3mhuVXtX z)IuF$HIGiLd!z6WrU8-;z*zdO3lTMYP7^j4b*?hzfk@-G8}E(h%3ux`F%M-x)@8E# zy{>}L*y^|4gRvKd)pbTmp;PP`b`9J~u2$>So7FCR3aVT(s@DR;k0a+DFh8Im6l%lc zPR{~I=;*RQjP6(5=_k;Rt;;=kUxb5%Gv&Xz;tn4Zy1HrSXRV8geZY_?B=UKjJev!B z9kK6Q2t6oe?lBT9WHlEXWY9{;9NxSrRf& zWI5}MKc|-w5?qswJ9-2&Q1pmSN98I1oSy!G@iE6j)HOceBFF#Qas1^HiC7Y;+ze&l z*!IQ#p4qEVOg`)AmYc$-8WUI&oU|0d+$+17@A61o?mFr7MyY8>cB$z!7IoZg%?^mH zy~Ze-z%s(>*-xmLY84Z=Yh?M-MF1X zw_`%$LDfD^PQJSvqTcAXF}5jg%R4>Mk!q{uw7RD;vl=wvo!&}SLS^$4HiAmqpSj(O z^QdV(CNe2JJmPS%j!2qDjV6k;bjcA$$N`AJ*q`i{9Th%$21d*k-q2(rS_)=1D2t+%Qla zNiTy;>0(;&ozH#P0%!*MpXTCyZdS+W(>FTC(@iQx$PN;c?x(yR?H|ldFBfG@FEiYl ze#Zq>Gp&MQVCDk|#WJZbGrdXdOxPm*7loiIPt@4W70;%D9#-e`q*y=eHZANZA z3N%c5SsL=Tx7LFJm%T)Ww-S-*t3ldZ&u zfo}#7iA}BkYNaBSQY14k+E>r^h`fy*Q+19%8pI~S{a0Yw{Q_u&fwwyu=te}v2yYi; zx_Z>{EZ7$46k8xU-fdFStEDd*YMlW6-LPEf?*X`9t8KVhk&dYFc0z-c6_JPW&J zuvIbs{VPi$;}ioxOqqEFe2OWM|58vr{(in6nPyR0jjG5n$J zf+)z+R8zgS5xSTEH^WJ6$F@5_69)`-Te&6_?Cicek8+RVZl zo&EM~1@yaxBrc>) zyVKQi7onytEn9^y*8&0A^_W(0L9D~9XH1#~2=>}G+5a)d;M|_a*&H6M>(Oo6hMCgW z@gEQ4N7|%Gcy9P~r8IfOGyYb48~MLXd%&sXW?oNht};8n{3F8V9~?+n4q1OM5)iE@ zjzjRPO@f1&ez_64(8mQVwkzPpUnT>tWi~wA@!P^7Yz8PHlK-GQAZ~2>zn>!D){EoJ z@yiPU*bR1K(1+>IGI(ZwqBIl#y34>4Ksls8+{Xd9PfWwGGc?sfBTIrUe8B}?qC1keN}87eL6s& zE}o%R`ht>%)ukoh(z|WQNyLK2Cb8>jrAY8vg&It-ub%yQ`}Sv;dK;tHt3>37Wm?6@ z-u$h*01=@9ctU0x1#A<#0=KH8ft;Y?6|nFg_-K{}pj(tFK!V~v98Ko+5tJo(MZm#V zj(GcT0hs)Tu>A<}J97@tc_D?`6ZyQUx+uK0k%k-+xz6b#M-ESR)x0wCo|MQYtF02wso;@KOV-7GcEQM znz~_NKS?}Jz9yF|CJdt4X>taVFJc_R+d|6!#9OXr^XeyGd4cr^4`5J_f2H?4!WaH! zY1qY}K)iTJ-^fF^swwStSHtZvbz^cmv+E#E?oDu+Y~gTQ$-{021%)n?r8~XT65+10 zxrl9~;`4`i#*z1w5?kLM?5B*ybdOZb&TbG{b$srOiaG( z1>xb!z&-_GdshvN=dVx!!2l31poPa>Jb7I7_lcamxwgY?jJwx2c*rXDaX!Q`qVStPn1z~`?8NoaU|>fh;>c`a2S89Y{`2@O9!=HTRf z5oqT>ef4W4XvU!>oey?5fNGDLA*4)Li9^0o&1{1e?FDSaQWtP)Yigvu&Uy>lO#1`v zO$9&1w%+NkeJ#wrq@3D8wb(E)AS@|a-$+jSI|a1NoM@9|S%lzo3>i|^rW ziSVH=LE)*#+9Q%RkqFI}`%Zb)k>}!3-z^3n6%oQVh;)3(uO{rtkt!7AO5GaVy8N!~ zd-&+M+>7a^fc>Vf*o%q3UVqiEiMaLV8K20yp$_$k*jypI8mHg~>i2xdo7^K64p6X7 zj(VdkzM&3a{$YCQ$fGUqqh?1!tOQf;cR(cIwtWax{E};+m0apFpzjsq^xt>EW@Jk7~4jMQ&NuYZl-$=@Ej(o zKKn)e#e8!o<=oijJ>h8ph`PtgFCkj=@80VOlp6~)Ga(gi{oFBL5FDu`_UJB`L27fy z9Yez6*djL;ge+#o=kyNS$UyDDwNMV~DZ4z9pEPR`HmheafnhrXe1y9U_rGM_0YQaS z=p)l-zDlTggruUN087N~JPHCMvJwaC;>UA+KyX)E1tMyp`fDKUQ=sX!%?$1#gohw9 z4+{LRw`CbzGhFTkWv7D!0aILs;`EP*m;ycaHbHLI!U8c>26MHn=Z>1SA-r)BL`E6{ zM8z%|8fRM9WQKXI4ihPbnSI^}wJ{HTRQCYR{<`7wnB_P7?0l>|cBvj}Uoq^6kB@j0 z&PTij4UGPnu_SmcE&^{ct_K0-^mrYbU?ucs-1}7aZ7KT*5C=D?t!$gOnu84GsV~l4 zIyHe1nj>6$!OrWjf}p6epB(2(eKRBnDbbYj$IW9cr z;x9{?{$&oFDN18kV0oyeqBRowskn8B>8@H%)cS(6-s=zRizD&%Sou|(IyGJgKc=RL z2ni2Sh@%3(Cibodep(Khq6!tO-T1n!n=_G3*CUK?pH|gw%zlwrnc=3#<-?))^%4uq zRolT8Q0p}lVa4RK_4kOZuh+*oWrTdihpBkg>Z|=^oBo)xSZ93bnbF`;+jGPo=bzR2 zxD!m^Tkl)UkNm>NQJiV-^6kPoIsd1^uo8k}&=xByuwYoSXI>8nt|7jnT}rh z119d<2^RF?wAF=TJu3JMNZJIwln^tfz?`ov_R)qy|N)py@q$IJSL z{p}m}cYC=B3s@XoQ20k8upb@5uZtr;NU}c|F*u68&z>E)>>Gf_NkFW)Bc^<&77T2f zuo67qZS`d;*m^mFnnKAom=J;kmK`Yc+g(MtPs**nt9?YO2D>3|48z1$(QaDV@$z5&G4fVSO~1AZ9U%JeZXi+ zLu}dVBEVU9$*c8ci8Y&;B)c$5;7Xj@NgQh_kD>dtM<&XXe?!c!uS24SXmtu9>4Ykp zM7cTOrqUfHPcFXb6&Bz)JSfCN2wT)La*b>m1{G(g&XVl=|L`8A*{R9Y6ymC~wxy^s ziJdG|Z1rplt&+OoRu&+p>PC*|UKsF(+N3~cN0eJ4>7jJ$Wv#>y+Gavxhb)v2`_t8q z10Z<@{?MJ?S9Y+7%4~g zqbJcKHBrKISCJNs-L3jE2oyf(VQkHP3D!7EMphs1OYVc=#Hute+#xxAe}}JR((;R2 zr{wp`<)r-X_}IVrXm}Iojzgx-AEF+9iNe|FyhZzhPpbr>xNOqZPm){4rqkFLk6P1O z%CMAhB|TZLi*a&uEU&eiei!`Q?KOC3oExF1D{!>-(ls8!LB%^)N7qwOFYAzzvN7{~ zrB+qpY79GSs!@DfrC-(ukVB&Mo!3}qKMxYox0mqSFq=^J;gR?KI+y4@ygT${SSRkU z*n&`t7bTb_36W9GF0vHdZQ}&3{HN{_NGPYOt&KNP7SZ9LFFZ>%*A{@Fne#rv0AU50 zHy|ZZ-HxMhKisR*+ z-9%xJNG+VgGya%1;ZDs+&vuWCipL8kfxnENI0BMyAG}T-QpCz7!w7U5X6}n&o>i0T z3flm;<%3FjL4hR%lO--FN1eO9?=pR}ITX)d*(k9Xvfz;ZMxt zb#)olBCeA9jAtb&&(Xx*pMQaWb;I@Ba?;!qtI3uex7oY-#57D$G)hVOaE9aCmPPZ#|TR403AQZD2 z&B(juX-OQA65*aZ6UgOtT}?eU=Ip3AQ*i@a zkLP|-Zi!M$srAXhs#@}X@J+6oH+$8yHqr61!MIz@+-)g25!oHN89oK_CCj$X0fcwZ z3B8nTRIsAIY+9#LD<)1_WzXQwo3%P0fw0mEiRqv^ALxifRP^Ub24kxDR>GZcc2&~r z-B(S(VzL-xbl>!USQbi!8_jmdJ#zsWV?=@upKGH~=LBla*_)-YG0a{UqQ1x<+!CqA zd8>%CWsDB`)&BKP6XH$+C45-oL@2EE zgCUh@xv>9OwdNH-UJ{Z3O(u#CQi}hS1${FC)J_SPAjYXfbqDB@IOC`z7`;= zKCX_yS?+>Er(pmA5DA*5#Pz520e9^o$e_nw)|@0FK^Fw5(+W}KSTx6Al0WP7_p=mC zivjV_ydXC{LDKv1=~-qlVZMnOpijHGwswE=89AFZbBFSqBAA_b0xJJyTrO!Cc(sCB zj;XNq&Ee|Y_VZs+#2kI6E6jjZ3!^&!;-#Stii&Z zowJr09Z=js9~pSH^6*ttXDtP^zTSn>+cg|O-@VSIoml$L$)%;j#Fbb7>gH&vgK)$4 zu11bdED}=+{dk?A6XFvMj}nX_tk2#xdR;2`i1)3^Z%c$)hZFFbur@ z#O2|{BpMA2XE(Y4HG>jU!+;9o@h_}3*ry0R?Livvf+9K#w3!*A?oYQq{gu*yB% z-81t?<>au#asNH!li7#wOdv2)t8V!nY%32x8>ez3jD{7f1LEGdb ziHm#&XFqq~t(dV+3kv*9;Kj7fy3y!=flD=Ip@2v*$JC7$-hcTTt5#P3^5FqxuBNbQ z`PLZW!-w~CS@BLUBp$yeF%=A+B@af$x9Br2CcYb`h-r%?=GBb}NcOr5Z+2opFpjnAp1bDd8sUWBX136R@#SC7qy4+1SoY}6a!rRo>{vp zx?*B3Xz6X9y7~koM#dhfuPg-87)Ye%&+j8=lYH$hgd z=&MDc>k(W4cyF&0LeG*`@fKKwz}q8pcRb%x!uZ9C>bZD(W=c$SY^-v`ZNJfuO&aA6 ze%mi`^-E`1p3MlZPHFQE=6b(NS`&9X1L7Jjgl{oV~4POYE}3~g;F)9rRgWw@PE8)42M zTRTu4#c2E<&ukcBC4=MvcT z5PvRih+yF9TvDCCn^Pg4q`c*AH~e!_Ab3M(Wr2B(B_SPs$Dq$;i-->RVYo9U&Y%xc zPD%gishdOgNod$lLW3DrLjUsQXUg3zr2OF_VGZUO|J&TrUM)eIl}4kq#pW((>E()DGa2c%+OP91Y;njvCZpMj@FIvSaTpy2tpt2eOs7kkk^oHgz`V zv+p28_#Urq6TE5^df3)K7tI!eIJ#kHEW;~$T(g5>K_V87Ky57(wC1Sd#j!dsoVK-Q z7&IqJax9y^4;Lr%6f)q?5IRTJmpTuy{er@2L`;9K)ZGj^%^KKuO!vNG!1 z#*P8eH?XLmtdeOE77uBWzs31yZ$jfG7z4e@MX-)K=f>oYYR`Hd>VI^y`Bo|O>zkc{=Z9{%827g5rN)+!H-jym zqtsN#5=8S2j|OHtbtB^9Zt)?!);J6e2}mWjjUKcNa;Rmui;ssS7F%pAbLG*Q7q6PU zL&pP5KIoNn@WWx=x@lP|X45t}?K6qKS z;e=Dr~H)Q@S=;p#Ww5m!OYkk?H`>*{@7)QaQc3V(U}L?Kz8ikQrTa{gz{Y z=a!+DdW%W6zF0xvz?CT|2+GqA3QZ%;y+`xkP{`lHof$!Op-3$*HKW$b>}iqCxvS}r zVr;p#d~nIaXaZ>CDfDgE7%s(0MyJDc{iQYg=b|sbK6!h2*||p3k^Q#6udHh(KQE0@ zRxT7%C42rnWZcMT{CqgQs8h$_>h!$d&88^vLznXgicOIBT2c4)^`$QLW!?-{0)4h~ zS1k(+SM?{+nt3x7drO}rbt~Ks{bX$Krrk|%639|a-BR^5#tvB|I_h#AN^~5**)$q% zB955&m|&MEm{V}!XoMDY0@2$fo-Vt$+>svEZOeM$5_i*Y)o0)qD7H$;fTu zDB8Svc4+LK!@?;rx$BQ16bO-~`Jz#p!d#t{=8#dE^*v%dVzeSMtz^UB4`MrCOAQND z+RMJ9oM5U+*|sN*iRXgJ(r87=DgfW%f;Z!v9tk3$?@gD@e@@9+MDyCbQ)julhRYW9 zjSXSloNlVpm5o^9UkaqE$D>IKB=Hv`x1&V8SV-mMtgm-eM!Utg2J<<%nRnyGW&A`< zul6++ShEUq@wAjS#4;%HdObZpB^iAGcQ9me&aTc=4vt+sNM-lge+R6M=Wz=Hzolv0 z7-T`=u+`RhV1CM9R+%`3?7Efpm#j_~4Rpr*s`Cou6Ju4P;g_?_+BK$8m+$2^P%(O} z)Rm2SIkbOE@nXi-3Xy`^^B%SLHKyqb^Lil~deoIDUDc~opXyaZCX%O}9|R^-2thmG zP7zx?xUB^a1tnf?S?iG|*O*tIlGQ$sj{Saz9e~e0UP_sI27YEtetSlxIWOYv{4pJf z@%pbqnuR}1V#(5abMx*=WuiPCW0pzx>_@rQSz@=%XKaWciMLVUkn=@|lwK%=hZ?o= zSx|O+2OlSlkeKeyfie z?GM)Xs*ULWHQ5jTWCsb_J;h_tOoTe}REqOhD=JpX>naBahq(Ab^`49OE(q8~mNAbD zIhxkiO8fiY!fkG7>Qm}`DHv>a1`o^+Sw%NfIMb^cm?cc>_^Lc(R`DRZZ;2c=oj~`#P+Z=xK3$E4pN`0fApK#Xq9sy)qZU5`rqwP>)8j1TX zBBjAL-%K-?s`r*i&R+*5WKz2A>#Vrj5;X;=1Hgoa2|7*R^ZT6^{j&!-3!z};R01_= zrRtIH?r@&?TTp`V?Jh0w;Ozm(^%Hv0?ah-z^HS32;zPsn>JFrQYQEhF+YBkscUNSJ zfJJ9(S65bO&BLkL?qDzZ+qZ=R9L89vw|66q4-dh*6)(bueiFINc&4E`LD{Q!T+&kg zD4ts4z`OpoaTo8H4z#B|QY{G42+8BS<=n=p$HL6RWe(k66mJg6_;~p$dGqdR?Vb&$S9$HWa{{D#+g1Xo z>(iNooa`j|`t@U)(!L0ZL{?Xxo!8Uv2VchoW2tLAHN_qB*2#!{f7(SSSNeJUZ32h( z2dQY>z&~fYZ4xHc2`Zir9a*It_{|VQx?fSzXyijYg(-?=(?EKzYg4@Elpy3e3sI0C zGKuE8t4B`@Zg)S$+NVLBU^Wr%iu5Qd9)>~Htj~M(HIGI=y!aF!)WWPSO~^}b9dRyI z?4Hi!++)bXpEUCr&h>kjG55q>C?~bK10^;75c~zSjArK#C#*rQfGd4M?&+E@tkQ#& zlX;Ez1LVn@LJ}o1kV)UIPq`Ys2U^SVdk>dNs>=OJFqkq=Y5l_n=+9uzif8^sR1`!DeyhC9C_#MX;i{BB-BxuH< z_?pT2)9a^-B!?4s)=Pf6q2+|`+BMCwrZ;8s6KgTSjjntyx-Jbb8lrbExg@1}2zk}+@TG|Qn$-U^J`#dsKA%=;Ajv>|m zzi9kCuit2VGXC_i;AKh;!bd?+@5F-~2dV>{JX4){SRhFsjP5(d5=}q{no-OpVhykp ztd}ksfF$H-#UpThy6&`~1#*_ImpK2&c+kZc>z;{PS|hTY7eZ$J8(+#|pJ}ze4DAEA z#iR(0gydJYe66_RvyE37-;onW-3<&6@8KSgB~M~t0`(Q*ldYBE5&mjI;#jYVN)2k> zUI)ASHD2P-B5z6_VnsWv+6~886g@k6=eK4gmO(6j=WuV?A6r~*rN~J-JyLb%PS}URBrFf9|e8>kr-ge9j(#lz$hToT9aRf~eyic1PH4HWH$43WSnmj?*FU`j?fXef( zve@DF6zGVqKt0*Q!W+k@xUc~^i7r|RT0Q_RC`2SaSzi$%k@kCEk+6Q`S4LXb=^LAq zsEkDBKo+&x^RJirc!UHwe{DTw3_SfHDCOC89hC$JBxiZvyT96&BR-6vF`ww6LIo@* z`K6#us;KjobadT1ARwT;yu763Dj^#yYi33U^lGugrLfZkLHmYPj)3&swTO?e8X6iv z6J27x=O(mc9SjV{p!VyA_ODmO$RUI52wg!q9w-n)zTT026q@aw6#i{P_uSs%WXJ(q z@#cYco74I35Cx74={^yEC4Tm5bup99TV3{*=UzT!UcS}GYK*OJbEdY*QnB)l2QP+? z4NwP1Kzs(8*C1Nz1z;SI#|A27Z1&J?kyIoaUXt#J&;*OP6s7p6sJ`5LoK$Ln>u70_ zdv4YH8utaJRLV(d=ZJV^6=XDqX!9#+zg*NxK4h)h?G5XH`>}HP^TqkFki2E@5e5%# zk7Y#SCuLJrDF(xVZllSEI|@Rq^vWc{^zRtgo%YM(og~UC#@%bGxW_~ILs`s>?@Bv) z%`Tsn?70t2q59YL(I5TU>bGVnr;7$`3VC0|+uUvRqX_Hn$)06t!}d$Oyr;W8ww`SI z=2T;`*jj9nk65-!o9plM?K}~KFW!=0eEa~Q9)2e4ReS8{1BR>lyfg5!sVOMX$b!OV zhN(o`*XO#?NVb@}73cy;8yo}hheDycK{I@9x)qQ~uV23g`68H@Pzx-n*gxT~bx+zg z?EUd(dWo+e@C{XNT#0+%fY_aU?1go*~Qw{d%s zWXMxQv0DE5CHdRNcgYZ{9$v=pFZfUr70Fyy!XNF=D0x$q7lZNa!Bnh4m?(#BYUK?! zINWW}fySBITKydD;#tVrEh9jqt7q?p>){Sk?YY&rJlbwk@);L2QZFr}4VU*JfCyCc z*MFvmcqu^E28iqngQtu~-hRLVVcO>WxfSXPB z@rlIbN6ixD;k(vA^hAsdI)80do*B6$tfmt~k##j);Sx8U&D~dNF*_xP@s)mhGN*0L zGiA!2XSlS6XkMPCDE%gXa;-gU3S)ZU46Vgdz}NKm_k(u&=;$bHN)RXivK`Gf^7m5L zK{$gl^ZN3*_vXS7EiG;T>8hRCr1a;B!VU&bPEI;HI;h!B0A#5@`{%$lJH#7w-o}|4 zIqg_T5&vpr$KBw2WaPW%ho?t9N|t6s__QlzN?0A^ypr};(w;^rlW!Lh*O30OAqm@d zk-cnK{p43Cb32{=ybdyY^NFO*_}U97mzsT4()#e(255Tn{bIz+OK zK${fktRqlzj!0e z*hjE2KsNsk90y?fA?C6D08m;`G+bQt1TX|hiTA(PPqO&%_r>%9D`7VsPORPS)v2Ju zzIVC<@AXCXJBEra!q16-&(;fCqclgTrKI>m6pkWyh9(CspcQ%o-v$(xr^R zUfWI?{J&%h#40>@o_;Uic#w$M_uRiUoR5WDm}O^iTHe<^>+7+kj9-oBnc6HxiN^UN z#-ZF%bbMm&qkp_i)XtM93(L>OXmr(S>3)+KM08vtGgXjo{@I8q!= z>25KUqKZlncyj#omXfN4mIBBm_vYMXPQ}K!r)nV4Er46_hV~e@@eO~R_*6n$yaMx2 zR&ze9$?SY;K}O13xXIFOun_Eq9RDL81UiZ;$GdZ^g6g+4aA9WDiz&t@4eM9 zH8u6{s0tg z?{2L7k3UPQlCWAvXe%o{-8~G?OKBDWRbIVP?hSvtH|cR~Ew~TVSTV&wp-2bK#rnkK z-e}%X+;VZ(AHyl`B{z7>8ZrpZ`MgCUT?h+w*z>V4=6aNntL$>E)$ z$bZ7OeDGmlVq&7cqGvf;UI@1lNbbhMUrI{VBczfC9zjm>-<>u`oGMe>n!x2d?ec`8 z$@&94ukWIF9BU;^jNiN;F0RJ%Y^=%mV^+GoB-VFO3;ym(6pQ-VJ8)N1deL;2`S2S~{ybOCRC7Xs zngYwY+37K&*KVByTRJMG8_D_0RZ>Tjr7OC9?&*e z+erUGdLdHmJ<19+w&Q`Ffi-8I{^a(@m|Uyq1Lc&<5_l&PFCPda(ys@2UA zd&Ft_E(*F$=bcBTKRSb+Log|0mbAH>o7=5hx9;En0h`LN3?1O9FQKfz<+;wZQMM^V zH17e|EZW`#B2ds=XHEU==2CO&Dj{S=Mmw%cTIg{33Etpj>0y6cyik>fw|=(U+v|Pa zg!!T$ZnR(*kDtr~$GAz0CtlB$%1rJlz(u1~?c=z0H(@Q!ZR~=3R8$?99O;(Xdrn8n zx?DH1fmZOKRI>I#;>h#t878gXM0%$VO$s<}7%muq=m7L{j-wXV z8F%7}?gwWe3qCW}_w4}$CK78c%-lL9TE*sf?nX{_?q6LfK@~IqdG~Fe3~xrcbxReK zgx-M#6}Sv}1Lo7k@BE5auZE?&QmUA|>VCtxh>64<%d>e7XVM(N`=QoQd2*GLj6GHL z2(Y9OBB1zd16Ry7F#qm2|0xB;#w_zqOMKRV{*ffS2Ygwv~s03Yf2$4q8 zq_IIF0YZ=Aoo`?NWbG2&WTg({UQk!=FImUVeiU3yKX6TD;9ARyFZrAao1=}m()hKW z6~UOZ%grEY866kr1>p#QSXV##L{Fk~)?Z><&`ccxCOgs`Rj#)lM(WAmW&jlk;~=h?6IrbNzz!J(ya?2NU&|-){86=mlY$Y z!*|jOCX9@)Y#8#UkxldTxIgJ||EfN88ciJ+KoJqwkjb}S-rq%rXpUcXX}%ytET|{&=8a`DZr*oE4^3g1Dpmvv9wo>9{C9_OhTTx0)>6S-iswqzOunc>Y zz_eMI*b`-5CX5${aE=b1q`RqFbE*9G(ifdsGHQsUJ&!i zTGT)}HAL?_h!uHlbG|K)7m}Bre5cRGkIx-yLvv=xHYTjC?XC4hzV{;&9aB>lLxiB* zq5poIQZ~8$)@>yvEz>a>$IGgf6N17HV~X#?ifEB#rR62G=4)U?;e6qk@<0w2Vld+; z72uNvd6kBk20rQ{LV|Z{tgrO`kx#UAC)(=`eFA`EEy=lS5AMbP61asF*es2=76iT1 zTZ+|nuV^FV4X;|e)|XQfxO;UcZ_2!$I7^z^dZ~Lr$6!S3Ae#K~WMPHS=$4s;vfDh! z-|f8SK(IOUMoo!0FD38e6$HF-3y@KGp47%%Clo{yGb%WY4ab_4C@5n$3gGho!FC+) zVqQ!@7 znYHFZQ4ccl@|7L+=T~E8%V`0W!hnWjTe|@@1L|8Ry#UVR!!?KidR;r;oq45J;z}t` zy6DMZ8aEF*qso0{bdUB1BqU6Toj5Lr3yF%iuMi;yr~PmZ+1N6uR1?G}fGrrqEz9pM z5iO4NZq4Tj{gf+g8aL>9vDnu;L!C*xoo^Sa9Y3Oq7G3f2R04nll$M1&=ER)zW5l-7Wv4c4l>srnJVgrtMvDB zhoMYpEb9nf@eKo~&*7k7+@0?K^5Rsm{4wonb%!-9iJ^-l4WIeFt|m|{U`X{cB6(vp zk9|2pmB-zd;*9IWV<9_KTIR!zTerqS1oyYI3x0kp;UPPac#5U~3kba4I&AfxfAmw& z3CVF6*zeRH`tuxbPUH9eimzTCxp*yipfX{)zXQ(j6>f|$AM_+S{hlNjxN;!}SzlYb zh6q^4%h1+7!qS=(~54g)x z4LQ2|-SQ^ey?%Kq@@(qirKZw-_kOsboLv-Wd%LTPQ>RMUZmx_&>|Fwo)@LCh=ZPs; zNiv=4W{RL5-O?Nbl+`vBhHNby6e^!k#|H(m&x_u{9RO^D$Kp#k_pBi0if16>`r<;n zhaUUylVZh{OhiNkn@U!Yp4;nJzTN7;;wQz>Z)~H7O6N_!Oop_+CC8S0&bX&t-%B*q zd}UT~UySc(x{G6gs8_qG>tT7+)cHIV3CBIQ;!EkTZ&fnJ$Hg_3)!rX$XMw`Rbbgpo zWIZQeSBhx!T7q|BKujn%9NYn=$yMd`ixY5x9CInqpe20^q2DPu*N*kgoIZc^ z53LBn_7G|vPaAaYnq>)<#pN}cc^rFh1rA6WgmwH965qY=VqoCaYva&pzZdH+s&nqp z%L6W!`s||wLDI1KGr2DmK0N`?>uOx8N02qc5ezq2WVA{C0~QL;)OC_u0usuFY3Jg6 z7h|UB-rUGd^kl37LNzn=_N`zZ!NC z{83R+Eo`e#bVK%&FsH@XxsIAJ3_S_flFhtye9lqwIQ@#7z8$Dc(UOu=VtvD)hN83} z&jfw@{^SKB%D*O$1P{A1L6r1l`fmL_9*d6rn!}foo^J;#bDE@sWK(gRzdBXcp~U&d zR$g3KHq!splS2~yx}|I&#U_E z5)Apgm%ouA*ks$3!&G)3AkIw@wtki`dp%^XW$?`x9ia4I1=#G>sgX~~iiRiSUI5Q`p;c@DD<@p0dwa{GB%{j(p#nZd& zhqZHgP_U>58PcO4gJ0dhyo@_+!4$~NerPoPj5Ba@a?<0xMt$YW$D(meGQ(3S`?1Pa z3HffS8P5C0txqCNOhhO-Gu(0>W^$M0=B1+-9z-s|Xs=%|a?GkVs=b4cjpH!4mIdWD zwEH2RL^pB2AQFiTKj5k!lhfB7Qg&|1JnhbN_zUrc+D>uI;~MiydH{BI@%V7ejl=w{ z{e$UWO_2I?=vu6{9Bq610+axSvH*3Eumv_Dav&bd1Ht}EI z>C0aSM7#-Z{V_C92V82ojKP=*n5E-KiK0di)t6mAY9fDe^RIfr7Yq)z>{9 z9oND?0A4S|M1U39@`Jbt9XZVVbYs)`!(s$ zu1^zsR^h>b?Td#k<^He7lrs6(fHBj=T@>PlVM=mHNY`C4%Ap6M5;m&V__o#iBkmL3 znm(Q#?{s^P;Rfn5XZz`9B8+Z4|NrgGtEaq(?@6^Gdgq6eM*dA6x8Nw^S%Su>!Lmgx zSK{8Rux0%G@w9#Q=WT0R6Ru#bcS1;p2SC%6eL0ybKK1b=K3=%_b$%s>Om3b3UPlN; z{sn)6=S-c7ZxYBi75~Hdw^p#3IJ7j9r>_JKy~v-Y>bDbbH6lz%#N@xXHNpimJEp90l>3rR@e1OTGOM@90$jd&|~wN9N^`Geb7 z4NGbC`hBFa2DhT;W}@j zW_F2*=@nGQJt}VC8XAu8@9zVn9c6}Li~GM#zQ0{=I<_zdviaV0Ycz+)F*mmnykR*- zMRnUu&dJB3n<`*-OaYWsMh&L7t{?&&6Rq{hiJMbt6?}s!9{p$_u#%CA{*UFMBNLTQ z*0-LoM<4z3>#q$8hs*qeg7?@nY36?Ya{o;aF8;xi!Q>-I{I}2e$%DPSH|?h(#rLK| zjw6qT0CVB)fBEveJ-6kQMcch8ZO>U%V3->VA<-;YAz#JF|1y?Zrik6x)EAo@8?i?v zG?t|Qd)EGsUDnHP9TxEIwe*-z%kl@p$2KQD& z{yC(4m~039f?y-{QVIg8ibcIOf4mggSwa6T1cK51_y?}RMoP=mWbX@- z55D$!A2uO@tWjn4!@KyUcnHB~7`l2mO?VD0_;WB!i|N}_0-5QACwPf}a&Va7bsWRp z)So3n$|V^Gl%AtdQ@0*Jw0bx>g}x7Y@B7NNE%R@aP=j*Q+pUG=g=vitbD~f#1VRpz zbTRMxep_f7tg@0b-o|Yxna$9`!lJ+LlR7iGVHIPMdTwa7OmfhVAIf=*v#s723r%ld zzhpkoJ(l* z!m5Q?S(N3HM6bw#o78fH*3F65jgTyHe;Jg{W>_DmIos%Yyw($eySm_av#(`Ap;f^@ z)Rj>h)MF+V6#{*$is4Tye+q zUf`~3>xkc9TCQ@@=bCqic3BdIj?pdeDP~-o2lQ(x8pu8qAdY9`rnemMxO&~E_dJIc zJuJ00EM>h3|4D-^eLbZ^HrK&o^93`hmV>8({4+kvq#4)Ry0CtJhpDJRu$iL%_dX{- zU*?)4E4BX&U1gU~MkG=xQ2F9`)>~k#rO{mVf<}|cj_X8gNWDNN)I6%^5A@K}(Sk54 zBRkuG@?xo}rN(t*O~ZW?FXQ2)KVhFifW}b6z(_5f!684wn9~b!J-Dyrjo*;cAY(u- zS^LoEnJ2k6oGL`qoXHJo1LQdBCmf)8gIi5 zp``KwTaf@1nvRvck=d{RZ@J?y7w!`0fJYw4(ZS&ez`WY+$a2}LJ+*qxeBhblEQ-mG->u%m^a1l@1< zcT8)BwtozbgpIMWvrXOG$IgUgsDE+E-89ywV^`kmf)yg_*}d{+Q+?O#lsXdv=fc$i z{(X`w;B~r9EgEW0Bxj?QL&--pX;;!m@4ot^^V+%Tgj#8msX=+~`N(JlJ;lhC*LOQN z!;EhPxXnk2;=b3nm+P@KxO#g?f>&DX<3E1E5BgP|;t7?%1UE3MUX&F=HxiQ*W!>k` z6=x1D4;KR*{Zx?4`&D;YwOD}eQQh}u0>pyq9HwcM+SOGgLNS1MksE$S>O4L%vHPnC z>OWPgueC`mm~-T)yKU+y+uBM_&Xx;W~qfZh7pBo&>H8=dR)4>?dp&?9SsrR zgTEJF{CicTZSU`HhDiU2Y>q^zQJgN#$mL%;{4Y`~Yr`*VHdJ?SP|?IuB2ko<(+I+p zAerY`&(|SAB8M%wBnxGR4b6vbhImh)ZX1h5E@{)x%8i`^g=3boW2amVQ~E9zUKoEh zS=TTN?fxFX0c6#}-b)>$(6Xg8`;zr1(eP1x<|_zG%8O!^CTj~qVHu3d5w0GH(rVwb znJDy<@(#^;vRE4lo#a{j+g8tG3qjzLH2wP8bjl1^bjw?gdr4)N9Bg5BZu1QS>qJ9v zes#NiR8OArdVfPvVb0DTbpEwX%Ka%iNyWigy`NiL>$7&lM}= z?14;hBd;^>?-U?Ooa;R-wMw_P!+(<$>e$9f_rG0FCSWtr7?`qttMjt@p2Wt*T>Na~ zM}YAgF*nq!aYqrg^(n~EE~(6D{Mc%|b0`M+{c$m$Rzj+qz25`DO}jxvqHv@G_eI8T zf6A-*0CshZBV&(uU3xJj_%VAd#tvgemV)X}k;Hz=NxMsf_H*x5hn^l(>dV#0_x=ZOEIqN|fAJJSL528WL zo-BbBn(3_MUYF`h&fX$Rc}yRp>a;O#b0gje&E;90(0-r6nkI^t^!8kj$pTV0y{Lo{ zTxP(>A0vc>{cQK=4eiDri|=UG9Tt51y9D{j&F>_+*@eeVA6PXovK?@nohdgV-l_@; z;YTepb+jY~BPwV2H;05774cA`0u^%CKa(Y4wPMM%F&%>DMuzgw{F((gsLLA9-byfr zRo0eT`H#B`D)ihVJk{My=xiPh0{U*H}CK7QkJqkoe*DeXqo5Ft4fF^`nUAuL0Y`BT_ zjlU|=jMo2QOpMc@m=t|>W=1dmIHtOq+qG1pd55VMi9~`6bKCs)u}M~!te6*3!_%$z|-Bs`0}M> zRXQ1U_4p$)YIk!);9GQ;h9=HT*S)>Rf&9aIkqGvTTD5b_wUEzr5;q|Uj$VmOyV;#Fz$F6RDIv4i+LtWr8LORB2_7( z#`*2%W#WFg(it?v;PqsA)|mBC%(<|btm$pXLqg5CK^6!lksn4bu585tNoH{E`lSpr zoTp$O+*iC60v&g~y{;8C)6om|k7c@4o40hh_}5B2#J7A7Q!6IAO9vK&2Z1yA8E>f? z+#=yb)oD@DLt~jmpKtyCO+ZMP8%NR%Q*m4Gji#f^=Nz<``R7?UxVU3?YY8}yzXu+F z%6QH0UqKqXKi@$6J%|fFUU5&#jYfmXCq6`4|I#^bjcIOfc;(ikHf)3dQL?RGURrOq zV|1!{>ADL;g9r01U0CS(pFed>L)(D|m}1nThcJkx?N6p|v^z@E-XD1d1=A(Em?Jbr zMMXK`Ty!VzKXM_z`}3nLfz|ANws!c5Z`ZnnXOz@T~hc8PRUOz>X3#R+M~QCD)Z6_z~@he0IV1b_dW zVG(uB?g)YuILO!G=levu?k$BG#foGHbdM)Wu*2Nkd_?qbRR85?DooBRaWdC#bj~kF zHkrP1`kA;JaES6TWY`(|y2kzbwYJzo2NsQRh=ZoLsOac)Hh+Kr6u&xoeT4MsX!6Y~ zVi#Cvw7}B#n)l2S)(<##Mn)-lIwv7`iyV=B1q8L4JH4Tk%^}$(k8KOsSVJ7+ z(Che!#52So4CiLX$TOO#s*b~@RuK(RT@I6$nVZbc@gYb9*%t+<@}k49LUoW+if(7P zAbHMu%)X{!E{EJ9v42E6@>uqrW?neCm7mJoI>!mZIQl8Dyx{t)X!(K?|NU}yB-}YQ zBDvWE_+a9@LLM7^PPGcBlbs9a=dLa+WG6KB@cC#q`^kG*!WArQ>wK*e?ztcYwh)^j zBt1f?kLVpFqb7k{@u>88gc`Y++U<2M5;A5biZPn=x;b*|eNwgS-;hOl6wYy8R~M${ zIKPLjzNglLrWE>tZhZqI1IvD{v}8p4QeDu}=Y-G&Cr2EVi0i+HUgz^%yq*S8=Tw~( z!T&B5Da>W!Lwd(}S zZyGhza~#)kwZ+7tFvGtlH;b0@jyr;sjDCmQU;zUdPWhna!DZgJz7z`LZ6! zx!F^)2yT(abT%1#qJ3EVX(k@`l)FQZeo=z9X;Dup%q3#5roD8G;#n`b**Ewp-P3CJ z;%gA0V$!oB;g&pdDqqU1p{u4li7s5fY1AfHBPk0?0xJ_a!6FD&k+*MDkxt_3BVFG; z`JjlRO=m%<0t9LxGCo$+33^zV2N`Vo#ra3lb)L1jL0vk59=0Bc%?7<@^4=7EnJojt zIh81>9I@HL)GyyrGj)b<@K&zb3ZZZ+Lrqo*AmRnGBNEv}P+)jgG(l~7<9#_bYRD9DD|syK!}h_hdo$B!G?I!;tizU^-K+z?~C|b})ng$`yWnHq*mC$h8eA z5#CIWFcy@Ohu)rr6>PqUJ#6}$RJlCh|Hi<`+|TYao#_j52QxJ1}9*@T{@Zt zD)_Q~l0%S@?RKOBF7Hv3*}B_w9M=yhN|^hD9enZHia?_0aY&bS*2^^cxW2D@+7_2H zzFco}9Qof-<3oKMAFf7H(y#5lgdhrv)Px%_N1xNWIKW5+oT6$=OQWh~KpIz~ThV3{ zd*-S+koV%_hsa1xO-)M+3pML1y>_Wu$64;;$-rc|(A99l!JDUApoyJcdIJGOq<+k+|Pm$y7NJsCDRk^?72@&d7jli;bh&Z-2O0_I*@>*Sm5t6BZ;y`UR;DM5@_QOdyd(ze z=#`*~P>-?SlG@Io<;AE3HP|WwaDi&GNHbx;_*~ko`~s0?6hU4aKkiZ93A|1APp$D% zeV&um8RoL;$29kZZ_Nv~J6ZF!{y}oX8*FHWdu!<4m6N~=)7%K-#PyP=U!~6QhgB?x z2zEyk@-vHz|9qCiy>ShULA)Gb$P5(Yahu6=Q9s$^aIhfUytIe;Mn+D=L8ID4k2f@Z zuSL1v%|*>#JZR3!Pt&_^Ahe=62Zf@mt7zt*>@7Dw&?nl^e-VCbmQrm5lIXPqh{v9u zOT=BDwQ7(sjsH)zbEo~NfLXh#5uVnwQMPI|>)Hq&Amc>y9l701C3Y;PCp@+j=Buyqw7{WBDXq>nI$msyB6`52rEh4W!=sAT*Nfp)(elc(If%^MX5HbK%Mm9P zbHxO^=i$DhZcI&fGNDwErEl()`;+3H?e(j#4r+#FhjBV`xBH{IVIst~Jv%j1n!YGfF4x*UlG&X*k#qk#99$ywbL%%Q_$KI_>?o{! zlb`MTRw3S?%HITuGR&*$=Jfj9k`3SqMZJgB^yh0h(a^nAaCE3`p|>}O?V3{!o}3+! zbs0(<1n0zc4}EY3B6M|UMMQ;8_hxHCbj5oq&?-+Ff_CShrm&u6fN zPun12cQf{@Rl(dFS0E)hDKQxs9RZ6xA+VHu#V^9|f(g-ZaBzY>m=4g7f{}4Kzn$$) z6TXkl;+m;{O=|5M=?YtgemmF*0-5qh>DN!AG04%2>AEcyA_FN zY{j(pnK{_GWwZ%q=u5kL{+-4z#rMm}WPyOI2@vJ6tJ6~3uoCHu?J?b2i{4Gok9d4*-Xa0`t-IR}+RZLbs#jM=6FgB~)WR44J}R`|qEh?$1wKRE^#& z9bq?~a`0ib5w!qZsEZn3E8?R)8f5q+H`A5c4?Rr3-A8TLb+bD(6SghdiH0oAk$AVCLt$>W;Kb{eKi=Yb8~!z69%k2o9st~P-c9* zR6*OTpPz$}FxT3cSXe$vEB)+80^6OX=*f@9+u`=aot&w|m;x6!2nUUmLiX>;$;z9> zZ}`%Y3BUuc+Cv<+UWXeCc1~uyi2fL;`_l50%VCQ3+vhlwN$YpfzLx29-%{RZ<7sC) z4%mr`#;cHL$$Z7YZf2CPdkdnPEibG2@E_D)B9fL7u7h)r5>gq#rI#F)zA4b#B(s?{ zFFo_Fx=^n1gMJ<+O-xLDIx?lY(AAhL+eS7QE7SDWEDO#FH$IqU)XpNI1^!d(`To4T zxFihjdN!T%~v5`>}(kri5$!x38t2Y^mSw^0lqY=r8iCt3_rmFhU z;l_!%b4>w=M5FjrrXnEDJ?+A;q%@x1@*_%P1%be6^W|u=#}7ObQ3`rs@mDFf*u>lZ7fss_IR|W6q>dI|hw9uByS`cg6ohsr{ z&f4G+H+UtB=nV2!2IWswhTLeDL|6^QF9QCl1OUc3G;_9?uD@|DqHG6UYTKaP(Q(xZ zD(9Vi$MZ3Di$X0rbzBB@BH$R8LX+tY#4N)bvX0j_C~N>V+2yxrntZ?aU8%RiWqQ}% z=q6>@{2|Y=!fxsTW^F47OB&T{I53G<#AUmr808s+(@eEB7Y1J!B7TYb?KajG)G zr_nAm;5*691=~UKQhapZ@gQ!Hs3S3-Lv+#muywq^)!OnBfa|&b>j;^}!|6iODiCL1 zDj=SdakSsvHFf?_q}IJh>TY>6h8@r)Q)F?&k{7Cqi;Ii0G8);HYH#`-jvuUO56)*q z?4sX)k0I=1HLqE}24v!mx<>I__QStVTSBO88K#wsk}O7Bp_UHqkb)e})`Ten_h4P} z>q(LT4EepGFZ>@4*@3P~d8Uj$vlp4OdEzVK!kwi74iEij~o} z3Yo${s@1$ zcSlt(se-D*P>QFzz ziv0fWPbCLEztLQNW9`Q3C%>R+%=n)^*8fK=ijKMcY814O4rWBPDOwl5nrN%l$d-hF z9v{z@0lc6%Ee3WtXHd=gim2#;Mlub$xKJCPi{}rf_Z4(2L)Lqg#>j_H>wS~20}!f8;w>FnhoWd3Nh6_iD$n2hDz@@)xpXIpQdVpzyMS$J}UITYc-PquTO>4EoBN7&#*#Yr##=J&aXJ@{`NhX z83{u!H)pcjp1m`7-Ro2j-g*i=Plhr2dj>N>jT(;nnnfsiul|I) z`c&=xZ8IlIw2qrBim8x~cngv!IG!I`K68|dP7Bk#(6buv-bM7#8%OP)GE*-=L=_o$ z)yDnrwt}NgjkKBEiF`s~iyt2~Hu7WSxq*g34vJYOI2HxY5JPq23D|#*(0U03qcWn~ z8wIJd9GH4cZ%LK)5W#9H{C~Vjv~Kqq9UoZ|FDx4VUjs!&a<+y6%Q15QA@^>M1n)oQ zz-xkTQ^GJgitwXnRV)O^`sXtu`1oag8q#*yb`y6GIE>@tW5Ll{y9uweKksH8@(o*B zNQ$$OKmEHr{(HU$kqdtXQ}TLDI~_0TaZm#mCpX>OKk8^;r> zJu)MQhFKE~g?PbBU-H4|Kq>icHI(!Zpfi^5!>_KcPGjFJ9VG4B;BwE-zD+Mhpv z%6{nh3SnnwH>r0dMv2e=Bo7G!s#1sKlA!}Kc;$vHT37mve|+F0!$3B+zUbel2La@! zbQ7|Y$m2b@LR!0+xw#MPQ}i2+O8FRO$mO)5=sW4FHyMZRTS?jNmLL7au2_sMgbweC zy(ui2YSau|&3G3TyUCPAg30IoDMAtB7fW%Q7Rcx^4$0pCJ`vYk`?+$rYxfwnq2AfCF`>h5$p%D5f__1!YUS! zKMA_fSlsnqMe8^g?DNwvQ?`n^839Ml zYv((WgTMH4nHS!5!nsr#&=z?GvI59*Z7brkfPcgy`Mdi$lUdJtC? zEiq-@)rvz%_Nqx?xpA3!YzD!K{!3u&DWH_Thg?0fN_B4Lwf=;o0}zQPoQsVoCvolO zLH#7CBk+0_h!c2Dz0R9p--*AY<@AzykHiAUJaVDI4 zB)HY*Vi}0|A2m9+dTG19WvQ0yxqamGZW;MNkd5*Kjs zR7Ey`Sbk1vh^(E_Q{OdAk4}|&_cHTvS;&6xa`s|MJE_F+Ky2Fl&*Hx@bUx=#u(!=f z6ZYL}9I8lL^V5@-Ru)Ec30+E{f-X>Nl`qJEppk)73rTK3J;?s`>qDtToH=A|eSPO` zi=x&vui3wc=-v-IvTMgG^pbAl#^zb~L0X36o!w+%p2UQOb^p~=&#G@gimo&tFC34H zCn>qG+xBctLBB1nEgV`^k`xP@kp8c00&56pdw9M0XTQ%BK6DpyPK~Fs?>IK^=?U-n z_g>UhZJeK5Ds;iREU#8?hf?e&?Z;f2ps3|d*A!dBg6qUffru9VTcQqJdgZ4~5Ay2l z2NEd8?KItxAySz)Y5Y@!C1?RPkl~*{^A9hALqoe}+k8dFz4ZE%{bMUY+r^Z7z1Xi( zeO!#Bj}^gfq``X3Y%H>+ueS6hdMK-?$n6#jMyOslHoonqGcIi}Yp$G#;8b^=n?K$Bs(n*}f3V z;gFcU7^V<=C?to!{GIlOfcW*L7|f+L&~JcqqpL)NPE734e-aWe=8C>;r=cw#cv|xG z!F&Je!Lk3kZ-wUwyS2N$HWb&kwk6VW2NyRClM1dmk0LH5C1vloW+5e%Xp9A451yQn zo&9S?Y}ay)4JH$WUhCR*r`meRmS6Up&y{O&j;kOq+{A@Soqw5@(5tbrv8~}RxT&r> zPmn>VX>6PK5%er>)X)#Q5RnRKzh2kWZn^-Qjrte>Q?$@tbYO=y9LM_Kb}wFi|L6@F zXBH(bE`7-ApijKFRyRp(zun~M(xfW47Mfo}nU|&OtS4v2lD(G#Nt-aT*#ij697pC5 z|MOWJ+MJ1_(=n?&Iu;hC#A_sAAn2t0cp0xmn~Q~n^Xy+6raEesbisfZgiJmO zMzCH500d-zqobo!;CeB{TBj~0?*VttBr5L=VL5&Q5g<4Boo9VE=!Gfit)$4&R{ckG z5jy+~Z!=E+WGA^}Vk!icknuY@Rn0HC5RhUuPCF1@eB3o;Ycx7v=&7q6Ti{3X`25Lj z%d=#{ZG&z^rA_wV934CwTKZcUq>K^DKJMz*`y&R20e%(FB84S@u9*(bND?;T$NEJo zISDcjompVeh=1#v$E=ML_#cZ;-jaUYaHGv?b7Oxe0vy~ZI>hGnmSLBNw!1IfUz5CI zK_y9gEBv-%x-k>t%Pl9Dh{oWtKKP{kjgiOk{t|q|E)V7!n@F3@cO`0pU&S0PN<&bP z$=Y?Ux4v8BqADk(mm(7_JeZtx`1to+lYuZqe*`yP*`5r~$Q0d03O#%qhReKFYi$;) zn!(ayhDq_Axr+!R!Wk=(|sAL54=Gy|4dh04qsSj=VIk3@C?^K z&+dIYx|Cp!78Hfz;B$GkOCKOg(5;2Sm~1~?fBzKvm;^H~w0a9vJL{C{1-Mc_Yj447@>z#76}L^5JJjnC`Ui3T z&?j+5nP<%t%E-v5=_G3wGY>RK=?BZWx$!2^=4|`Z7tJZfCH3R%j6Uq_f=m^kW`8VK zKAMEjN=Nu-nn0K|RpeVez)ntTAm?w4s_tk;pqb^3ODVQTd`VAdEg|ZKNK$V;sAhlX zB+J7|t>de@=B8kbjFaRX!oeaS2rZRKoKv$7U=5&sFekwW+%7go7SA(E6#!;j!3JG0 zxRvM<6KLPz7u7(%J`BQTCu^fwqB##0(;zUp#gZu|J#vZ!-x4?e-YNsKDCaTEHg|Y%M(PB)E9ly@9N4<+TjZSToFK9h+cA_li;e1H8V^b zjieKa)3j=p|!3bL;VQRXsv%J1<~PomA+aWfn92 z!V-2bWr2`prTH6*xqVz?fn#igTmU-j{6s$4$nvt*%q`NsBaz;K^OZb!H(a*=Q2PH8 z`~UfyOM=H{xc2VX#&>L5xp11V2?1m6>oFoRloISGQR;%8M^qXpW`Z^&8FlzPNg+$E z-V#S0P-m=;i$xDgvTwLVt+`|jhPNgO_!O4*ii(QCy7`KrirQZG4cG+@lVuG{G?`-} zf44>$Yz7mVUZ(NWEoXpc%vG+hlJ_HVpa?Pb-mlvrBp^umdoslFk+w*`=IfE;il6Po z*O#qcXEGVcD8|k1vL7Z7SG&WCWJ2(4(DZK3_g6U#>!XT^q-mjg&!Km#VU#pUa8Xhf z0Acd;6Mu39aOb2oG7=JIQF_MFYMbRY-^4ok7%X_bdjZW}ghF_|m_3F}pFk}{XM1~Q zA-oQN%vIA#BtD@;S&<4t6uVe>tCa)K!!ATrwyHXA%$r%0Mb@CeFZQfQN`v zs19$;{7KE?wizbNgCb$bfL7B)T|BF5zRJ#Q*(y82$P(ZN4{;mm=eG$@?$@Z1?e1Os%_mGGLKL*oSdtqo^CL6=*_V z8vIo!Q18?t;oN>rgM-~siDgI>i6Zx4Rx5YUfZmO`d-j~W`cxaqVskL5%kl>*;$7}f zAqDfrOr_`7U8o#tfG8{h7)UUlAAq)RE)G@2^OfT|yLsdsezowDw1nL!1-k7|{mq#p z=pbq*ebeG{y!Kr|{ka`59mlh%Rbo+3PVmS~D~KV(vA~pJ`&w+ctRfs5=e8RuRo&}B zgfvTrGHekqKd+&&h)VCT`bdWvJ8LXVL_!FhM7*|fXlA`uqIahgM5mbkjg_||C9fBf zBKI(Fl!^#kl;Dhj(ue_ehW?3a!sc8EO|M}qfzo$}HT^3suIv*{yP1ovCun#6{8p`} zFX+=n81lkVda)LSd9$q>A5lAZ%a1bHngc}QhtEOltyy%~7aZAtgGCgr4y7jpC>cjM zqXY`qm_h#j@25Z2^>-U`+7UMY^M#M!+{Sm}-Swp;I?(GYbA6XdnUa*`6mF;h7sZOE zok?zhl~!AqC8%1TcUt}LUH##T3L_5qPIqYJv(OQS*W%|zrDshM-8^XP=y)n;8C8Cn z)XxSJ64l}7D<`h?$z~{v+6UtPmi_k%yz>1VR#J~mzc!JJWrq>XfoX3gtNtQZcI)XB z`Q()d5~gYZV|u3dzU8}Z$w5`vv?tbPlxKVWJY?+%t=ki7u_Jlnjxv4GK5S1Oe>uZq zQx)?X*KTryo)(93DUT_oFPavyCV46I@Y|^~0 z&2hN9@}Sm6%&1T%%P=Q;ljZ#FZk8w9T(6%9H6v%0$_=}lJ1KzS=XVmK&@h`Tx9DPW z1_@Jd=cSz=5`_f+)t)_f7fA=@>$XZEeoQXqY$Zl0k^T;?jCCS2;kRlX9csEvljXqw zDywn?)Borw61mleEizg_^fW;p>i@%=5S22R37UQ&=srZw0mFzuz-A zi+qhOmiZjX=WPB$oFZu_l{nPZt5#)LT$YX#T*OoW=z%WFT3TAgTswWJCS>R$U}Dg; zC#S5xXO;LX8HSl^mUCb_P4q~3&|4}y5qz3kEKjZQL=<08D57ZsbeG}3X0_=I@eX7z zhkvzpq^yRFT;zJ3Q<>Eb_k+`kN?O;z`7z@u`&CvFV^U4k1->;PZw^F7@;Y9OD1)Ut^I{ZwToQ0_HR8LXMpy z?N@wk*F*;&1^$+JdP74&ra(WlbvFuzuG7()mD;Rr1fU4v|~KkiTnei9zD@0B%T7|_Eu1g)?B zBPr(sd2fXGN+^R0ZT_;Xi0d~CJr0LKDGy(0N6+feqZmegFCT6=dXeSYzk|eAG!NNs zYI)X9Y&}wLh{aDMtj)0&Ai*0ci3AfKMBAt@64{to9{C2~Ypma_9YtYuIFl(ccQ^}> zB#-hBdNG*6MV(F(?LMHiS^DI-u52*&so5~;w$RKZ3akDv0rMjHp9ZRekn_L84Biz5EFSOyjwapTj_Rm;LSi^58zL`{BL z0pe~%(@1!awssC7WwmXPVrrntx9gU@Z8#fiN(VLXbH(;Z#E%lu^ho3YnNIYRLas~v z$s}s3ql4nV-}%1yO*T{Xuv8b^9e8qcRS@Mo5 z#Up;2JQT-z3JSpKxjcNZ8j+wgl?F+C^6HDSF5CgmYcL4$`G;TK&Tc%B!vZpG+h^{_ zApxJb<=S+m#93(W+qg%oZuxwgruSgtO`&-xfoSNzj?vA(7a2h$k%eM*2+V8@j+aJ) z+%zbHbY0xt{2DerTpwA@>wh(wof-QEXTrB)6UGdC6Xu)2=@5#zrHQpb8RRG z$=1tnzxSv!_VG`NEsY%&2gt7+Qoqyy%ePVC>NN^a&)FgJLltmybp)_3ouMzxNQ{W~AP~Hx`Dc#}!6BULH?2|go z|8wzssrB{hkuPjua)#!%(R8>h!^24D?pm68oLA`{bRi8queS5`m-@Cow84>_p^@Co z?{Z1ya(j6UK<7pOhdUb6{K2cgTOu9@QEibv*3X658{rRMM(29NH0+3c^VYo&vGnMG!rZMHQH_C5Uq?T&Z{v5R96!yw)eSX}hY#{NQK&EdIl@F82CtW@0cjUtbNYFU z>s(14o$mn9795Z-5N1Tb0UaPZgRWV={?QyA%)KMpjVGJUp={ozq;={vw(RIN#FhAQ-bi@esWMaNj(ulL2uk!pF|*63vnEoqU+?N!4T2kf>B%4GyB;mjU!+q; z(vvzU1sY$l@&N}CZg|N!0MK^P?sykekRFRwBaEynFu@w$EXUwUIaTSYsc6?y8fTM` z*l~!TrP-5NCzC01ttoJAb0Tm$Wzca@$Ujda$PkGQ zkLk?t_l038tonhF;gOL(6;R7>3pznQ$Dp2+VjAi1gedr%WXhoXIISIlLgS*}Y?bTa zdCEz2_?^N!rcJxJ@VTk#Jqq7fLt^X*^nu3h=Z9Z|HnUr z%t@-}I=|p{))CbfsX2v_Bn`Se;f)8s^Ta-*7zr>(kBYy1`|$q#8TPn>HH0N7nXHFF zDBPdOzyii)=1kN4I!GO|9)OZI)W_^{X$Io~jW2xZCFS2xc9tLRFF@n>;3Mxr--lp( z>xUWrw6t{3;zgA(cfPq;RJK-Ic-{JtmFosV^l^TGT%r<`m$1E_KpME)7 z-YSvb`Qqh|v{45_yvs@Rsp(q3OWt@ADiVJKo6e57rxQWRXvRc82R-rn)YN;rHcH4{ zZ>?@4yOCmJ$VJ=%Y4mn$c=6l2UHf&=&eR6ld1dOD0N(OZbqgO~ufQ&8z|4f_0RbKd zTR}=8H#S~Yk;7VjR6~&#c@vyNCpoh#N9-2+7K`nkfyLR z^ulDkP@Q&l2SNyE)NnvIrZm;kRA^0PoQtv&fy;1yxSR>!_%Zr>B;m5>dh}^Pnf+uj zwJZVCcZRxb)}$cNcc6}};mGb0_)CWp!3d#}+bK!E*Q{mk>~q#9L7%uvOtU<^j&Lht zZ$O90Bg=k8(}MAb?}HkTmK@~djCBDJM8*`6&fmOG5GdH-j0$2Vt#tU50b0)d_~J~C zx4tiLP7iXKPeD=(x`C%U1|j??fPJR#m4ZkR&p9|(u>Z@15Iy=s8_Jgbs=n*6Q1R=2 zT3>B#KuBLwb#hn?gohNtU+zeafbOjd)$Lqg%7c8ee|X|v(nP}3CX=!ZFZ3bnRbk#??u})WjW!I* z8Qd?0+(9l#5hc)HBDCK53NSLc2rf5!mk~S}{sAOZFjLRGGRNO6xnGb_i8}`8r#Ba! z6pWBK)!ZqAxV_x0K))g5!*SnF%+tNPhYqPrfdyw;3%$o@&NFc?Nu`|Rd+*A{5G&Cc ze{&;#Mf&|1mw~p-r1e|N+9TB>^iRDoZ$qg-$qz^~U#mTb2=;o->0*#eXMf3`kzz*s<8vEwNnF#=^afcpA?XpGIGAM4`xqNc#`Z9$x}nV(7zJ! zdMKIM+?<9KZ(HN}EAKNeqFG%NUvxrpO^&JOv~nRs4*PF&bCFQBA<+CcBnv2J3y?Ser=5>GG~8in*aXN1qp;> z`>GD4JEOG$d7MjqL{0{HS#s>FACmf;Kb)Z})Wks;-P^R0B5)%X&XtIv7!%}cqdlw9 zPZwF3QBjH=P||p>nwUn6?_Wc`q$0bti94Pe2ztikl>TQq8xD(3EM1%7hb94+8t(C1 zD>v@@?nOv4zx%@F@oKkr2u5@leo)LLZfJ9D9+B!WH_QdIaE|3A*^={0YU%<0^CjUL zo5%#ooPUy?ODZWiw2ki!A}JpAtChh3U8o%X_z|PvHU~v$ndD2*!;v5iYPoh4N-NZ~ z76Uh4Q2I~^QX4Qb^yea4`yb|u@|hI`7f&TxUT^(8#)j>&aHy*ZIo@AqUw++^ za6X$!y5azj62E%>X*pH=E+IVWK1a6RzTM^SY+Ar!s?>ddj>a^i3UkGB=jMec(y&-0 zlEf&DXuM(~^_C$Zib+YJdYC@o;w~NpZrdaR2xwN7Z{PgMD8;%jq8_gI8aIB%)3Xb- zT$Sd1)wyzJb-=SMb1FjCXOBPw87TsyFzbiFm=<>yol+`!WGyxP zR=CDr0?R&s>!wY-?{DA73($Lm_N#$OtFTyraC5K$}q)l)Nvv|p|N zw4R{sHNR5z@o72Q>;L%^OyfbBoGmP9G_E^+eO*pA2EfR7d=s8153L*dlSb1*&@~U> zD!;2u!gvAJ)9}bXvxS4xiU^3JTAkx;HNfXk%s$vJ927nm(zq-(Oeeq#&nL+JXb{4X z^D|wQCmu)B&nv(4=F4N@(dkW2%6_1%9(E}M$!v9&1H}#<>7LMEat(x|kU-^6)jD== z7|hP0Zf+>AtgH_U$yz0m;(u>#X&%zQ{4BHCi*6RR4*K(`zSC47y-JaOAZOV&69n_5 z(hosHjV(uAs5Zmtb`xLGHNJg!xmhE4v)p#qu~tLDfd)x>vcb`6augDh+WLtuW1{)GRLj=ZE=WrgkU?@Cyd zI;>oX6b(L>GM#e&aF84}0||6s)*g?z{W%w+%o6rfGsS^1WH3T2?s1UbKZ7;Ry%G+5 z)jr&wF38;ZEfnBm1Y4r#>Krm9{M$hPDb1loA_r4*jCt$o5k9;N+fb+5ghk>RdJ;h{ z?#2E*gW??wWDYA39h%U*^u4{kue4=vV6oS#b0Ce9DWb#*Yp4Q8SkecL<=HbC&&`>X zM#PaH8!K+dg1+0M)Lxv3jU`8F=!}u_o1Z>?Iz4q0Wg7x*#eQ-qm2tf)N3iHuM@0e6 z;SNpqlL1g967-r-r)I_>V8W|K$ie(D+;&+h@2+J6jjxs>FdKR`v&J3vu| zsySX2<4bO9jB4Qf=*u2D=U4&WmoX3W^#X0%w4c_th|4`3XAdE*ze!SL&7j<1(kA&` z2mf7i*kW=Lb<^cp#e2s07AnO+Dmb+M0;~|VTrR{UML?*QlA5+W%_K8(@IziZk_v?C zMLY^vB88|YRRSAa?!P4ATja9J?<%JD&wUA99yrm z07VAum?kE@YWO=N8Q&pU9W^8Ov0JnO0-J`_K*MAtBowM1UswNG2`dbf6O09BVK$o0 zvMn>7K&}c|17=V-V2?Y1-;0d-LwZRWH8T@qYra*7sn5I-`7EoRxmIi>GfLb zEp6$l=8un%URG2R(hc;6gy`XDk}lI=P?~L{o3NV3=XmF6k)A$Njvahzmgo8cjS?=Z z7~)?a%D$P(W5cP^Agd!PE_N6C?x4quXp?tcF%ICss_KyuI*KgS8@4iJ_ZlP(jtru` z691G^D7G>g4;5!`R{%hPnZbJ7Tc1#Tcgd8;7_3f8=h>oxA)JvjG&m?DN4j%p9aS7Q$5Qp~m%#G$oW^+I6zHI}CHeHu_f8U?cE!?C&^}_euxcSQldJtpXIv2&+0s z5|2+wVGv~)72@=gL>nm9E|`$*Q87s8^1W(Gu74xb@S)KP5$UNx&$B0$GD1n3JqOF} zrm1!EWNqA%6{0s#7q-_MczS&m%Pcm8{$K7oqO308ha89zImkc4E*1)orx|wS7XfcI zZ`!^1pAC+im&&d;hf+{sP!f^(mFdz*@V(ccYj_iTR&&BeX;p^F;%QVxU844!vP$L@ zKJ|$nb1vPL2@Pml_@r5-fQi(=3H~m>$)c1;)@sWJO~f|oFl;# zBeC@CRff-S^Cg0eGcF*{&Y*k%m%!y)up!#}V6!gA3+ov-QnIjE8Pc0nWGawH#UR=L zdFpcGi}vcLz!dX%jNS89mLd#>grWKa3DYNiLfA7Qioc7$f=8 zqq4Ep6-?wUu|s2I5G__Q`%b&u=0HZ1XIb{Vo@`C$=Vh$4w5cW%gD8|w0(qWO3uD;yq%%6(>bACe zlbH#O-UdH*nH?Tf>3a-$Er7*IS5cvJKT{OUC`~Ca(PJDLOAe*j)JmUvFb4^P!mT@9dQaI=w(QuT5 zJ3=cbGm{up)sV;*g{=NtVd=jXnf~|xud(skTYPr1_Zbw+>YwD@@>0Wz5@+NWwC6bA z=Z6Pwf8{1?4+ICLowB!j#zaOM-r_gj;>FBg<<*0<$EQKsOB-;9&okc`W~4JXpfuGa$!s2o4~E&}dhc-{XH*bs7PgySmqmOK5E1bQ1$$j|y4BXq zAKr)@<>uzhw3@_H=OCbUh;h$&g8IQwkBpE|c3*1;W(W{Iu zN_xuMBV&fvCSAl}s9U7=GkW0|PioL$HTg_N@#)U8MhNtZl+HX@74g>MYRnlZ8JTtW zC*HNsiA*5Yjud7Z#WDwe^o$^Iva@V;?IeUL_|X=5GF)dSX)|Bj^7~6`ErCE zGf?8=OB+eugTKVv@Q;N#zAl>d=bAV_!^fjdXW1zcCJ402ni}P^0(WGRl$Dr@nKLRl zh&54aiDNpXnVHB=G#fBKb{l1wJ=f&W;%2LH2l77|%=pZYv+QK_LR&&xw4X7jKCES^ zPHjXw53P*_pd&V{w`dtCM^TWp0k0x~xcx3k98BeN(?3M$U;A|F1%!l)Sj$S#`<9PhoBYH5zt0C@&S2e zeA$ewN~U5_2Lm~gO3N`qIdq&8dafj1xl15rM2gGcfv;+G1hEZMfMgj-F>^pFv)eG9 z_dsGn%?W$Uq`Z5vEI(O(zM|SJyq(m7Zuc2o{6vXN4d#)aCbP#phCGoxX0fWS6Yc|; zRA|8yyvvIAYE4r#nT)0H%&9@;9|fhEk3^O3=}WgsZ4)!EV)4~h%auM)i?}Plu9XEI z=B8?h{e>SS&Zg1U>(HCC+Wmj*pwat@Pu8Z92q^_e={>Hghpcl{wB$P!z|Ya)yO5v~ zOpYWLp{I#RXf#{cdR|WVoB6HT{AlJ#Ym2*iR~!N7X5rVM>nvAqxw3=R7Kh#sp@B-k zy~zmPZ3?-mc&=yjs@`X0%dNfqXu0T0Qn-52o1v|$M zvsq(my@vbE&G!n?5)o&SE@a5HQq62hqG1u(%V;Hwi=(oF6zQ^-mDs&z?Gc4q6l$T(*@Vxwsg@oxE1{n-jLdGlQ-9QNm$j$umunDn{qkJAj9BtyoQ!jQ=q}`X z_;+Z3v0=+6V*y+GDfvBq(HNwZXQ(KD-XFw%V{!VRuZoyzX)6+$OZD&c2sFZ+6ZCj# z{HR|hiVA>Fso?iulfCSB_uk`FJMY5#&tK-R{O9-8r6y%@N#wGF;C7lQneewT8V2qZ z&C&hfwpF*}EtEQefVx#1NZn`vn^=gP%?(3G2 zr&o|Q%1ksGr}Xt^GA=lk=3Rvf`=osgO3oo#*uXI5mcYVJJX7g+A0z?ao1Kvnc=VQ)!A%wu7IgZBHQpH>DgksI!|)jdK2RrqcSA}Vn&hKA z*KQ1nhB%-RRq=*0m;d3aDLH)b!QJp7@%bOwq-g>CWrx)W=8?fGvwQy0N;>*S0QS-D z_)*y=UlZu3fS@(4*&0OCv84CE&6mZnS^3fM-D@0?GFonPGO)S5_%tvyaPgzg=~JWH zPRQl*+IU=2(w8iv!ne{qsuC;_Hy|I@3YaorkiUETWt z=y3$s5>p4*2B(j;C}=8{E35ZaLbJ_-joQ;vD_NMCwe&?WoEMuQdVn`p;q2(>Xm8JU z&d<*exMGY&8~#wHP7);BhNGy;fL~p9u=PQ|y(j&FBAIv-Cno`g0A2!MD;w*Na~*B> z-YTW*5nNuAQM)~V6E-Hy$cI?p8}QcP%WM&tQ14Gq6#M(&WH38W3t_O9RY$BA_56xW zTniQH^wWM72@|s*Fii++(PxID;ZU7{%pW^vGp;)G#IBJ{8)7g>nlxoP%Kn$jvms(P zT@m|J(qJ_a?6sTqJ$E;)_OOixizk%wiC?njLM^i28EnJ-=;#QY$^kDaHH-WW^IerKA6% zBHQd}o!Q<7#Q70ifz0W%R5^nIS_IioW={+dIKwKESb9J&w|ObnAw*Pu%bZnu~DlG z^?^koI#|1dZDemoM>nPQ{tys0Kig7prlh9kKElLnu6A6g&m9-c9zvWtLjo`U9>skq z^`qy<_Gt!lC9-I`VJ5C!@W2E1nD|};~Zu$Zf%XFkou3}kXVe;e%oEOe^>t2w}(&`um~GD z{C-bAly|5IL&=$s*Ff$JIQ{jr!%m1c^Mh-BMc>fN$Mq^d%+yrmI*mgS7+4N23(h8hwSLN0Q$eZ_`mJEj>pE zism`Q@b&u4ridE$4ZA!lU>x`0BS8Pvces;mBljGH^A`nOzR4Y4g&9l@yI61U%eN-0 zg3@8$2rA}Fu%|%5BaL4Id`G{RbbT&+ma)~M>}y4gTpXHN3xFYcn3U^<0{Nx}v)5+M z7S?mc(jYM=vz~^a(+xBYl<|`c_1@o#mV1Fsue3W`l=9&le3I#6PZdD@QFgSF?j1dp z_BKOSi?`v@^D(^;L}9R{kJMT+cAbd40Ny*NF3}~`*%a$>^ocz zJ4>x*itLhw{1TE!=i6(31v-vDCp=}!X8hMp>>KLI=S z6L_iB-QM;?%WebQLe?5axmmu7ZT4x13diRvZX|^OBW9*OpEsbOBF) zH)~hqYLN(GN=Z26znEXxZ2`A|F5fYz@{JNH|=FMK$m2lon~h*S3Rvxrwfb!J(^ zl}Yl|w{R}p(m2-uG%UU?+_Tj=u-Fszy_lgWBmPR06Uy{`&5upCw0)#QZ8uIsf)j_@ z&~~*xMt4S)Pl!bAKCbo6zoLpF?|OR@#da9?KQD7d4>hNKQka(hV7=}2w4%{K7=n|~ z#L@Q3>s)^$bLuCRj`giCTCB=>kk2L=Rg^;C@^Y}3H>5H*$kSs0Y*2p-9b zU!N{|x0hZa`|S_j=1Z7(Tz0-$9yJWXLtF4k+-V=jEPI7CUFhrVy|EU9N|ZUXv0g_4*}o^NN-et$P;9tTMx0ngVvU$Y-@*+@!W zK^FR94(C{C2~Ujgt9a^1c?-_aOE4zau1|90tkjmblBu}ZmK=5Rfro?6N9{_$t@yu_ z*BU!$i{iJnA>U`-dPDSxJd#A{_rNX-_srR8_iwNXiLx`kh~Ag}gVl^=rg90OFG~?7 z3h$KI`{n~a*X9rX{>JV_6n>%j!^{p5+eJY|ujHLvM;};ShiLz?KltnQS5sLV#FJ}! zCLT@8Xy!@|q70CkUyLZ+3al))8PF>81ZF4xr5CwHENyB%j~!GeABz>7$7FFFIjALi zH~mPmT*9og(H*JSE9C&j%kV!lBo{hdI;(6nf)n?Ykx-+kEuKgCrDo&~?|HGBjn$U# z5Jm7@*Owtn?PYAs(&g1KPCBtuxD_5ejaCP5skKxdn}-uBNo7JTao~)WhcI~?K}2Bt z{n`ri9CudLfzrlZo#v}+H=Ey$RR zM30ip1ROQ`ucGfR?)d5<5bbx_ekJhTC&f z=+#tRUfo*%K@bvMb}epkal$7u@$mWEa%Yv1ijTj~ue>Id|ChyR8v2Bcsr@EA*(F<| ziE-oMPptXh2acVslWDDzCfvW0kE$do>m6Uo{?9KWYll|cGnGVt=R~~DTu@|s+giI& z;(t!VH!-hcyk3ggUmvc)R_B2^JDlxO4&RbE^#99C3gm&By$l>7s~J5F{T?GM^oHB( zOx)s~af}|v|GXW1(|8a76EOJg%feZrFo(3B_Q_*MS)&g_#Xfv&r7L)P$0~Z+58fFF zFRdZ5#8o_K{t*B@+PA#q2>9Pz>uvcP4y8r1-+)y&B`^i-Thu>Z zo(|zO+HvEN!wWXFpi47td3_NO9JFp$Ak>+aJm~w+mEGNxZ1t9M1s6DC5bH5%-{nPlxxBF|p8xX3#GO3%VeYW=-xs}s4TIPSeKgL2MYSi@egF&s!J@^j1J32sD zC<$3~mFlrB&#KbeEevK6Y;qW$){m=BKh_+{)>1Xa`g_4lgjeFi2XDI^CddLd+g>kr zl>_w0Bn;Eim0$F`sagEW3LAXDofJc!wD8CxtoPybKecnlnSK3VE8_WRGnF=YF$xM8 zoGgb+1sAAd_2qMK)J#$CE-tNvJ}Y93EG+)8=Da$-GrW!3+JP*~{Y&V3$Wzlm%{h=O({5dFca_I)gUxAhV(7}Bf^z7L z&5gz{Y1_5i^TV+6eR~Z_XCuvg1ATpC!xt$>K<4*O!Lu1R@A;QmY!Fra2dUU$26~GlF`H?~8_8T_k1cn@m zc0h-!*Ycgab-%@!70D(B_1ESkmxIgpq}JY3+mm)#l!S-2xc-8F(mvaf24@DrZ01`I zc17@b(|6L8hA zOvHL;S*^&Y?s#DnOZh|b-a;r1Tp4*z59eJd4W0x|!`^zzk5adqdfMF`$G7|Vfhgp~ zpPvb;=VGwYZmM@5z!{1Ag5&b|vsGl^)&>UpZKsYro4@_jiP&$g6>DaTy^QT-FnvRDlCV8=mdBv~)SlMf@Z0B2ifh=kbB}tERp*GaqQpJK|lB8JTS> z)Yb+lgMo{h*(n-0O=n%lHN4D76b?uqB9|*Odvtg7BjN9?M@X;u$|QvT2yma_CyXZF zcQk?YJD@NR_yI-F#oG!RRyd$4^@uAW`_;!Qp0V9y3yDpbuy>PK?sIBhJycxbQM)*I z_GY_pzSrM#<8vycE;OLlU<6R>@(@H;K;+N?c# zZTz;pErEm7Vj0tKJA1)mc!*SDt_2aN8*aTsIsdwjjpcyhh8|sF7FGeAb~5h55wt0h z<;#(bvE6(Z5z%KWrPvotPv2*Mr8}X$Du49=cDUVFTIY>v0a?HXF(u?-A1r?hn9yK} z=Em0g^6GL!+o+L|-J_Y(!InNxO70(f7vdH{FTpw0zU%7@JV69*Xbmy~L+?X5RL5s_ ztLV4&(h;!Jhk*h~(X|jkH(bV#A7jxhN&#a+?{dB*&h^pV=};1R@y)+R8@AsE7+$y= z4pPJ{zl=ZStkx0~+o@b@bJaV`Iz=?p`WaN({d zFu={vGg}{iif?EFxhr_!_wU*1-=EJ;R@aG}#^CN@gAeO2z64LQvG6H12LUAF{u6Zm zzkN6;7*PlhP~3A+Xs)sNW>OF7H(M^q3o|%Dl8j|=^0TFGRYT?kuuj8yPBXxQ4tJbJ zoA}mnsH(b^E)Pb4z!&;oJZHF{TlLVJW0r1eF?dMM1vIk3Jd+s%8M?Zd-;y8zYqlN( zU&C9C8BJP1e^z8x&5zuucFJrwtSabQCD1&4+U$^!B2f{Dp^ zo4>FvClXr=!KJERuNyv@2I_ydyw~`@XMkK@7q7(i@?ZgZQLn%A&9>}>+PH3-0wPt2 zFr`GrhtLW9C$KaQl(X5z-|hq&+v{_w8xMYK&?SVHHzn@1?=J>BA*H|%EWKS{rLo2G zPXG$HYSHgk*X~lgOM*-VLRERWssB!6@MpUp+SE+rhKddwRQ=R9!V*p(`XMVih=+{Z=Uxu{T&t>?^V`?aVnXat;fo z*ypm;!g&XQ!%?}82sEsw$5Pnn8b~^3=JdTDx~I(bo!~Tz9nco86oD`xaO_*l!QH0c zdpnfIU1UU%yfdy=POr30$XUa}kfcA{`c4m%FDl;=CGoh*o1`4iLU(?(3C&r?D&KRJ zB8diU8Q*mV^x2?rVn36Pc=@>PVO&W$e;}v)z_7!W`W+BR0+~sNuT#0{ENb1G@tkmZ zr8W*wWdnT@PQ(LFps$Qxe+7lMbqfPs0Z@1Y2<$Ys6fp)VLFZ)a!K&Bcj4JjR$$ceP zUwqf@-Ma^rE#|)$gRfqLoE%&<-TYSx_z7^`A~akQ!`%+?LW#m4z+?e?0s8R|(3fg53>ko=Fx5NtBa9b- ze``7+GGl4y-mx9+6&c^2)k+tp#tAr!vy;qr%)neyf zh0d~;4tKHjrBm?H_T!Bw$zbpf4mGe^-CRD)A&_z1D8@1mxqgrjUK@P_&=mvu(2XlN z$4`rtRZ0MCG}l;}w;e}f`~hzN}XXav+pPB;)=67vvt)34tk;uD zvLG>p@l@dJVA~)`kiz#&309tZ=?5JNBIC(OMgLTU><8|V_qJdj~|hmi99E< zfOi-X-rp?@_~lQ@+keKO3}8P#fXfNgL@RxZ)E<-GMZp~Pu zvuFCD`v=+p3g&-%J`A^QL9R#vQp&VLGUe6;Y~!z^X)-zYVTGRW6j$i4QuZZrr7P$v zLfamG%%;|z-a|6|K1uM=R2pX{75W?sf1o>42vuVdQ|x4h+RRyW`|?iyEVI^Iey7Ov zEjhbHK8fAfCW~Z3xy-{c+RoAHn_)xmm1uQFlO;-(3`s8PGj$t2sllM_Y+(24n*6y!hMz)ZVt#T z!$X3*7gv_cc$|;-=mZx-f}Yxy{ne-7aR! zHyXFq0?Zv-c?LY4n*tZK`+y*!edOTgA?a=}=|y)0u-S>aDB|}la2J)m=`IDs)VIEY z)Uhn7vk4_U5itCUerMO9de1iLkQTV|L6y7i4}VWQ^@!ML&oStrZgOx-^eg}Qp$8}k@A~_? zf26pCXho$ssG6p&!%f3Kv&&Zmw5uy71Y9>vAngMx>FH4|%4N3+kg*bkVcomHKgD7Q z{P`6()cSclAP7*KrKv`rulMSu<<0#^MHV%}6?$Js( z!q)8y1CMG}gJ=fBfDcQRxN16k$N*K zQ{+=TkfSH`3*>~(K=D*Ey`Q@Msg1KB7vTtmrIzGiM&F|hd%WHpoZ|NRNYFJHw%@VV z=ITj#qF*)Rty7q%kd9zuNbdApsGCSS5>O>^!rOG)DNXU=+zKBIQjf|Ypm`BX;>e$X(5`F`IMFG6`x;vWm@ie z(53!s8K0CoC0m)yGhR!Ba zMH9Qcubh@EaM^{c2IM<6)_4O+q2Vsj3guJqNms)=%ww{l`V|Z#*QYOQk+tM-tiA%A ze`!<3bUfDYG-XI70XiR|8<1T>S0c}vdz$|dkOXX0I#lyJd>733}Wk z`LmX46WaT(ZQ=D6Yf=&4V_n@EfuM_HptyS!6Mc_$_fN=xxIIj8JEBL_hvHD`Q5T^A z`PX-N^SX$Z(KI?|lTX>?Oed9M=ufm{bkNjrGGy}bBknu+J>Beek()LR_1Ct&xmkn4 zw{8rgShRh;!$N|7*rik+(}O5bH9j-M6XmDY7V?TA3W)~9f#9byZ+ zA|$U}6iVt1PDu3huZxGsG()`ikLMqkP#025K2!bh8t=*CH<*&xEdvbfEQok;mkISw zv;@Bx1uw-1nv~WFg74`iVYP3bzzq`Eq)f|`Ni{V;JX#8>`UQ%N9@R^Njp|yGmXlr_C=#BQzLPRmeYd#asos8r71&z zGl(_I1TdQ%pFVp!IuwKG>^3Xu^9gW&=n%iB8v&uUGwqZ-U6&PseD306nM9W-LNQmg zBvVpChcLJC(o~AF$fe$Aw`z-yp!Y*z!&R^LiE)+n*{CTx`xq0kivR2{b)+Mc?&+Pi2tln z#&!fDs%6M@(jR&*IyZ09_RHhv_Ii%$o0oY<6gVo*-#frC5)tx1z}7d6 zjHs%bqxo!BMMX@Kj#JY5B7RC?m~c6zO}Mq)`o&VpYbsH_GsPbV94>;9aW_BV=Ey`a z;@=1#WpN78w8Qnw)zuX^S$}GwS0<;mq<~~8*)9cL3p zG#v6qQokk3L#qOm7MytF#qtp@s5B_m>}~odH17h#2^7#q^5ecmBN?~a)Bm^%4MQQ* z@KFYu&)gh5F&_**J>}XPL9p0QI2r)VNql_#+sn0EIrc3^RW9gz1DnUTL6@)V4}xbe zEsH+(pL0JvH^n$#JD40M4kgohJ<7=C(Dm2@mt(mI$hCBp*@qmh;M*i9V@~p95oeQ> zUV%nK%E}2UlggS292q)DOL)=7x?1~<@7JL0+?*92OZk|uF$K0+DAc?zJ%D~D>ZK&3 zB;O|J8F&KJtu<_TDgbQg@R-@DXmdHcn0ymx@=uMAcLF)gCY*kd(ExCKzH$VO9E)p| zH$bI*f`Cmf&WF7Hw#d{bH@7sb*`4Gm$Z8FXdudqpj#ir|8zxZeZ%d<0k*}T1#F2Z2Is@3c)xbZg4{2+(r@kxFpT@0mr!#5hzS&K*7h377WE#D3ER zn;48Rk(P!`JYx+5A!i)ss3LHA-`FrSW8*#e)T~?dv)LyIC~5xyTcDb!g}GO^8x zLS|udU)K8NidJ~NGp>179m_M_nqrR^73PHfMStFQ;cRg=3=_^i&Dti(7&N2 zb@Ur^@C&Yn6Rrv1VX`iopIzQ=}2``ZfppZNfJzm6@k(**jK4=PUW2zDc9TK;>H1=*4`X zjnzQ*?5f0!z>%cGekl+B`i8k`t>i&t@J|I}#xRsctiVU?)?ATe_AK=D5G=9kgYWOF z{80iifGj)^+Kx?18bK+oXd(!Jj9vqbn>euY^f=6O#?OB%utO2_z}}KoUf;+tn;wq- zKo*s%v6o6gh-Y!mK;I&am#L}ziqi_$!wo{X&e_;$^=+;sbrl%3qL3S-3{fjRW?2w> zf#*&_=7*UEa_LuS*gRt&0397jvNNjdt2>#h_^|H%GXFRLNL_Czp+d4JH+m_YzNH$3 zUlzzgX`#LI_X8$;noQJrmq&1#aCY`7IN{!0AA}g(Y>^jPi$P{uA!rnVAZ0Sel4`^J zfvb~^S%goRC?e0TzpJi_BRJ^=9{?i{P2UeRaSI1^m^M8SLvccAs; zS~;3T`KekCRJaFOk4+=B@H1k?Pd0JmfO za7;VXr{8l|eL{I#I#c(R*^@uYooIgk_eQSm$AGUHxM(XrbU%EeQPD=}1fUCtKh1w{ zZEngr?b7ML{oL+i0u-mt?|P%HY>&}~5EWj=f;8JXa?U)GbninsKhufbqL5$-PG4yL z>)N%a^s*b!F1yz)p78DxLhgtkES9sm|wK_j)aXE8u2LBWrP9%R80`4@mh zXKZpZ3Ay4_JH3EJ1+W=^oXuz0kJR%G?fz<=iJ42_;?md2Ovwzrc$b+WuUFy;`&8Y+g=%$ zMl6;fLnjZ5lm{wJ0#V&eB3N4Jk2GfRPJ-uRML{kW1od`*)AGa{S~I|6-12vBZcyT` z%=YF!%|oasBFYrMaddp0Mm(T&F-vcU*K1|{NGY_L-qsdsB~;8;<*twN;QYRd#crzq z4wtlGTo8s(#_J?3kWE&k`x*xJ^!uLnnc@x)`B68ejr+`<4S@FLiy|x})PBf^@2AoG z^tz>{98h2+Z`gMmW@mKT-^=|}w6RoEV7dahN2`EGY%&N!(zQeu6+k1UWOKq}LT2b~ z_Ra*sV`=FySUw}DGk;5ftpG&RGX=bZd-yavfjzGxOMA|% zZZsks+?3_$cL20jLi}NVfX{fILqH1gCYuS)wr^pE7yZBo3V%{w8VwHDN25yw`_>x5 zN#cF<@W<-^`&jUe*6~XlpGF99K(i|u-;fRl^rgNq(IkeLT|aR#}>4%G7`6h3oa9*_=nKJ(U0peoD0!?}Y-q4mzM)x7umj zW0L20q$hd&#P1iO9OS82G8o*aB99U5M6?z_Bh=Q9j7jQz?}ypZ-yg-gm(D)&n4DRo z*T?7VkwcN|bZ)C~gw?6UTRr1aCb@^y*L2OnASgfXt%8_-D_D z6Iwl%!>E9ua;#q$sF<}%5Up|W`kl$^zsp--n?D{)Ud!9#!p3n5xavV0h~uZ{+j?E7y-ofD zI`^0kT-G6TKMB-NP%y@_LSm$w5%!Dy-FY7d1>|KxS4#@3$H*TIRc1-GXP#!!!L(;h z?nzwZy-4hKn0yBy)tt?J)S5cwN3tZ^p<&9Q%yUoCn}p{RNSiUD*j&_UJMp~Zjgx?p zLrVq~SNP+X5JFI6Yodoa3Y+p$RK)_j(N!#8B5Mn%v{O$Q^czQyMny$gSu4)5)+#-( zKp@1~B3;6UC^YHhAc;Jg`#R(JfB~4gE8ztIqyH7$F2cp7+G8Q8?EV`=1BrG2#v!GX zmKSC6^Tp1sbK}mpH>dqm7hX;?PIa>(t>;5M=DJrs-Kes&79ZXSB}?g!?;6QX0`T$^ zEWQzfwy>3##KVQ}W@D=V$9=rXeHY-8JIV=lgtA#cM6{89+is3Q2l6)T1Pma=H>zp~&2P_5y z;^fMpMo}y{2VcWM(bC}>2j3TJbXXrgB!%w~?dLH$%IkA^Z*V$Z>fNNzw=`clJg$8) zAYS8KDTYafFc#z2CHqWLbaM<fTQS7Yc6;>}k65X#cxEuHN^PC1)SG#YTq0{l8BDD_czo0}N1%7jf+(Z$!fG@>D zw3XKs--t7NnT0jzB9rgT}Ih0bno)uTf$a=tNqO>4bYff>knSj64x(A z-YJ#0jRJ`02--1GPEJlqNpqWtl2b*blt6wFa3EJN5G{}G>gs|yP*i+M(o(7k)XQC( zQyBx!U$!b^xwN-(|JGr$kONnaSAr4Lq>fa1ZFK_72k%nL;?`P*U*W2?xM)V-cvMLc zZK4d$Eg)cgPW6wWbm~XLYP#y8Ra{JiDr#y`8tj?SLwYaGnu5oVA8!{cWAdFNEY%QM zaE!Iyh!YtIXPely7aW$5nVzC=Zm2#EDpoo!Hah4cvBlO^$y zCqTr6na-t{|gH5!RnhyYz?~ZpT1okHZ#`>_xLarMzj2|8XE=*x$DhO zbLcchkl-zlhI)v0dIprYWxBKn(>?yuSfncM^%`7^O=Q1hKH+?#bhNBd6Y{Fzg++*) z-^r{D+4V5u6kjkT{ps%IWqcxarWmZa^XR9-acQgW&exEGX103mTbhQ~0b8K&9%c$Z z78ewIi??{t*C)i#h9?S11CjaHvZau=9j}~$5Y;MWeSc--E+iTQ3=$4ItrmkB3aa8& z>xO+YI}GVOr;N^yV%-4%_XoqsVH507xyJq5hy%P1$YC#9q)ZU6`5 zysSs!r``DX66?_S$s<)*NF*Ta}nR9YH`{`~U5qI5-ZafGQ*-F`O|M zZFw~z|42YkP`+RdPqEuiYddHHuoFjCd6^Ne5xwf+IHz8fYJNO_bGnmnMNX4&#w9Ah z-Xob-815!n7_o)S{Ub^*A`%O5u~BwF0IJ9p1w?xyy&6!{CgG@m($mf+K%pHf->*!3 z%nXXSn4n=ONXdXdXJM1_eVnd4`M@5Yq7(JmV*_#m^@h?(8?g+On$KF%V7>>dNvMli z%DM#VQQ#PwDN)Q}1!#ih1$ZismB_n+tI*@`@8xUmIbzGLqRymJu(Gn&Eg^lT$&P(# zd8;e0iBvqLeeg}S$>b6XoHWZwf)umbTO>u{G(Spb!p_X8si~$WKCS2Mh63>lnB`p3 zy*0=IbZCgGIFU(9p&-W7rniMN6&YTw10GT7OeI=rE7i`7o-olmn0Oufpt-3HhceS2 ztpw*&5b08a2?AZ2yh$$c_rkqie!az6Qbg=NY_p1S)vu+Qm9<#q+P+ACI4`K(y88VSo@Ram4FNPaqOF zgC|czbWSndc_g+E(#Tr$as1NE;Mw_Yvv>;nq~7h6ewqq<{mG+-gzfbaKg#&0T)k{{ z2?&B7hHcm|S8AQz#3m-&YpG;XEWTkrRXXSoo%5l}AIwJetxUQyb$T^6%2BAwwlDHD ziHo^3pYE@nGFOxIl$IgI@Cv9~38RrAhXXokr!}%IH^Y4IBR5x?lGJmRvvzUWClR{cE zBLf5Cz48~Ig&L$d9tL{sk-A9bw%yv`Nw?wPbMuU}i4o#x}y=N@Y&79@*MJe3wkFIPjD27RlUwJJKYnbLAJVlp8^!Cp6H4atg14dF~FKBDN6tYfBtRb&?2>!y0)Q zo1uj9j2;?V8+wI}Pp`axUkc?TV%h-FM+s&cEIxN1X*q9^qY6ClMbL1aOUA}ErP!UU^=6QNePrf3xWdT?+mzGTG9KgSEAz&zvv~n-5J+FjJFM#m%a>l=KYYr z_O!O(nhg_9TmL`*SB5p(cPt2%sEc25_sbkay-+ygcbKoRB%bZ>FNvxV zP2n8X=tFEhxu{n^7OomygDQP~{;B_{^--CSa#^##PU8m7_}v52;+HCaYTa$}0Pi*x zyue^hl(V1>eM*>dhYzk#z|#$8&@Y#AdEGzMaz%ky>r1rR3-8K7D{IPv;Txl^r2%MH z%Mx*i*2b$mShT&Y-J`7FvPBt|8$ueF{n8^1MFV( zGfG=w%Z<9JAv`}{F~=ahX!gmGQK80aVmIqs_LZhRG5xL(RS;2bCw|t4NX(dT^u3Y|a z$E+W|@q6Qu7A1aS6@^PknLYd)VYo9Uk-1{UC-_}$=RQsE_6VC8;-q&IARl*6zj9dl z)1620sl1*8_6>#Y2tV@3mMz-S3vtA|s3yC2AxdN;S@9#5xvjplDb6es_7)A1QthnK zeMe3zy*=p{Z}BNPHG`35iT&U@@5PYBvSdrISPp3B(_7T0e4aO>n0_=CWDkp_Uhs;h zNr%$+@qOf+H6%}GguhfGP19vwpSKxNe0HxQ`N5?Hs&$Q>-2Cveie~WcmIPa)UQwHu z{SiCW-&J74HCX&=pT^5^BD-b;ER`HY@+I`&k7)G9`tMg<4EmZk>sJ7BwIoZi4H#MkQ%JaCzysfD=4Xoi zo)WZozufQ8$+9FXD}I6vZ)yhmy@FPhh^n9(;{!6i48y{Tmk;cYlg&{&K{^t26al>7 zxbfvCQj1NH9#nt)7$2O$5<^M&b0mU9Uw(XLZociZh{Xrh5mtuD0pWf|fOsDzJJSdS z5Ezs{4f+i)$Fm8kslNcw5;#FB%E`r+dNdB)JYNBF^MA6&1+mNAOcMYR@7|mR0Ff}f z3KjC|Jij{st##8eaILnTtMV*pnw)Wi6+**j0R?zac?D@Rv(1;iMG8LW|>CtiMCa(V?A$%eF`H$X6GWW(ULwUce`y4u?{EU@){HN^_t_1NFojOrP31t~`m8`UMAsgt z)s%dc3|S#GAwC|wbJ>2PPoTQKP^*VE0|@FdCO=+cH1+t<4SgMA(R4D-rKvv9rzj$vyB3cJUq!MvyW`PqAPfvMyf8yk1VYvx-I3PNya>L9Xfna58;EVyV@KEtquhJVC$e)0Izv} zzB0OPz{kSJrXT=+iCi7O(0@`KFSm4m(Hw2G8d*LxEW>1|4dGE>w4TIOczHRK7@`g_ z+*jYAmavK`vN`D?_JRr+qhhkAeyQ`TrPn_nTL3=3y$0JZ#73>MAEAJN#@Z{K8Qyda zyjkG5p^SV%n_4xU@R-mI&uhpt^1o5mwb~8#KaXG<+Bi>vJ}wd-JFp#lPm2(0va-_j z?BY-LMvNi23x8DR(D`2I#lsoslPSU+L2$!#1J)n>dBp7JiUB6!uAzer+w$^3?kWifpkP zysV?JY+#_S?uD!WZwT?B!SONu{2T7GJQ65szoDh_390gX!vRYq(U!-FPW+k+$u#1|!KBTg5)G7cnBc|1TE>cNGh{xw^T&l<&{~TSlXLS6;x_Um%9aNcVAr0M zDco~}@+B5N<3_DDz8Jixio;;8uB%HlQRKH72b5liOJe?jb)D%*PTcrP#G2`f6$%Po zM7bU}=d&T33+G@0*aAQUr|P2zpG;+q_(0FC`=MkBKsDv!ZjVpSu63 zz@k1HY%GbJGwU?FJ@2+%I0y|~x{-ZA6g1yB~Wh{4gzO#^s-v;j=MI|@F3?Ja686Z~Yq9kORfHp*sC z6afmj#XD%UfpkYcaDIS;gTuh}N#3~Nj3q<}7lLQOAS84G8s#3no|98jLU(1k*aK2N z(Zk&Q-@f!bT+#r*_w7me6e+8yd^dm1mmP(&D2A*WI;dyxBr`Kp9=8hy8g#>{0JXqf z#QM>p9v3~Peg2L93?K)i)|#!s=dDmgXBtG1$pApNXGpeob_N}n23No>VWo^%0C|ta zh2RHMrX(a5zw!Kl>m9w{M21ZBf9owlX(jrQj3fd&{AooD9{g&lHJ; zI8>0xTC+7aY}7o$?mhqm()|5~83fJ;`3*&^NArue!p3|B=ZrM@&t(EU0H5XQL5Nw% z^&cQ_lh}WFj(PU~(Dl{DY943(`oZq%=rNsG=NUu-Yoe~;vzsdUG{fHlaQOhsm?H=EBaksCG`Gf2 zxbBPn%6HJFT5n_rh8dQNepsASMf^;tcuV@x8|t%uowsocmv##+Ad{+FZ%QQTe{O~Z zuAC`_Zl*&(^@asp(Mw|E3m^CFZJ`Ve3z#?H1e%zaqF5w#O)(T(Z9eoZ3FXLurjTkY zdjsH<5aH!iK&4m*pH$lO_2cBfU=Hl9jQjOUN@vM)gM9Mi6sRkw2&C@<#D_is&^-N7 zN9t+SLiX>;U+7H!ZNdz8Oa07NhxQ3FhCc1uc1{cc6aca&34_2H!e}#u%mAjIga^m3 z0MOu_?I&;=Ks+=BB>ni0f-uyK8PB~2h-U3ZK zXOcry@FJQm5ARWsW1VvBd`hNEA99)rE#wz$0>SsgASWkSc&VM({5kMaBm~`{qc`cU zIu5-LdA&lX0c~3j9%xdzn*qu?wm*58r{ z{m3J{3s|F9S=^J2FMHVqd#~^hxfT>Bu3l9kou65bGcF^C4krf1OSvZ1QG?DjH3=7Z zGP81xN=DN3mC|>xEu;(;y4ZpbNwtMI@b>`532yBx&*+38E$Tj`2hW4a1onm>h-)C1 zxw<~6h7t$;xIsHl`4)nAiiPB1*8;~yl6?+bswn&6oF0e;oIH}pFIuxfCQF1Zv9ci; zOW_w)I6VL1D^2*eLOrN%sP2;Am3?iD3k10zA29jpm#U!)5)qEVAmKCU9s<|9d^;ji z8$+FCjZ~4WZho$ss@WB-fSM2Ip7mIqP}F+{@zlF(q_g!glE+=1P#-6!wChoQlvaLP zZs|lG+>^EV?Q0q<1ITmaJ}?h6ooOnCO%d{0ZQQ17{!@}SboOg)^~%4eB*lvbTBRR2 zft?X2aF#yXk8A-aE-;FGLcz`wwTDE+P?;zb8uQPKVbg0qoX$gi+Vw|4vBOd)iR)b5ih*_vH(Yw_fG$QN)An3CPn6&097Xh7AY>U@2B?FVaA-zmO95t)kSVOHaE`fyS+K*FKgsL(+ai-;R;iA zoAdB-8_?S^YKrOpU443WzL%f4Es%8BM`#!>X%TimupzYcZ~@o0qZ(BfDRr%=gyNp% zk=47g{6N@3(uazG(vP^T&w|d!f%UydgfQl<@+f<>!O=Ig^hwx!P6WX{=NEg1+!w}l z51sfY4q3&A0hc&IwBZG{r0>r?M1%2F{tZPmVc3-Wg$lz^9H(!7!2NH-2;eKx5rWU7 zTUP$_EdkF!@ADx>9}(U^#?(jyGRE{#c+4`4z)q+|ITd9@Jcqbn??1n@()E9TD|;B6 zn7L_E<39HVPkm!9H#UvAd660sXu}PmDe9yDJorlTt*S@18u1?2CzhwBlB@GUzf0Qd z9+oP~pJW(+M*7!k7sDNVu$y~R|Jc>-2b(|MZFaM^x*T%Rrx^MV@mgk7<%MsRloC~@ z=6e!-=8)$xD0yFH7C^|P4*(=uhLIoP27*=ty~8Trp+*0!iQhnqPqjhlvNre93am6|cli zR3sy&bfP=Vw6C|Ya@sk_*#GbE1S^&;Qn?4Tx-@#8(Be%hyvFKFsm|_I7c4f58 zRX$+t1ktGV~;%{1kYlQ`XHc28B__QI5u z?w-71+xIxvn(R^b<%*d9^20-HQPdo;J7DY8>Jfrp#|aC}s&2wxpE^pYXxC2NtjEDG z8i1bZeUcnR6tA`oenQzdSbDc0L)%@a{3nl6??kS}R_wJ7#B(z50NT)3MTWBwiljA!Xrcj)?>>T7yV2*MK zRacm=_5M+2;idaPe0_Jh?1B-`pglmWQIg@CDJDbRC+ES!1v6cWS;0*i~Tt=dZtg-v786-e-F; z0omA68cJ2gnb6*R;1Qq#)s0mMe%i~x#MEkUUHxD6jNcOXbN2mpOI<#`{MX_jzCXozHS7+G0tUNG0b+goS{J=f^o9u{a%GkmmKLCh4kNl3!FzqoRvfSKk>1#vhu z7yueWc{O*IiZbt9td@MRJqCux6OfLGUzf36B~EInNl?bW_rIv2a-i}D5EFssZy_6G zuqE;Dva9)GO_;JQ4$ul$N0`k1TT*o-EC=vgeeY06d}s0D`A>eM9+I-|1v0iUh5Z`+ z10C9=9Po15_FU$=A3k8`Ns4@j+^)O$?g~{Uj zS`fJegg?x7Ilq`{0zKN%&%lY$+t}}a_H~DWydM?P;hSZCdF;||D4)^xv|yZhhiOU$ zdgTMemrkX)FqgQoVxL1KQxt(RUzFUOL~K;0k?f@Utsg+ds zrWEkWfdZO79UT{Wo*$~$c^cq-PvDRFV^>LR&!Np$9zaI^yjC|nqbaD|yhG{TB<%&v zhgzFF=F89QzeN9M@m@YM(o$Hx7%-686&MLne{e{cv_0NhzWg!aFrXEljQL-4AcBE5 z&hsCG07?Q_D~5`{leW#g28?kkId2^yr1q66ZQbQdK|dxmEz(O$`h8u<{Oo2J4S?HipoCsJCjb~ z?sZ_1Ya$c*92*Va+l=TAYUc6@iqh>T6YT#kau$;=u*6{}Ml^lb5)nhW zifs{3dCzu+pCtl=gYWt_0lMMvyk@EV$sFqG&I}<%FA*B}pYgN~Sky)&4PF29x`Ak4 zImZWEs|B8xIJMfMca350??1pC1em!%n}tmSECh*7%6ZZ^{|Ff(rjeR8b}bf3Z!UBA z-|8iaa$EAq0WLillvh#?rpM6}GKJquxL-@~EFX}_w zceSWKo^d+mTJ<+W|GGNFsQ2FXG#WQ^g{bX2ZPynK=2?IL^%M^Fr+i-|P`e*8{IT)Z zs5M{zGcfD`BrWViwDf26^O{{37x*~d+NOH2SWpZ4pMPNn_4pBGXd11JB__N7x?HeH znrw!Tjjej9S!Aq@aV|FL0D9SAaaY+mYUYx76L~l6*{t*+2TS@?RkG-q4V+5&J;zbD zr9A^=7OfOZ|2&jM{zB~ie7&fGCM7X_O0k>cP}Xh#_%Tu*U+@=LK7hdmbMqk2C+(%~ zhf1Q^rQ4`RUp_2+b++BJ8<+{Y_#AXoY>A&RJ4v$eDIw_iEn-d(`( z`6U8EnU{Vy&*c1yBT^Trljh?Zq;0vGS5F(}QY(Izo%Ms}%Kto?G?!h;gyxKf+SKuB z15dcu`co)R-tVU)5o^~uiazfnnGuNfAF{#T1MeenD0VLS=RbbRSi6@lSEy+n6G}Ix zm5}i^DR*%~_&>wFLL-uncw-;#s=h1tVH7(eOo3pR_cVpE1H;&rybV8aX*4&|Rw=?+8VCL!HN0hnRBiFK^Q_LzX2^!H8GS3^Z&NbnhVK7SN)Po};B zl#yYXm5!~Uyw}bpE&+NIuPIFlItUu0qi}WJnqWy+yea|1RX4rG{8H<;pLN< z?g^E!tl)Xb!o?N&?`snA05-5}VQ$@{g1-h5+cHvS#Zwu77%ip%jnpJ3XqkY>Hd`oDez_!v${J`4p}SvIw%>KQQ3lfiV6foFiXLoM&SD8;u(GFegeD69cQ?fQJuQJjRs}^RCcjd^ z8*cJg5Pv#cGJsLz>7V_%hh0wT!vZy~A8&``es|a}<|*ZUyT}QvzxCD>z~hveJ9;G* z8Q$Z&q{jZDT~X#d5g$o1`|x?7;!;X57rkh9KC8sgr6#=xYa9N%vcr*9d89_r zJT4=diN8Dgr=EU4Fv01)8rFH;6{m6f#wfbrkX!#S38l(pW5D6~H`n8vJ1Kh%OL=ta_*@ zFB~(AD)MCEKeNaj*a+Bu-_IAZ$A=%PzU|eLoQ<>=c_#6=2*q`NV*axO ze77m@S*Sh(`np2{40Q3&!9}_QWKH(76YDq2cfVLhrTM$(9UTUnFYAZ%sRt*9S-g)9 zSJ1uO3mLuoz2TqzeK$pulyyhN|5o4Z5XF&<`oNLn6RHYYN|xvjsl?$UMh-u-OyFiu zm@@BM=T?MRIEL#!pp;yv7avrnvfl@gvxV@tEh}u_%Azp9a_R*XJ+<*Pz{kbU_CZsO zv$wn2cU+Jj@iw$@1oq*%{YLC2#CbMgWiN+ zJWxL>S=b4|Th#hAiZK4*;74DcW6_PJ7(tdHOKpsXt9?z4YC#jFgiTC4(1-Nv^bQIk zSH8WR-a2O0?8k1e3;t>V-73|%AS$oW_jaNP(bm>xNKvIi%v6~d2Zw-W>G6D>V~8de zY|jLC;bpt*E+Z%?DA0z9BA^e0K)_Jk4nI5nou*CHRg{b1XtY+mf?&y2eL8NWTqJ4d zl~T|jzHf$)vwJE@u0=LD<-989?;0_<)CX6 z!S5EE+Wu2o((60liw$hdrM+1vYr^w(Qwk}7RN8)1GqpDS5*$PNMZ+Ue9S;q7%J7|F z^o(cG9bj#qji-Su{hE@M9eP}KsGc}OqJC#$mFZ-{V5|tXryIx)(12~_w2?y!-WeU zNE{pQLCGu7U3!z!&>Q4h{!NaEY1*~h!l zS%|qEy-PCQhat&)*$sZrwn}GIx4dOlpieroV^4-o9~tK;2LIH7z5zVAzw1^Q8y@3o z>-=p%b@8C%Ii^3I{cg`m(~Kpi#~yQbcnmxiK@SoJoo=2UqP5yUVR3%#|P%R-52$zFUob`z#-Y&EXL9xNz!p}T_ z;#kMmZ#H)l-5D&Leq>{dpTNJG^sZ4+#J$aTOYC$*Tj+9jr9Jlh-NN!^%O<}5@+_xV z16MmBJv{+j$NLPy9+S~5kg#rRKGtJuCGN6Gkhvpr4iyZ3!+Pfv7gwRU^Z?|)Z_<6J z@I%sW9%_IFF&K$rG^h%Z$W2hFvZlGYMBKi}oF+4_S~7?#N2uVE;xImDoc>QPU@o~K zcA|hQ0Tz>$X*8m<-w|)Z-M?YL^?r6@J+S+H+ceL-H+hHvgg46pq7JlA%`>-Ks_*KS z>2HnRa}0p*^9uS>CoKj0<$11Z+MQq_fU6pi!}@3J$tzV*Ov`^xWq|vFdBjSiN-&BF1&AsQAph{f?!D{i;QN840p}V4 z%n<^54F&k@HO>oDyf!_^bBceR_xnltf7}ZJHW~t0ivE}p&(Qt}U+&`o@BfN?Kwee2 z0sdDt)iX&D5&aXhz%Hbikpsf2TRg#cP~HEk76v@Fp$qKpNenEQg0BDnk`UhD&d^Up zNdLLo+Ii@Q`t}*xe}CD_r~UN*b9r_|1#DS5Igw$&4ntLiLAMNA94g*@;YkhFvwgV< z{+H(jps1bV=>Dz@+(<~hnb+}@KhY)yx7Eg85RH3$$pRnu@Mi>cCnFGl?H+zz(FO1k zFuhCN$`&BRfdy3a?E5qE9yW&{Lq-cCjDr>W=OEaCWrOY0uV^+wi7u#xa6!6`?lLzPpizx;TAdJ}RD2t~&O zORY~v6BR=`Uc&gAclzB{5{~fv1$PvS21CWp%b6r7C`uQ*BHpW(?+FG|8IvGE4}t?f zGIaqN;&^P3syAhq>`{e*t3WmDCb-()pROdG>gf^1V+7YjpEJOGj>nQu8<=885wQGl z4@V7~QPuSD3@u|%AfQZp=fS_RI2Twg#2+K()BfW{%D{~?*4($>u_LLnSEpfW#BJ#h z`5&&38`Ac6lL3f4BD_B8Ve^%{nK@q1+(#-@hW+Omw0#>621(6E!&Jj%p9QBLcLBr^ z`GSA8${gIfN1H@Cl{(ttrNmLV*gXf$glNYo`3aV?^myM<*oV?uOWI32H!=lGaBTwU z?ET5k;EFpKpx9uIqJn~c;Noqz4HUUxxvTI@-W44lW2hU8rY5Ky`rHXP=D;sS!NSF0`P11~%8Chc3@>w?BVQNFLT;a*@n?O) z<>7wuKKz~y7XhF7xM(Pt#QT1ALrf?@`lz{faPBxmBI9Pa{rqn1W*JB?Kk`-@KCD_O z(Ib&au0u=+wbb<@3uPniDyHGM&iXz50QlGmp%Xp0^^0gyD6^4)8j=Oz(JC;Ld@;8^ zXFTsweTgxf#IyN9-v@5TUy>@|uZJg= z88TB0!h5@UJfCLftC*LGMtbRtBr0O6tphUGd9O&JDoY)=iM0%|eIQPv{XGlsen$bu zfxG`^#E!#t>tJ%h$Cq>hZRlL7i2n~v8yXQ2i|!-q2NaRSUQn8xVTQG2CZpg*z-gN> z!Z9NDn?*C0u{X>0GJ+Y2FLD#0gQ4-L4KevkSPh(c^~dcpa9KXM3udMAO)M1Aq_55Q z%I&-mMvnvfd#o%KR6}(deW2Dzy0AM4TQFSBAvTCrw^2UzYlU{wS*UYKwJbERbxq7y z2C5vGS}&uaq2b`bdfMad(?EImcUCm?)O0jL+jV3j$JNoCtViaQmGcj*u-MkuJo7+*dv{r((w;OHT)K9i&~Vp`#A z6}rABlVX0WytpYtR7c%^UN~ZSfv^?wIiE1+@)ZH^3pEHsJxpzSm9HkJjG4AK& zyspCPK+&m07S%fYtAZ7yc)~DMdtsm!jj2nEAnoo z1d3^tpA(kY{knb=Z>(%j==~RE7Xiq7y)c-a-PE2!PAB(AuIFtdjd4roE@zMb2;y(G zD+_yB=Z{-R@w^ny03Ev$@3Vl#wg+4VCs11Rlb;k76;%xhR^GKbSCeB^6DdIjiFw*b zQyTOlkwR4t-d5GCmLy(*DPU3bmCOej0NZ0e)uri@uI+jMhMk;*E~x)$yB6dL(r_g| zvNu7t!RsB2XTnreR6t1tpSGaYZgiNwPzDaUG6Zg%!Vc3D$HEKRFH-8;6LJzY!|L%!dv~v*MmG3`a_l?lCIr{v>1CM6mDCP7+I6d>=hbqC+7#|e+Bk{5E zoywdkj`#?{1z|QHX>c6$`MrS~lLBJqZD;egph^p$m~?J#4wS)=@_VrRMD>Y=)DPu} zO~=j5J$KzyvXP`$kGVGVy9mLqqcHw7S`G7~Y>R*Q{ql)_)0rousM&A=*ZLV-;=uPNF^huzIfcB2 zt%B!#9rCF&Of?sD0 zptYzk9Fl9gqGkhFW)g8i+^gO1 z7K9Z0`Gp-Nx)T*QhB#+aPB4wX+74wh`Kz?cKfrx6CQ=yh2WB<0|z9oZaZy`za?!!sOe8kcp^sN%5n%BemNZva=0X9(a? zK|A4}%}Io@6itH&=kOy!L!YBbDauJjLxzplWYWkLV?ckTSdwwm5Z5R58EGvGby(r0uR#Mk^`@y+rGHUfq@;Nm1PnmerNXz!Nty*OH`YP!L11yCn3mp)u z!M#nVFhP>v>hGgVu`gthdk9x%IHlxEBLueg<-;fDLjk34hn!Pd;IbvfO~#Mp>dE zneml&Ae_=oVyG{Bt^iW;l-j6T=XVj~h%dQtVzN0!YQ7CQHVkr*3cYtp5rVe8WNacq z&rJK3WK>jC%*>wRC&Ez*`kWtdmISoJ37xM5oe0{ROTG~}g@PaepF{XzTa=|v(TgLG zv8>grc)?|VMK2XWJi;(I|9zpjqMGdx`d|u;A&#hK)!Rg5r1upe?Uq*t)pdv#%kso4 zvvkxXuVbDUZUyG+zSN9|Qzd9{bv;f2feRVI%5Vq;;`z-6w%E%p2_rP@OPY%4WO!+3 zk?hm=P1u>e-eWS9e|QO+o4+3!$t^#rBQ)Yzw^e0Ve||A4#8574m^wj0&t&^L6wz$y zb`u<;PCK^J0N8j4D#!{RemILJ26F7To`s-#8kF0;diClZr#%VTSA}eT9zH(Otz-t( z_%TpT7^2(s(S1@r;^q{C*AK^t$nDLx6=(j|Y(Ry-l#XGnWbUhjCKScBbQF`Oq6KvJ z_YceorotbF2F-Ql!O0~#s&(&Xmcr|;tJ(;GFh-$(Fu1oM%7HHoso6OX1rkb=Gv@7X zqo*98#SPrQaJ)anD(jkwgPZ6+pu3?xHXh7b=yB20mf#}hKJ1J#;>IG2;a6tRAsjCa zVN);q_s;9BgTrFLYF&}C_@M0&}U^?wRn;F5Y{r0PG##be1XT8M{Icz*MAJ}y= zhr!BYkZ%bNVW2|<#81YqRaNg}hK3v_C06ux+BG>4M`EEnt&7sZW9ahyQC!s03BM{b z*^>+PCWn`I=Qn}AP`^m=N%!F{0-(qb=f}(Y@ke4up`zM2rRO$6HljAWZt@{>EpZzOzd2?-f}?H=D?Fz@hoSKmPdye4ODjN1_~tN!jqpmfvUh5l9$^{ z?N_iOzalk_p?XlNK8xT}xK^OlLb(gM>J9(^OvT$6uB8W*>axg~B*E+4XTrLrSZ4mT zpps9N*Z%G*G4xcT<=>by$3+q!}LCqO8yW)kXIB|r+X{;IRN&+7a zi-za;AZ3yu;3(wIV^i6tgdh$v3X^^d*c}(YoH{~wsfLnwoH7x%wzO=Q$)bbxNF5G; zZ8Kxb7zf)NE})v+3lF|RjXQ%8y@-{udl*0DxX5m>xcBCv3Uu!^eOK^JhO{BC;J^#` zXbr$+RKQ>Ol7Ilh_&Dp&ZqM|(@5crUm~9JGd@*Iv*u~Y*oPZWoRwXncF)(p6QPCtq zr|@u6tAWyM4(Lgj0Z8amL%^Cg;*alrezYn#@yh^GkTU$^XSrIn!z&1SP4N9o zzh5GfMP8`$Ve#~DQE}1P!@(NTfGv@U7C0f|`5kabk}j41==ltWslI zF&`1nBY;r&8>-5~`+EKH>{z&Ka;I2PR@0}9D+xbEAZ_c2JfV1ytAeyEJvo6`N`-`D zBOL!eJl7N(_qW?%d->p<%f!?4yx^;oh6yF~aD}uf`yWh*lliDdYi!M_meBBV7 zx=17!u(To=gW8<>1?FkFoBfwN6wkJPBc38ZZ~c}a(_v-e$bx{fYv@slK*8%{c6W=g zHJ>EN-K?poPC?00JKh+eJK-1Jh<|u_iri#lV}b8XcL%P3zJq zXS%yT$QYB`=&py4{BSUVDhOFfz-VFxf=0Pmu9@T4PEqt#`-bmT&NJIz6+CSGI*QuE znXh&~*mpq?B%foY=SFjq7;>n3BV&1b&E^(|Vi27t52K84xiOi0xb;_9z8ARqTo{qH zOears>!Bz;jk$vt3m$8HtX_jqKi~{-8RDcgHA|nd+y62*={U(=v7j8 zH#H7@?T&{Rau_+g$mpxN%Jlus6xZ6%n{$C)&__?iM8%$jVpGc82poViYU<%Xdg%avwZfxVuWaC8 zU~AI)cQM5wPzX8rXq~5rKTDwIdAtH_5IsFTD9C5S3;^1flFtk)Zr>(_^S;^U=QEu~ zkDJ!~_WN4}6$)2%epfAJ=cRN1WAj9sf-FZtKRP5qtUVI{O)j*}xPuKiZOI=zmx^=( zY^kYuE?um+r1qrqx5O%bx9fs46_5x*w+01XqxVvAPyugo7rLCXvu_n7J{h04=F)w( z(+6L>e>Xquz^F8tuHaD=60lnZ;Ll&U3(~JdJw=Rax1zUowVNNVyOR@pF!l5EoY( ze1T+$;9-{-*~J{8Eq9mv?xd17YGzyH;1c;n*6kN_v0`TL_1&-HXsXv^>ue*^ThO7w z(_{bPu4igg`jJzVd=@5A-=U5?E3)lunUp{1(6mC1Nj#6bc?->lATy2=!cR$e4yKAm z{O`Os!0H33rGoTdqq_3h)JxftFy56mDye?kapbLu4LeE+V<3FBfX@}BZ5jdLFQz9$ zW;5M0e0Lk@Ijs-&$+Di+PX1lc-4M1-ub+iOs&rC{_^1%bbIUPiM#dk9CGkpFZI_q2 zy`gby?2~z>#Whm%!?LrRQzki#F;&#DR9x~w(~Hq+Xd`xbGGkg>P|y)U)G>7)xOVq6 zcB^e8Y8nE2O*-vJ_FH2gN-GW7BF}jYX1{$LhbW?P@oRo7lu=EvsJPAxqj=wi$+hTl zwb#Jlv|!)l+i*r|nh(MGR2ZF-QgdWp^-)5eV7OY-7VQAAp7DE+c~)2J)W2PbOCJJhoCadwT|) zo=)b84bbK6a3iA=+1fz-;0NE$QA|{uVZPKXB?9^!rFikJc zCK0`X*5PofmJYk843(gOTz^kJ8Y&B8wJX}lSa4QtU6hfF9W!g=!rw);7}#-HRxV*D zEDWfEC|32P&uE8a_9w`nW}+4 zzv)x9mB6UEVY!6pc8)N6uG%e!E=@A0IErMOC7W*?W=ayF!9-CPTz}s{+~&0CCuGD$ z*DtStGA9{@xevRGtJS$>t*!L2^V4T`vkwwG(eYkNC-pjTd`w-Be;bX@UR)Mj=i0TX zIar=;Ga?W0f1=1J7)=rs+xWE_}De}F6s(&McRCA zPE|c>kHI{OoY&d404*&Z~D@_GCuBit7b4HMI#d3l`Z4 z9v$E%$505Gd=0GP8YWSrWz*tm{3W4~xS#RFQ+m`mP&K93VVkR&Vqcfc8z)KLuHF3p zGSv%xpV6dL4`i_7scskyBmo4ZUy66O~+LG^O;!cs1xPR|ejn9Zj zr=56GQ*P< zQY~=S&Gp??Rm;OHP(f;AE+7ac43my)}f)gs-D?8G+*TgssIn?#b{EY1TywF@M}R*(HN*v zM0|I*cS+{HgC5nDa45U1B{^%^eP6v!9(wY}rG18AdsXlmdUwYOa@$`LE!P&?yvvMR9{N*a9zDDk!#p6r`+@WjAEnJqCd4H> zruo+E%~M@TgDb;v`{`_A?9fUNW6hloo)u61yEtDb;HoI>E%;5XF?_|IP*>!%u){k= z*bS0-IDViZr#(RuUcpa4q20|3k)Gn1J73OK%ft-_%>p`;x!WWY}B-2>?|V29q8e#pd4rHuEf$Z|8Vstotr`DawQ_^ecNnr zspBIO)|F#9ErVOUUrWO7yVyE~(qr<^vclf&r#(kjM)osG^2QmnNQ?O1h(T7fvx?P6 z@SERY2OMW8is{^*J)bs*OvTh?u!xw9KH4RZrG(bj561DS;OOhGIWUB>@eGUIS*L#{ zK8HxNI)IV6AqcKPg8~_|_B3{9F9_%K_h#(bn1|xVxf#gQtAXo9D-6kO&_>`@^tHG= zM@=dQ9_4|EWoJJdK6^;YSPFT~?YW;qK9O@1Uavl)c>&Q>YjVJERh*29X*b+fdtz=Q zLeonco9v^Pe5dOdIR+jd5NN%R^6r>7SI^1m4xRyjH*d9?s!~sTHJHnG4rz2ynOxzL z=9xg}K=da9Q(V&MY2OolgW*2JBD&woxF|iGV4I)TAXs=GO@1sRqI_AqFa|-fLs=}A zUTEht`t+c8H?7E0kv}?(&uBD#)^cHDo5jjvwFix^ZCHiPjnB<*xibRk-Fw=`SXnzR z5w^v9zrhL$UQ|XJLA^o)NVhpL-{u4 zq^4&-!*vdm>%liV3;jf39X>f^{xdoF~V0c8b8q@P)grzt2(p!d^A?4(UxNI6uU8B@Ew+eG?QCE=>-#NR5@Y#*3TK7K! zhHb49G50UG8q2uuO4*-W%d#pejD@r`5gu`?q9}U6U5z4S`?a=KSXel5_xZE@PE&4x z+#w^%F5J}f7+*I1EyD%H7yNI&J$`C@mDiSu(x?2`YH?ky?)~cJ=W8`!*1_<{;Zb&- zW65|DGi}d5!;q**KLBnt5XtHDp4jD#Y^5Zo@+Z^gpHvl~=ovJ-+ghZwX^G>W?~lim zLBCHO{qAfc%IwNc(Q_=AK(rPjA8tJ@4*6KW37)tw`~Jb?Qac_D&!PD3`TN&iNw{5tE^v}z1 zMj+vWL}-a6Av!BO)-1VNxdsazT`_1|y#um7Kl-91DnK`B`j?d4dHdrRjg$|Zr;}$Q z)%nW>!bZpsLBcx>n^M8_Bfv@#u(0|Ylg`G<`Gtz>yZHF{7#Xc(02+HTQib~uK%t{H zqm#2*u}0WUy_I3cAYogy>77Kqqa|oyFnrp%U{wEbK;-8MTc3ejSIjb&bxr;6C$$Jg z-l`;?hL__8AL=WTd)91=5G*!^my&M@(+mi{41dLCxGH@R0wCZ;2&oBrmHJ>iBNoAT z++OZ5*}v3v`V48ZopETgDU*kg)$o<57Gu3h#{;koPLzt3xPis5-|OHTZ?v>LpPaOL zQpat)Zb9e(nhYSULqkW0_1EV^`D)jorMSgOs%M?x7K4dR#mB_N1YEfC=^s~5K|XBf z!?oC>*)4axdp_YG+5;nqIkW=^z==QT#8$)ZbGe1#0MdVy#?jJ;a*&Sbhc;@gr_Zsp zpPfpamHttb)2rj*+xIz&Ja8cip=G`k+}LcPj`X~~SgT3@8rL~8=f{5c9dmAUgVnaV z&bHm1uSVHpf1FL4ZlHfGrQO2sHEUQA!h7iqy3>5Y@)dX_t4vgL#*1Cwqh-}}`7J0< zV$_1UNfT2LXHYCl6CsZ&g|u=yHmEW1M{`+F=5P5tF)^_|lJ@0@uA!?c?M}5^8RI7B zr7gkBXHAbo0FGi{Pr4K@l?5GT*RzDYbetdDLy3rV0|6lM`!$ity8@Q^YbmjCdeY6w zpU0-!C|HPyC{gESJq=jrK)OBO01!h%L4W;eoY1Y+{2jcm(>E5Y?S!quqF6p|?(~lq zk$J3{e6DD!j&f-NAuYJY{2*2RG8I>o+nQrLHU%r)Sq10M7-&If7#p`;^12ZPwa?P(T7T(+*|7GZJ{B9{M{?^!=>iRLieJQ6bbIrre>Diwh`7885H@1wVDed z!rJe!ZO)5w4O-l%ZdKZj;E3Yw2}wy8sttvB=NH|QrRLl)`kZu~(4Dn-bsYH00-ROK zV(g_#KJu|*Wrfu{dU$vM;WTy>JLE0s4C$(LKP$?vee*QLAJAT1lVxRPOA3m+-nlO^ z;6Mdt*P&b7Opj{O=o4Et4U*q+OWgh*7g{r6;W3ijf|fGV)as?aYi7Yp=P2P;x?OT{ zlLI>5r(zh2ngX1BJk>SKO}WgSH5FrDj>u?9kG!@?4J}oS_Q#*Cxk}lm%ct<){ocWI zsHs2?>E(P?a5(EoC#z{`qtc?0cVwU&7W38o~aG1e%(&IJ4>5!#I#?pe_ z>)9Lb-w&E-3c`WB>GSdDG0u+34K`2)bk=ULz`!uwR?!ZBefqjOy8Vz-iSY< zwdFHdw#dqckJ)G!kL5Hob=B4;uFM}PTS2BICO@ZT$nAKyN`zKjEA*e^h(i#FxlM+={M#kZhk=_F+ zd2dw?l{$Xxn;)Q~Jqj?U(IpElFIgPqcE_&Wa9KMs%Eeqw?vOSPGD;3E^p3;1RJiNr zd{&`bURh_oK9|dzQTr#$9}&(da9j0@y1?0i&3_2Tuu@Y~gDrS2EUw)}8Q0?4g=Z8P z7|4x^r6r@Mw}`9A55h$Db_4&PF>mmKeXo=3J2 zfBs)<4x>woPnZQ#jLz7PSol!aYX!gexY(J9K$eM;|KKq7sVJoJtx)=%Q?aujg3G@5 zqrKqlgiXc;FXLTu7VeBm^!emuZrb>t>`eZ{_1`T0o3yMg6H7#MYkg@$ld((OAqq9f zlnLqvki*7fT~({LIRZu%ZmHJbyonw zGs(}9`l=|O6?2`8Kvv$T?c7=tWG1ju##!sciiBll(U1W5M9P%k5!4A^?{}^N2Ygy4+>?PQTv780>7%%64W5}W zSEmBrZNBkfC!^OnCg%e_b(}q>Xy*WC^1a2y637gO zpc7UIA>D-JP`NE2`}`qzkrU~bcdTgFA~7e?N7F}R!^FZQka)6CCq&4lH>rM&9I@Aq z1x#LEg?VV59vN-YZkig%@^jDEM#$*!}NQYz)){2Lzq_SIIVMr zAR`hq@38<7JXP;8QZiOL0YElXsdSy>=+T-dapf0{6kg*ojHt+W`rVjuZb2Ek$_wBW zpE^Bq>#gH-TYcZW2oVCJS^l>dmH_r%c^S*M|FHJg!;OOF$6)9yQ2W}N(Vx;+BzT8) zV|kecYNa5#+4O2?(hEmX;U6WY z9m%HYHIVVThL6RO^SLd)nbDv24}Cf&)hV*u`2MI61P@j*7k`wzDnSqhQs{eBXKmDl z&@JiyfdSY}ucrQ+l<)@VpB(%&sx+BQCaQtQIJ?2yRP;5@>#|tSkF~8HQ?0gwhAG~8 z7TrcX<^Mc{&ksv!i#$m}s!Zhzy3x#8d)!>;4PxyEK2;?L;0b~BUVJRC?`8epFGrRx zn+i?r#2b+W&V!HXj&CYJ*-Y}BfKr|>N1Z@cQ@n!b(}h(ATjw>0fWS?LIiW>mTYPG2 zqt~fT!b)cN_eRKK;B*b_84eCAvKB>O+iR`=S@ysCofEj?8R37UTjPFU@Mz~lxIpzF zC_Rw*{vE8Ak^$X%s{}9t$7c5VVI!AnJgU)cQa&TPk&)$>B|R!_na%qjcf?CLz)!~l ztr-hT%doz}Va2a6v1X*&lHR7IoB}n0DEW@iS$opM?>SUFPUfZ8Nxxu5RH^GdAlMkq zO!oYG^s(_LX0>JB-RQuk8#%){(0Uceq7>itl(3np5&)$FKtWW7S+@`6dWQdJlXqVr z>ds8HCWGmCSDaE`fB$j5EtQmRyVo1PgIpYte*yq8{Jb);tT_C+8e=gFjeTHOxS}3b zX_&ljD2Cn$k~mVVnDdMMT{W3eqr>-)K#`!<2wS(&?h#ek!`4b`pIqBR zhU;kvB~ZWkeSj1r+Q-(DGVK=FA(Otpu>2d~j~JbGX<1le5I__nDq|b+TNOzE{Gp6z+za60Xb!hJ?`a;ri;W zCIhM0``&-8Skq&OxGusQ)(ff#Gt!hLZqCI=VfH^ScLnyqW@LQa5{0?lFXe&BYr#|( zpa6M?)#(#3`PhApwDhaIyomB#SDMNohkSeQRG^!ABTnY_uYn=NIy9+Rz*@u_=0Xt2lP0@feN`N4*W( zU6fOT5~#N$ro4<<%{u8+`IN2Om1>(48}z`FCB-5C>MVeaPQ3%{Bf{PTc@(2+FDrKKlxGn?if+!Y z+||>w;kb42ld;m%=gKLl`EDh0#4%G(vFcO1j~$uJ5xMLy@}MadvV?!1e&PB;{K0I* zE_JfU-I1#^dS8R`7)Ux^wLNA%Of`L@d)K_{6|>xd<8$S0P%iz9z<+x^8L+a#k7?sA z&DrGctd%{jzpN`7Gs^92e=TqRoYEd(&6XFP$A6M@Q_B)A&;&fUA+2_|`M#~hgN^=zXQY1>4e(bCgYVG>jSd^HJ zaAcS@uA0YyxcBfSGXe7x@CL89Ev38Ami+nO!R%ME3jp&!lhD=H zk4)R=p0!`_o@qIfr5;`#bVeQ1*3{FKJh;&`n4e~I*=Y~vknEvohrK%CCs-a!ZJu9K zO}c}D?LRE`5y7V(lX?P5KvD#yOF$Y#P^3d?q>+*a0Vyd70qO3L?nb%<=|-fxq&p?fT)27mexLX3|Cjf@ zu5%qf;9_ydoMVofV~+6~mUJ}QG7bQRF*BKL0ET1zW;>peE;ce!B(e`61JZP5S^LOF zF;Xv5id@Mfr4o486UVlX#t8iaATRNbijMma7(yUzUZK zepwuu#Ec}Vqg(OUbfS{-*nXdrIgOBPOO6OB)=vT# z0v%Y(zQ0ztta|3oOL`_MDr#Y|U1PuJhmjHSJzwY` ziKXBj=+~O-I!X#6+Z$O5Ie4HeUzP(+3=4sgHMPc^>)T5v2@ShPtwbc>cpQIPOdEER z+Se|BHZE9t1{!$Y@b}bPimYejT`;-T(FIE=d>A&!i|w4=-;$-~g;lLisHCob|F0RW zscGQG?@*R*G9BA+|Y=5{{UjpEy4X z+v~MjaRD}#CR~k)TZ^z=XoR!1rNyWPHVC!t=itscva!sbhaE2Rm7NPdop#U;8Iag? zp(+9sa%AqGAEnUiQKo;y=Tg*plRq)n5zk#4IHZgcC+JoIJDd_`0>IDvf{gue2DRqW zjtB1EGRsO!_dSqT8`Jd{z(Mt8x;*@~DYN6W96$YzCh?mXzIY7nRF@pTU}U)74x_aX z7HVI3z9X~TqN^VMJD_7Px#^rc@X_k-1o9QLqXlN<2+6-exTJF}c)B=r(9JoW zd6#*Jp+K@q{G?ITvc=k`gZVr|yeoaod&E@gg@~+&EB*Gx{K^8}XXBK(`(D0oeg#hY z^U+#V)V+OJ7Ut3}KNjnfyDOTVGQ@@KaSR%yA~}4ts#ZaOqzWkWS+jE!19koNr z$|8#Xu~gapqUD#*#%TpKZkoo$X%)V#bvVdf_o>k|MN9S@Cg)Isacg1i-js-7z>3wj z+z6I4NwT|CE3ud?sz`1|qoj>Smgu1weNUmuEhHF?hK45e#;VYEK|+W}OL)&&kC{&E0#zQkXwE*ZcP+p^YsC*AO}P0a^8w?Atky|qVcRe9*wcPb#$JjZwcsg3 zac$qC$XfHxw&sPd&L#!9Up%wJ>iy!Er)x~~n6K^c%kpA)EGA2mO*;aF$Lv0nY=7@b z1M#lQ zPT#QTZ`N3JOfqX(Umw`qgD{>CyWQmWsNkRr2@h9b5ubKNq2|q=7gI6cvfpwnvQp~Hfu*d(>B|;s z%K^%^7ICXQnM_`q;h3tivGHuZ6wyu+7E9LC`WfB@sx}cnP#H;h1)N2xgmBEH?Hx(7 zNTSP8P#$Oo2`ahK%slyGnn%PQdbPZhXT_+7wJ#qt;HJMu5t|>D=ax01pyA5*p#;+$$ze&8-z}RE4+OSDb`uZq?wSVQzPvte~qck;3jTitdN~=CcVl9w|OPb@Az4 z?Q&S(orxwczY317o%t`x=~!ZO-vY%=?&?R`X&YlF9FI>QK0RG{3Lg=5U}d$1UCBf# zKL}s%C(^xD5)U$_U=rO7;+GcH2%W!m7+ie!JTgsNN9V*;xkgkvYF7L;%$Lo|c-Gdz zqMNR!4~D}NpI_lYqd+Khnt4RoY=)ThFtZ81M;ZRaU3ao{?**yWz%7D2-N~x=WQhe2 z0n^0sE^WGT(SA8JbEM>;Yyg-|P=s*5M1Yr{qm<329$4CqYdt5<_^h#e*yiUp5T#eV9| zb|!#OoKsOz0kSYlMMeC-ZiA{Irkd?;H|pcK1d4eq$JnZvFE-nWR6BH&k2VHg{IKGD!U!+L#qX_kB-ARr!KcXKN#M-1U)h=(IYJ%4(utP5Mm2i z>ie+V9%>+z?4(2Mz4Ff%zqIrDA(YbGFZBCznJ;uIq{0OJnsA&nj1*?dqQEfJ*Gpe0+z>nL z6k9AaA1XCp-m=-S6uO)4`nZ6IODoA#bn9~vLJ4M+>+P@jtEz%G?NH5mm)7+O4tX5C zBI1I;Aq8o&5I73>H2Jl>vHHb0Q^X`h%-qM3&Ua9?%HHjuiEvwY6z#n<*$rM#P7ViwnD0|h=;ry zwq0p&_3V1W*^A;3)9tvl2@R=6`UUm7{;T^8E3F!HKfL1tInmeQ{JX{=iTY z3_p&r8((4IY{8(B%E%w#f@R6??Rp>6(ZBc-#r!_7jOY(Dby(2?+`3e%wg-mcyq3#UY>)7~Ctl z&9SF$AUA@62)RE1+^tDW$vrbaCYL=Z@;MZ`2YqT#y{K;-~Jr9VOV7}yva&Zz&h|;!L z9%V>%br*SwjRrxH@K--@qfZ-kaR{2CY?3I*?aP%yY14!z(7Js-OAlPcBS|Ajl5R8)5?k16+mX(|ni^xG?CltQ@wu+YE*rMF!#xxL0Bk%2q1#l4NL+9Y+#pni+63ehon4eDS7Qpwg0>(Zmc#R%#0C71-L1kemPPlriAHyb%kQH7ux z1o&^?I5WQ^Kv2bPE-f(RBvh~rF&{|_vC#sO<9NZ#I1y0LOP_;iE=qB6F_#vTMJ>3@ zbrPsNVGqAoGE(t8>|9(G`E#6g0DoMy&XHTBLle>JWl66<&gVfFa4nQ6b1J$Uzq?#B zkVofYOW}gi#ecRa%aSoV*TQi1_nVaipn@f2IIJpyGIu{o9K8nFj8cxHm;jIU|DnMcxPNG;C6iZpNZRHrJ8(?8sh+{^wi>~RN9;! z{RxmF7B)6n$8QGhR~;WAst2yHt7{8m0XVletSlIu+=EafG7{1|BjAs@ySv-2k30K6IMuig|F<-XqK2m4t z_^Sr#>fsIazeYX7+pkSm7XaQ^JLLhyV}{s5i&=pZ{3KCE zk+OKQpWaP54`>o(h4^r2#NO}rsR)YwS*RLPS#iZwqUZC@LYL-BGEGE|`kRmMQF7}I0=vYOI znF+*H-AN%v9Z%oyK9K)xG{hkw5`r0%`OSMnWPIPob%i5TP`*1NR0G>kcw$V~THy0* zv}8sWmJk;PDHKggB0`x8Vm*?`b-st#9WwV#-RNXCZ(`lQ|7f5C>(*lCp-^+gd;{ID zXFeX5^6Yoy1yjRy976tRf*IBz=Lc8>_wGp`8a)2Sj*=F_aYxV3L5zani+r$jFr`F< zH?eRSxgDzSh~)x$Ds8q*h2_osm~Sk5{1WyeMOp%bcg-PCf z2?)YtC1}EpnJd69V{*uck0rOWym+VKipI~3gG%L|8i)rqgzkBy<79kIx~rZZBB^MG zH^4!bh()p)`U9&|STxi^AD66>n3h8H%s10g9w+1Hy9+>3alM`9bkq=yaI>%eBI9ha zD`82gvg!)~R5}KcdD8`NGA&C)R4F3K>1}7P#>5#&k&oK|1_I|N&E%>#Fc|vPFdPww z{3yS~e3gB3%9VLnC0mIR)Sb!5rqPIu&=oyr_rB3d<1*WBq{i;4XSn)`A~IzHeW*Y< zuj@E^?(BKyT_DM9B1*JhNlk}dlqfEo27I3GVEOQCqF+}$Q-FsmwNxQC&*SU!1x67- zjNA!@Y9{T@$JOV5#&Yn0Q~!@*a{ZZvRY=ehp`+(raQcz{OB{#1RAX={&aK5CnKJH* z`1kKiiFLT4`wt6=kw@qwgiBK@E}iGDd$-91$kJs_xkz;M?Q^DOM{9&&kTtHjOVgTR zHz)>tT}A(LpDjm5;T0E-{!!0;shGiZ}PFXZr=0~JZ0(+qH7hZuISbik^kMf;|2sil05R&I`* z0H9(v&`(Uy>Bvz3Vt$+;fFo7K?mIes_pv@oA05ywk}gT2^Y4RA znuTGKK{v7VI*N+z{o0fQvg9A+8F#`S?vSI(H_rl+DUi9|>L_qf9A&WX%jFm9t*B`? z;hsUyQwfuKFwpV%DQVoaQ*a9Lpu(N(+DvSir1knpe+S}%b&nivk(S5mUCKWuX8V!R z&s;*VV2Eni6rim`(mQ4`a@0Ej6rfbSQ0D2GoFt=!$z)Y{c9|?9?iY0IG4x<9#IqZ@ zg3T(z3-lrfrP$**HMzi_XdTRP#9MV5Sb0ygS0tB6Z-l4694u?Cly zg#@^>#eq}?4O}*Z~jw z#nqQ#nG+!cEzWPYttN#D5wcVHdToqH(#MmOe(d~|$}J{S(s25|ot7t6i_guyQWW+^ zEMA|}@1*#A!#_V`+rM1tw))Ulnc-M&pEs8%AjQ2MS>N4+P>VTMLRj195#^> z6!Y-ge9@=;mUh+{IqM0Hwia~LtP#G@F}L-5@4wbhVsHBo4V0cMc>%%z8el8eS0WF; z-wM^=Jnoo2>$@~dp`$1Y7D;uBAo0e|DTStg)}D?TrI(v`PwglOTnw0e;^{U^^3^<` zxeJXUW8Q|enCM)j6cyZI2!Jir5WtY5L{>uE+RIkWc4jGeb;Bt9YhuDxYr-1Y6mka; z&K6|rm13f+^2P6yG}cK?9W9Q9^-95Js2gTs(7Mn{ZjKG}GdSra;ET0K<0M;O`APL_ z2SHkJuNVBIdgK#YiwsJwzxpYZfWVR@wnzj2%U_A1eM|~ z$b#Ae4CRB&i7^X0GPM03zPh}05M$k?%pi-24W2npB7!mUS=t~4v?-wW3M^cdwpjGoRAU0%1 zBf!T=)6laIGgZh9y#FT;d=;*CFOpFm`yo^+p%A}(MZ`ueu-TLGT;_)J2@2nBh!gHV zY~wN$Kn+sQAjFyX4a=~Zt8GRd0$8MqTQ{|!8q8x}XyV2VW~f1_0sa7RkLxfj5ydqY z?r0vP5}-p@_iXdb>_HYdaK}g!<_`OFkA~M*Ck@a9p&kU=RPKYSB$Us$Zr@)14x0Z= z+l-sWVL-OeGNNzV<6SX_7GR~=q(Mm!Ud$_24&^HV>71bGJh_aR*KQlsANqQGCvDdr z5goEoPS&u|cZhH*fhL$zbK% z0!SFqpvxJ^Q&Xq|WwfBX`DYJTS6aDr>C}Pcr~7>`;si^Lhc`14os-*s{E!#NErPiG zpY&L%y22GL8Zsi7E$3{&-h}WKtJQ7p5pkFtgWh1WoFJUsX$KJn8fpdli1-;0 z!+s?uM$5>!3Hsi67nA1ef>w3fgm#peiOVmq3B=1yU|~m5Xf#AO2I0#0{>YLO!klrE zzULFv+6Y%q$H~2G)AjYQ^F_e{7XTIG?65dH-a#R;d)BN72PZgR$w-u?)!v+_99FlD z7nNoPW#Qrn$S^f<9y2oXdP?P{7L(VQs0T0FmELD2*j~_T_!tb?Ujme){;^0VO~zzD z3x&J4G1kk#$?0EjHE3D(CuIE;0?JnzBRU^0i|hdCB<;~(3m^*~8y_Dexzl6m!P74J zFOJFeIGn258523Ro4w(A-_B%@AHM?~+Q=HcmCNx72r8V<9T)}TxZY`l+zqlk;wjrX z&_*a_@(XmMF!F^dg+Kz*m69hw7mXuD2O(FqFey*jikN^KqNU4AOP_O5L9991VG z;*!LZequcgbl-uXrGCHOl6_&3LwnG}?QR|D$OBIXQ;B1Ji@QMJrS0feHml?pDvw*%VF&UI(l?pfnH!$L+8n(T&O!%+D!?=)y&mIePUrb*4w zjZltentMcOxn+nVJ2vx&w-cpd%69MH_QW-gXV>BxJ;2)Ky5bQ29eeqOljXz==%fZT zRgq1No&n?QSIRnzW56XPHRo9~26`sZ^`9#tJ-4mZkQ>JSo$cOMFa;o5dkE!E_D8tw z-Q8*8+uJ3PFEHo?NKoLu&^$F?;7Z>YcaUnuiZJq~T!bbuCIaMyPloKdLx;CA{RP`f zuFM;fJB%^`zaa#O0@LflnVClSmPhPKy9Z$(H9ZwpR9qvU`BVP=OmmPS0l?0Xk@!x3 zkq2u9004ylsERvTBJMhj^t*vsOfLxcEO{z4H zE$!VLGIyob2VEhFbygzzWjR9m*ZNaXmT#e>FNjKD<0FhvCvp5kp@ea96EOeAW=QMg zsKqr~p?ocSZi$6ujER1$2G8jJ^>X;1)coG8e#H>^LL+Dn^crrqdsQ=A_>Uapq?i9vdKFYChy)PRYWq*K!t>qot5)fXOum7tX)dd!uDCd2x=s_&R1xH{ z$2ly=+jeaAG%drblR)6{KSBv?PQ@*rw}Nf+S3Vuv{2m1;^Img9Li_CNOS=qom1Lqj zS$@SF>aOp?ecD$b$xl~zVMs~y(EtL1!~U-wkkckJ01`~#tt_=+g*aZlR@7H+^R+hr zr!E6cD3gmpnk^h$f+jh403e)Qo^QFxdmJx>zCylDat0FYWGgZ=4FHl-`gDbo3~2lb zviD>!K;AIONb>>O(!GcHAGCp+$G1tWB7d1Qpp!}74JjxI2@5CiyCl#7O&@h|uR)vy z^kv(%0{uMkIu$EH=?8#il*(!A=!oI9w*++}LI@CtoC4qNzqCOV;N=-!1@=GAE-&Bm zUf9`D{UvB3X`98N4WhKBW2IQPFl7K3hLVQ8r?G0enL23f37&%a9By!~t?dKg5meyu z1YZ-^acr4(WukKLtLb>=5(oxIYB;p-|2~L-H=J{M#|Pwa|5rb-IvH7Xi70=WF<=hN zlCzPo{N*SSR=!v5nKCIo@lQmIxxo?_zG)K7b$hZ|F%Ka0GSHhr(@r3N)Yy>4eQO(^ zptt{Ai05x3uWccfQ^La6akCExLrVVn8v`x=_gVu0U0BEqwMAJtPzX)5}B`NlZ}$eDg2F&=BN{$l7Z zOtu^O`$IYP>p$~iGjAFoTodT6*(tm?YNFHO=$NLJS6%dA)WmC#v0CNF9TQxV{Bs+= z_l}kTtnwKgPAwck^AC-WH?&*{K)yNWZ>C9ZfnmokJ(1p{M*0&b5hR^`TmL>&Zi_6J z%Zp^$nfdD#;vRhZNV%Betd=)z6T|OE8R6S)qvsJ9_wp8B$1@C2sPXTlxPih$IeAn^ zLNGYj>A7HUWrUy(g__I53VqM`m%m=c;ZVz`UI!)Wm|4)j~tHFB|jVMN6fa0{ReTOrPdNQYnh$((aGW$M>?D2fxKkd`RM4#HSkGav8-2h22eqUryXuov)^We zkI49sRt|jwyIxYgZ3#}tV|zL6)} z?c1~X5HM>&B2!?$LF%-RMOh2%ABzvUeGy1xhzor{-4X1O3mZX$?{Lg_dK;l4v! z+^Hx!HMa_(i*;&|8_e8oAE+`+Che1f+0sVBeY2JJBWD3&G`h$aoIY4;t#^f81Tt5P zzE{jq+=zagATyLCKipt*C3L=|n8@A@4Gq069fkxdja&DSEV2<1qH<+2ox6J&ESt5F zP*Yv-J5gOsS1ZyYl`tMMJj$NCHAtX^50=k>F!9hn;QcF5jjFk=_a>zfwf703zN;yA zgBT7Ssus&`d_e(y>HaF(B8H0a${u>?SE?u;)^R5iUn4#ALaaaR>g5j8ZugS#stF#n zG(k!XKTfK2<3MC*3ViNSI7N{ERC7*R3m-bq%r*DyZ z^_1Q>KTc2Ff6d@(8V%dzc=Xt)NCg_yZl!InUw_J=D2*jYO_69_Os4pVn|p{%-&OtQ zpXF<6gm!vI7Xtcy^2nG1QQ8bdnU^L@G5dYG$*l8I(D{$p&tDxj_@&}bFK6ChHe4ZL zmBl3CrUVaMwCZt0QDdYFy?K%v|^2E*o#|F0GsvQ;jFkroKQq^dM?(1+O6E) z&&rXwwfo=bb67Rbc-@H;R?DB|ksaJmryxC}E&FcEJ{HlN3ZiN>ETNk-wf_XeB%mfeeYj!(;Z|FSsxdtptX?)O#jU(B)`D4Xthq&3+T3xB8_je=2-iuImU9h># zxA-UX_T9M_sr=OhP?6y@(^mh**OJJeZuvple_22lHymoUDs%BW89~G~H zDv!X9FNV!<7XXWp@W(o>oAkUxvsp(!@13f)H3#UWLXVRkT~k)={UAOB_f!7M-bd2o z&Gf?En==p=fTR)r;$yGax(3hx)329bAMG!GDSxBF-rs_G6+}`!vjENp?)9LEt5P^Vq3V*9e z;tFKhPR`CQE|$%T?iL=?;ANcOV#N=ztmu#`Eqd{XDE!3ekQUOl1?ugE3b@ZOeH!+- zZ-EYCVx!A8iIYhkGjySQRi#zFp3!T8&eDbyw`P(<1b(L7Wn^f7J4?q}nEo`VB9tsG z+P}26qWpiPfqa%JSKYqFKmo{R8o!O}=RjYk?2-R2s!l-{``nyQ82{8}^_3TD@$TSl z>ht2+5wGlGn`qfa@lnG_C?Z>dki%Hs-O@-2cjfgvbLM|pj0sM}v-7N1mr_r{;_P>P z6ZbyU2mN4X`*IT1Y=g5O;U{~~lalu_6SvJ<4I`UPenXUc{z`F$w`qq%vqJcmp}%JC zEDr?AMP;ap1w@45$?2-KOy|Du7S5KZe>EceNj%YpS`KtG zYi^b8g7=1R}9=$f!F$vPWlCcN}kI?2wi@T)%= z6lc*g!VF8cb0#0=_%2b?C7dl;#;CEd!5?!LYk5zeyL*!<_TPm}yjCzG4u?}mNHG-P z2`Ug7&YwQK^)x}s2s@&$ZCijreU0Y6XROB>&2B!{PB*#r26eUsZJtb&^9BBBRjvyd zSm=h&?}+aiqelXv31LiwvN4EhVT8FN*GDON(J)O$N?i#vY0}QB zEf)GXoU1MD+XjF%Hn;(O;&B=1=Bz=~rW7Nq(7PuW@ z{w!NuVbeSvXO5P!?|jtLS3?KrmEPsEgc~f-DvntWy>e90+$k#$+ui#v`zyvj*IK=_ zjg*rkp6wI#_5M}p_;?ls_UZp)Cx1Q_E3&3LD6p-o9AVzy-Dj%*Zgfb!VxHIOxVP5j zsi|ML^L|Oym^D$|`OEvThv6Yp8j%O9FSVxtM_2~$)Z+a7zl;VbeUAv9A# zr5si*yVjqVZH>#~N~(4~kP}Jzo*(}{(`o;($kCu?oa}%9j|Ka$k#HlbsVK56D5#fs zcMjH@`r*{K&ifjmg|GAL9e2p|ah(@&g{HcosuE25=$*#=A`MuLtzpZ!@QNp5SX0}Y z#5B2^9+mME^ZeIQw)e$$+wY;FeJskJ3yS<=RP(Imo8q)=qu6 zzg9n8bkG{;=yJMTKX~DE5-Efc@jh5QD#VS?c|X#F-&ta0+S1TgCG||+=+_ulHb%+h zamsUrgZo;BrG#oh^s|p1J0+H|4ONAug>40-7aG;-A4UF~;%cp;PSXg92s-KAoTrp( znv>G>rp)W+YRd2vt`9#QiPJ?ra^SNMdn`#*QTzBs$q5s?TEC|}BkffGy^KCH7QM)h zPr0OTMOAb~QAZzyI5_p!H@1E{3uAMPpGO$^!Fvb>MPa=d`QedMzmnn4`c3-p%nIQ# zIpw9YayX{zaDVBK0}Q=}UGI@7SsMnvp4Em34(EpB6Z!qzkuMxTAb%goA9*&ukzaxh81 z3Fb-}lOy(4j-9y@{<+e3&f4i<~q|6?Dkt|OGjo>dhYYB zSY2q-iA3}#zC~gtyw!%l>4-9t%a|rES^vw}Nj*&bB)m-%YcCnsl;MBil5pK(-Po6o zxxGA6$vK&+shJ)ZN@h&!j-53SN4A=K-+Ag<$++g6UGgygc`8IwWm92mtIU_LLbc(+}%~V&SQ3Kf;0gN9nR++csv0t)L%T#%&(4kRxuN-IXt^&q_}oF4s(# zl9EwVsii}^2A?-GB7E!gXKMgH|MBfZ2(=vFTdlMzZ|x-CZRp^=`=E@yMrCc_es%-;RTSsSRfK=O3U1}-yM|lJuRcxq&W_vQ73%yjmXG#)de-l|9wv> z4?(6f>jSD!in?B=lSI-52{f2zc-L=ugG`Xl0$hLi&92kQwT%ry4`C-K$j{dbAF0t~ zBRJo@o{Jiv9?BWvpHV)-K<2TrRy?Ki_h8$8alJTR`1}a?ZhsyW-RN3-n|EA$ZP>Tt z3es-`5`+cb1qu(wdAm}`E`~UsU!uL7P)5pwC zPGIlXh(3?gxZWma)KAccdfhq=IQC-Or8#6(s zOw-u13vJdrohHQIU9pvoai)6Au?WJMZt4UbCih0yS5XEkB8}JFkD~ADa%w9ju#{!B zyf-x(^x?9!8TsL@k1{?WZlBh#eIDdtHi(07T0xK)gP-_rH^Ze~UIAe*)kps;=^s`x zMMrjXLl++k!NtF)?=xP$T!UO?%jq~R8s43I|7A=-qpLZPy}#cW3*N|G{g8WkV(0kG zKJo1&h!Sa)=66@1I`@Q>kwl6{!%NV1_HyfSc0-GMH~4=0XIn;9|AU13%+H8MI(Q3p z4o6=ZKPK4b#-a8|xn_x8&p*L|D5oVmf%axfd=-%uBF`1~7Nu<~0KcFrw3 z;$#o(!*Dop6r(k*ut)veaat2J@hz^G-1sTu&V`}aGV2~k;_G}B_O8Xh=TrKWz&ij_ zRGg(vWetEQsw-}mdg&)LXgx!u8k>^27V_n6!WowVOvcfWX^T?(Kp{MWtPi@11JwKr zxu^YFAdp6#8WAMps-x(bI5Fn%djBU>>P{GD&vzz1^*cP8mSt@P8?sjW^mC%zCeW|FBZYTEIz$s(Kha1enA*DM^V#(YchI}orjFl@N z*z9{(Cq2_`9YKLThTY0p-(?sf*8Du{UIu#eDrV-|=t{X)-Uzv0%k#4Nsk77Zq~C;J9TQdgXZh(JO8kQ! zPUQ@1q4y%q!=&+j%=^WYyYdqe{Fa;>-D$2Yb<30x}>j=uuH zN@W)VFZb*BvJ;rrqtYh`==Hxd=`bg;^mqNOjKSjRh4ooT_o()$=sU5UR+M2jl{34haP>8pkt~2$cQA!B zpk#Q!|LcdA;oAikc;|R;WRAo#41^ERjHI_<)RcCo5y`?7}ep9RSYRW2BZFKcekxrRQe-ri+`M8;gDcalH#n4t~ z`<JR<8;+<5w92!;qH=Gyg}N6`i0mNgOpz1&STZ7fU8vyK>Tywhq0fjbL`Y7)V$I@DA*o~HVWWI~l3!!btY!Tm666T9 z`hsHfMFKbStRoJ8`9%uCxWvSQunzIo+4RPSty9F>dJ@Bm+ZWHNi+@UxZWb9qUTcr& zlJT(#|g;}$e65T0sxK0WGmn+wU%lrSSs zH|Cd_wyC*D5IgmHOZk-N$w~02m`Px_*d{M~Q8$;9*Dpam-qDnj(9PJ~+e8^#EbPI> z?{vqNG|eiK(YU`NcUF0r_;J`AyB#@ef+HS_YFp(!Po%+?yhq&)6we# z3L*eR(j4ILSTWTO9e!x*Ds z5nor1i))hODihN=Z}ykP+Kqm^#fg`%x@&p}Wc7S0(dC2sQsJ=dH*HzaApEhh-YH6K z3AKyeY+bw00+NtE4ouR+EN+bJ>u=Wr^3M{!hVSYxq!%Kh^&-&&m7YV@eL#B#zBl(0 zNrFK{EwLzw;re?%jnB%{a12DeVVXh{CD}?UW;~vQd#qdPLmAUC%rD&_n=gM$n+jBI zf&6+bv1)^MG_vaDU>Skx?524v%{hSU&@Ygj-W8du4aKfMe`?;8FyWAbt^HH4wv5rZ zE14~=*VybO6gb5t#Pu~?eR`}Uj2xVF+s;jq4~j&k<+dR&oe$r zR5||c4Wvw?6yv5nVsMEH&HN#!8EbzFugeDR#~fq)vBeEP)R+nE>eL{YiN&4X-~AWZ zv$5$rx19jmr|8FmK=UQLZ?n^X9OCOyUS*A>ZE)45XO^1GMiSO)_8lbTu$~okIdl&l zL`QBPR&|9GhnQ0H$L_1k&$K=z%3mJ?TDOUw^u#Gp{6lY2(3*>Z-J;G+>uu=^Cd`_w zG4!ZfxYv`L_;Nnc<&}rYKk|DDbwN7X;mYOe#dn;AIgGjs_kg?zYvY4Awg>b{(_5lh z8<9_cIQrNrvn_7zuFMx~#H5X3N2&iqY!U&ea2;d0t#kbN?@rhsllL~unRY@;IdV$# zFz{r1 ze(DX)V<=hY)tBO&Z|&B%(h7&8EYCG!8QZ}^al6_=6byl2soYIr2qY}Kht}|46(BRZE4Z8@NHf}6=&9N zS6Sdwyabu@)7s3C8s;BA@R95N>J>Cs<=hWX9FcE9NpC=+*t1*LKBC1Se#M=&3>=b= z$dCbnISIOa3|C!;lw8ZD%{JftENv8wj1lFJYl+wJ<0%$M zxFcT{u~UWTXaVRh!I986ZZFZXidiKOq(`pKP92fpL*{9eNEi+wKm1x)`x4{La?_1? z3NqbSQ5V%Wz{6`W6Y&}#JM{ld{_hlfEi}6uM1Uf|*ttGN+%^Q$*yaI03X%R?3_kvq zjRyGODhdC;v(Z!=$on7&V;;_T{O&hgIpJ9f!!YpU5&RUB&5!doGP6F!cNH^20fD2d zM79}vyFIO>f!+tOnQm~|%zN-d3+)&^Wcj%WtcK7(9;hCW%b&f?>ji=G1Lt3RWIP)l zhC_n)s$skc8}`gx$WwkTJ+HvrljtEnDcpnxFu*7j1cG!*5Mx5#CZy^}+A;_|y)2&B!! zGlk#1d*|utDUTZ(AAb%|d__b=;I#&1iAec%r$$%^4aC+wK0#F5OO39vm}MPK4Q%?! z(Y3C0>-2!|%(rhaOG<#y%ahFPy@_6=ZP48|Dk0&e-f+IU1ko`_{c3M-A1&5H%eVme zj1)eteuP$s>&hJ^AZHd-6If!yv;E~Lih%Rd9m`2gMJ4;XV0apUlarx9i+z<31!5Y2 z(J?zk5BawjW6ZFMCbUi=jsOJ4fhqtP0Lt62Wni#jjSr?88dCtvxy=_8EraxZVlSVa z`BsMRvGpKkfw!JF4H3|h zwTM{zyELBRCM;jAAYU(@YSGMAMJRV#nqKK*m*;|;CI@Bry*kbC&_}05ni>2>wvzbo z9n>rBz3Ox9tsx;^eumH@HY_LKZ`e_O@Myfq2&tavu7C8Ai??% zU#J!>qg*;aF7f6$4@}^H>C4}|Dr;?jH;VPw5r6>P5+t3f6BhYx`708w|EWvLF;dC} z11)1`C+$=XOF+$!*=a`U)t&<%HPtkZ*+$J`*3Tog(d=oxI$NR#IG4m-Y~4vJ3mvqq z_GO86KUH__??uEvv0{sTN}M-0-J~3*e9GI?L{HhaW<`; z22i2UV0Wl2$cJDmcGdDS%oNlu07Ml8BmiZTAr%`D5h3gV0E=iIKi&Z)?qu|k@GoAh zhaeHf^4bZfvtv(4FL|Iu*1B!hU3>P^UNb*T&w=Rn4!!3JHC-!WK;-Ys57}>eA=rT` z570eVyOE9#<_N%&&HkU}-ZHMrt?L_IfS@9R(g*?)(v5T|&7!-z8$r4eK|rLtQ&Ku4 z7a(1NfOHAc-F@a#_kQpDdEO7_d^`Kg-u|}ebju~e&4Bp> z+>H*Gz&1;dGF*#ExVrLODUFv|Tp~NomFgXPtpf0SE232Q(a{RS-3sf_z%q%DpV}o^LqFa~C#Ghp?ApMVzK79W_~{ zIK61nI0BnOj|_S}ilC?2N1P6kGYPaU1&+BA&N@}~Y~qzP$3hMW*YTT61JU848kz;U zO?w}}KnKb7&eY+AH)JSXP>%H&fL%9%orE|R^}+4U?k(3RC?vW#x7rgxpc}c2I?*&! z_}o~TbR7p?fb<5WJHeJ`>*9jYXkiusKv3VnKx|SaP*(00Rm^d3N*hm$jirpGScwd! zNXW%BRX$J;E_4Vyto(J91DR{_5!_BFZg_UM>n#ISr%x6h6HS^5a{Txm*H$n4^5*c4#dNd8#DxXl=~ih#;=UUe$xUwY z(MbI6Rl`AU3r#GSBfW=di`VkrI1_(SzeUHH#`9bsG8a*m@&}P~?<)7(WqX+~3BB=y ze@=?NZe$IQO>V~i73;u1va0ZMeUF-r$svJG*;YChjEy)U2wQH&B(S2=X7`Q|oR}bW z0H@xXtX%C+Jk%DY{skx|Of)p-fb>&`eP185s`Lp+CyQC4q4{SZF>}SP&9?}4(CgG6 z0dtHje;)uj22?H4eV3WnL9bN;KV{oB9jgv%(?DNKK-FomW0T^dxQah02fFo+0>-<6 zD85t-VuIB#<}OE#SP50ek-(mFGe%och>}f=c!5*F8HnTcmJiB11qamSQSr&2W|ZxG zIm80k{lMDJyvbMDX&>u}RC05`EsLxdv3OotPC#m%n#$FJe1)jsLkxn^7u2${92;gj zUX6kz0jFY-LXA10+?*H|Xopm{sA6W}r&5~EddM$`8f8bBB1B4k2{UE#%^`hB5j0~P z`g=NBupB!-wuK zBuwZ%sW#Ut2unSW%Pr7X!7HfZ>>`13+-QH=I_P^!8?iSe+S@6|--LOird+ENdAfVC z^WF8pMjMG+2zs%c8;w2KM({>P_am4~i9|b;!xiY;shd zTsmXiz;K0$6qw{X4)IfEeOP-_-`LpbwBiX)*BCRf!&0Vz3f41lza{3Y16F8NaW~hO zO6na-a{Mp!VZK0+Gb$?T_G(p7`x|y8TRA& zAed^vNh~GR`w(7P!K(=TYqehX#t93tOtxjhlsA=zhd414pZ<-NQ7AKXjWEM z7l}^_?&?swT<8It1cfgKF<-ZV&@tgcKtMoU9ao{BaTyMEJJ3ofSODl&7_tltZ@Xj} z>?eUH>Mf|#Xv4Y)lv%^8izEEmT#4<{~GCksLq+3sPmM&cNBv%ismnPKCo3Qhav4q>*@f(5wF?hNi>)D#`<|{ zi_#Sckek}Q<-&KLq{Eh4G%Q`|mFnek_a9I!H&Dhz9?0Nd8Qzy->Rb?A;boXhf32KY zSOK-F>?3l$lEi&ImaUW()`Y|pMP>~8A9U(GzCfQfFK0;L@&n%O?(K~oppaqoJIqqH4&j&uoW}76ArZ(7kR+)wvW@@cB)Fu#yMW1+&e_ zmS0;?WF&^f*(#Z6GvbztTR~tha6Gqe!jz)Uar!k=kI`6% zdFCkfFs5}iQat83Vd|*agF5^GSnF-gaj4&1@U>J9hX@;zbTU~NkJ6TeeW1&E_lV$1fl7h0alcW5@MS0-Td1>lv_xpHi;x3~iVAJ|r+%92T8nquZ+qdFue-8* zQ_7Dk(C(Ds6cfQXY^LNl0HF-2{rA-`6Ome6TKy^w1 z1qT>It|${CGq=zr9dqnytr0zf?-9fazadgUK?@8B_??RH0!n^d|LUJSoB>)84+%RDJ>C;zm8z@8c)a}7 z48OTe9HANE>grK@C`rjBp94mW*#Y^|^P-Un8~MO6+-8BH>*m-U-6!)6CiGrG86#8X zP0(F3ycK5$K8BM#`*)#~yx zA8^#;bn>mZ=fkC3ds{7*MS>IhjuM)k>RsvQB+R636PieGG>zA{i_G1=vhjE$-sAZft zM~Ah}FB&ac-wV@loFPa}oKzKzwapV(F2#R2%F%S#&u zp*TNCjX%TPh&piL0pv{YoAXtFe?;0P0dHVh15TXKYQ&Zx+54cdoul`C3a;)9;2;Hk zRO80o-$ey40=Q@=AtS0=3>{569+XN?@hz>*9o14j)xFvpK27I`-p(gG2N-tOHeL2V zKER4GaO!W_mF2GhzChR0N(YWo)Mb{7G$B+>XM@Bso{BMvHHf{8424n^_9|JZ zux3nAOu5WG{qv%P_TT@c#6E;O8HL{_&euUe%7FJ60hgQO-)GdXUh#ewN16mBvd=21tVp%u@ z9pr?+or;)Y)&}(qyZD`}hxhb%DGQjSzB-{& zcgl*^CfW4o8hs}we4p|Ll<3*L2N$<|1>t3cmvCwVSaK8?E)m!M|MiF4(_a<-2EdFP z!T~S~8;k^Jw12-Q#2Nu{;&5V20gO)SK<5qqyj!qLVpa-L!E{s}+Xr|dg}(Fu{v9$C zzr&>U^T)Pwf(4)f!GGA#rsuie0f<>5F_FMAtLf^X4~R74^^ubPCtGfu`Vs7N2PNXf z=V`5{ygYPXv0_3`&cLhxjw$dI@asKT{f`5XO33r$=_U`CTol#&`kEAx{cko#gURh0 zwrj!#F5aVpt6oD2yFLu^aalAmI>VYsQl8_nmTs^FHIq;zAYCK8Gl{CWXW;CCzY-GT zVeE{wymOdEQ-LixKiA|zhg>#^x(!6R^0}krxAk!l5wu~sweC0MF9gUYAe&Vgfd8tV&>%A49y&#^Rgn#y{L7Q zKMQYd#L+Oid$5v){g{X-BDrWVgWX*J7EU`ISPDWs%l*s=tFnYQ>;&E}yMZ7}NkRA} zO0yip26HHkj*`ff)X0x|4;b;XjGz4-4^ZCchp@q*FY-r&RBOalLdu&F;xel9MJi@c z=y0Z7rIq*Z0NRg**U*|gFPi3!BkzW`DjFn-dA#q#);HQaNL3;gnv3WwZF$C7Ar?FLTi%v zd$2|)i%r4QQ-6wb2b2lxcO1l6{Sc35U`yHl|6#nDU2=8l@qxDPUbOE^tU5fpr2t1j@uV3aQ* z4Y=|5^tC?kP)g-v7G=}TN7)0{CX|A z;R8FWiwgOx9<`nvM+NIf;)6uPW^->{sH62i|Mv+1vH$>JFoL{PT_*yWPGF#Z?=WSs zzYj;Pp{mmQwj`+tubXc@LAztbEH7%xN0S2YW)C1DScmd|uT+tcSSvFi4@C2J-&^?9 zv;EN1@_p;}LwpK;l5@TTXJAaB;@S+9Lj?G4TX_!CHu$b56Qi>MmlfNls%J>r!+9eo zn2AqOt(q($8h$kX?k9i#hzKxvzt>z2Ro|bztmzOItGUJY1BwR_V;G5CR4I_Z&oG;| zJ;};WZwNf(fipT0kM&oe36t*aJ`wRuc=GU}yJdHD>vc~%gI+TkXuJVa_>}7?N)myq zy=;DxU4DI-lAm9RMHpx|gy3Uw11IJCSXA*1P3EZ4 z80-Lc`|m`yZL$I_oJ7v!bzdpMKwb53^&KKeM;rWmEY}#bkyznAX5pasMB{E8&RonE z5^8YT9nLZ-0-^={=f*7x*lU`NKz;H!3=s&1s`bx9I|nnDxy!C1J4-Xcm*SdxH%d&DVn- z>0RDq^)eCNI)mG_KY&IMnMbq5cQcXj@TB>YI7=jHn?)qfOHsFy_;eBWZi}>W#sl@cd)N@9keh^fEY> zVBzCaI=m)R20=mn?nesu_ptt+;L$-mJUpZ{lG>xH^n$A)dV(c4rbC;#!YrKlbxO*G z0HnoH`UDivNZbEx* zuIt7R3$*d3)hYvH;;?P+4wTV|Xxk6+j+QjmhT^OXV0HcA{ZmKz zgPfo-i-~E7|AWVlg(XWU!|m_uC3`P`nmA+vlo3JMYmT(?b{$nc+% zR4SR{Y#6M4L^>g-!AL-GG~o+NdQ@sxP^nw|JUl!E;{Jmy%hQ43_`ijo#|mGgDKt2z z{+Ry|tW$Y1cggdf8ad|h=CVP?H76E)xdE}DS82gDA=hN{jw@tDs_qr3eR^>MADY%% z2a33Qiieas`@$A2a|u;(ml^hRf|L@uzWx~{fGkt^%psMBW8i!O4v8H|3hwm_^!sxd z*b1!(5SOgT-z?27 zkfQ;J3Y&H4-38i}fu2BwTc;xw?3F&c(&qiKsfWN>ol8s<7+&3dBAD{%SpLBnN5T^i zy!E*Lq;(wwf@}xbUf#OG6zrPi!)}>_XesF3euFt?DP(Spr|e_US<5Br1_YjLk#5s} z%e>4{h8%A?FRZO#ECej7kWybv3R^v|>H|AJlmZf?JDNy;l4!%ae9_F^X@A=nPYKcP z7&0%&yhUJroskMdXt9g?r(?*3s{hylTLq8CW-#LJ?w3=6$O1ATDW7da%wiaz&f##m zT`m4DAn;xI{cwD#MXS#REVsF8&APP){S(yZThW*Y&T@kM^jvkf3$4QB#5SKa8z{-+ zjmERP37ZDb_;M4Sbn?7q-Gt*qBBrB}WoiQkF5Gpjmed;%JW+b!6)3ftFWq&=?q~tR zn-?VfmiO@R?jCK^p<+A4d{|xEHcP-m5v5im9YlgGiQ7{OeusmlyzjB)@(5U=nkAibL z`@`Vuy#hG3URY1O|0Xbtn@~JkU+xiI>pgi&-Sqy*2isJzD@}6cu)F&$?SrU>sL(ut zaIXZ*CTP02Sg!e41*u(d1@TyC!}Qn3r|9NgqTQ_E;#0N)f#%o-2WR@9OeV6$fATMQ zqEgs9UW&)<_xwKs4II_qI^nzI%|E*&`kf&D*;lM<_meHrljZ2^Zc!SacHz+yEl$wW zmCi#^p7VbZZ7|orn!+C%`kAPhVo=rvi<{e9o3D-{uok zMU0vXFXnG}ZxMDm1kV0BS)M0P{oGktxY;_)S+$zK(Oqem0@^t%{ThxyvfAP&m~fqb z|Cs*$I~&@f<7zju0Kc_}x%t`Ju)*?_7>KDr(LKDw9;U!Ru@3}iXIGcEa@C3nywX*1 z|I}5AwG+oXbSpD<_GLU{Q3wx_{v3C+@X^5Aejr?uiI3(Q8jRuyx(^G{caRCZ2UOOJ z)S0_`3aQu^=&#u;jgy>0QsG z;V!18E7UKXP;dYf*j_;ekbAZx_a832XHhT;yfd_`s#MLGa0zcY{GSsTte;WP+H^_s z!KW5KzgW2|DDP{b7{sJIf3}|oXy^i*ZzsU{f)2wTWguFC_K{{qBAY1X4bq*m5#73$ zUlCn9RBBxu3;RL8-n67Ht@2HtBw7r~Ch@(SA=VhJ3tm1a%kB&hBjPl*#wKXsUpyCT z&(7);e@JprvwM!m1l2JzbA8=h7{N=0C#wpgh5hai(^&qQpMs>PSR3(R z;bta;L2u9vsZb4LqE;@|a(2+Ce zR}ey?aS7>YkM5HC+SE?Ap2{W*wAZgDGrvAKi!eB>pD(1{WYIc<4T$@ww|@*G!fEk0 zTs??C_rk$GTWr!8@uR`Z?OGB|3_?^^#pyvl7i1WL#s|$ZSw-wu#BA6Q?6Cb;PXIi2 zFLxHWotfE=9~>fWjN@?2lrsi1;d01J(kE~xn=AkO%)eU}4F=4l)!*F%+?P|>O-AZo520^f8-`CSyXmKK4tM^?wM*KE68|ZKyF6w9S(t7rb>o$MOFxs3m&_Awl)q`?j+{eF>C%X)kYq+>H$pgoc>c$s^&Ad@ty$x z4~;QctGDt!Y;ev*3;*AH6z5aR*h^oFe)wcjWkT&G#3~Mp3XL2`tDglyI=3q9^uYo+ zR}O%36OfO*2F__$?<2)YysCiPBD~wdy(Kax+6G6{{Kq|Q;#%+XLC+W z%^)ALsEL=Zgsp)rBpT3UTyKwZMvL17odgS1A3A;kV~fAR2Jgl-MuRyjK#)f)Jq51H zKzGxwemxJ&c_VN2+*V@AqJ@heQw$QI&ngbKI>04fR~<}myrh@3<%#g<2z=V*Kj1i@P+TM}e`40g}u-EEA%!2(dKQvh}HZhy$hHA#Z7U~9eawgdW) z&dToV;Q=JyD-KB#5&`RkQz!CY)=5s{Kdck0o%7dmo1&s(MP(&TN?KZ4Vxr_KGr%tDLhZ|5 zmDZ0-s9hp~$EC~(xQjt7J^Kx~a)8;c=VnRm4e-$f%CMn$OwFLBFZvuZ6OO@;_|RHB zH-3ptu4GQfoacJlq1YBCwc0Y{w%>5=rgfXIPk6eVXAC$vFJna#jeMHVzBV4_wcLGP z-MD_5Rk_}!Lby8}uBF%c0F$D2QXfI*sSi6(%Cfp9eqd-6ggDf=bl z^6}^dkXj$c%|MT2?N8>ETXsf%<83<&-E{Ix<+nO(EK6TlL9CNa<>5(XWMiuYQ$`{s zAgLMCgDq!YLwicd#=f`YsLh{0-;DX3zw`d=A{*;Tpl*#; zTFCT)_)cEa7i{^?9~dm^+R**JA~tENAgvE$7;Z0)`+gD@{$SmwgrZ zzeqH1DG~aA+|#jr6ZQ=`Au7t@S3kiMvQXn7`?LB6;DxJGfl$K+gj?Ze6UZL+pcBSI z2pNP=VgWzpTThMoaF*5QFTXIS?v- zGTo~cx`swhAWcA>+-JhtQ5*98Yyl2lp{LJLI0m+MU~;7g3}#eS zYe13%TUx;w5k^%f8mi z0e!sUE3%n4xm1cLMI3S(naN-_!LN=9^PC-?gR4>&N`)WomA2gUUV_kYmR-35I!tfE zqHnWq71s_d8a>Y)Vj+31ZbqTwhWS&LD>L`)Z2+E92bhkoWI7F`^2Tpw?g9N@6K};` zFdSCu#8MW-qX2@tZ7$!XIn&s}t^%_7_>3nJC%lPi=r(!zt$sePuXmi`K)noVh`N2K3}h2^5%dFW36 zPE-{+C!nA=hhi!UN>ARtr={ofy$KVXOn2N2H)7$V#l5|~_wV07mr44z^pJB#zC`my zBogHGr%L*(ESCE6LpC(yczTVjCs^P06}W{v_G6-{ObdSG`tx1ox^wArKU%#SEmfTS z_D#N5n%qe$_HbdkjsfvwJfY;V^;A_bVy_<;!(h^jP_LeXa2SR;y&^!{gm! zy1yaNGkA7AFh?*6xbT?ufZ|lIX0z**F$%ct0!N~^xtDl!wK5iQRn#=n+}Tpk@nov= z3p4sXU*0+@NM>q0v^L2F`VPj8?B8Px9SUp4;XE#4`uqBUpYOS~9Tp7YVe62mywX)V zh+6FP!&@I+`JVPO_DAYLTg-&wmxC<%&Sc$D2czYB<>Nk474N&Bd4r|EI4Nzhlh7{b zP>_zUC_g{Htc;n83e2)!u-{EF)$6h3ycnG#4G~;%OL1U8)Oi-5 zBLUFXYkx^}BTyI2a9G5C4W>#F^YijHnDoT~;N(u%10N)5q#2-x78~@FJB~LW zM?xD6pOo(C?lyfqp5L9g_$l5jczrl;aoc>=mj&2D5|q-CuSf(hq}oVYD7Wlt8>tu+ zy=h;V9*V#jRwG&1zo^nJaw%Sb(|D5FSBYt3Wu`bcD7vILCUkEQ8f@j9+nYk{dXotD z8{(6YJrs5E<(dzCMk`-vF?(5>cDPmrZ?DKp?|C4XiD;!5Yzh}Pu0by(~+Wet+kk7P@ zst_fgHvm4jJ6RdTiUoY&1@xBzauFN=S&o3A35%{%g^KIvPgS1SJHPPhBSUln9H6hw~?A^66CG&ZLkr3hWa5f4CR*HAlk0ck9 zKXIN-M5ITNwY(trWk_@g88Z^BICcQ67mPRcArbe1j%{XTYBG(0fbQIQCfoQC=Xs%0 zyu$37f-Yi}gj)2$V{=Azipn`0htJ~yGC7V)q zbVD4gYbZ7Nk=H)2cqJi#BTg_-gtpc#;Im2&BeSA754U;a;w21w-tdF$@!+cIpt@V&RWge#(!4E89<#$$c4%G!Vexh5? z?KpYy>^CR3RK(_D~u;N`^wVNmdV{t^?kK3zXtcE|dS8hUh^0buQ zdq%xOr=xTth#S~iyd^vv4&sff)APaA(F*fU#Y z;DLU2?~8l0a`gfY*OVWr+HeqE`+wap-1eIUB*N|16RhATQh)Ha+L!G{kba@J{UiW3 zb9H{`=jWH>2UN9p$;Cf@qe)6j`#4035L&F)lD6tykRI~!(IY> zsIj-?^ua~*ArQ;!lkS%NFu_`Iqu3tg1DA;c#AOtZ7NaK76+J;Rk0H`3op-4`EkfdLPUE84z6o9VQ2I&L2dJ&|e`h1`X?tWVOB@ z7pYuX{E7lUQYaozL%CAdq<}^$@=V!Mp|YBWQ@zbUZo$=rm_T(W=7X{gVJ1I9f5Z$J zhusq5<4b&cWnn>sN3dM}ob*LSk8UA=a2jxHqahwFb}z=43BMr(P(44x?87_)M1eu7$^be7k0G#}V$ zjF7Sc7}*jqB4N}%1LgNVuDL$3_pd}`dwKas*So{nUBHQu`sF?aGEM({S-F3H-<<-A z6=Mr?bLy@H92oZVrxFJKM~bSdxSL{9Y_NcpzHVEkIk`%`#*Pj0Q5A?4UO+C57L%{*7=X9E>*!Xd(n6lDc1wt{M8;^PhH+i zqbt1gde@1}Tm-+{M10`0G6pFkk4ftU0*d1tmBXU4<@TYRUg}Qzs@AI1l!wevl~NV! z-3I`{stPVdt#9v^YF2Lm4o6~Q;v4(5TOj-Y)SD)K4bM;P@xT_ca_rEd9KseBpdexZ z3L)i*^Gh}00qR=i-n~nLrhjID%r-(&keS9izr~O5=yG_5;c1M zh3IWTs)S?|=C4wLh#HD#P{|Jrf?ttgM&Gz;gkoQ>#HyWaO&<0!wMavupj(8Px&cNf z*t%8bkDXLNqv*Z&u;|vz7dCG0FF85DP+}jXDBSXljYk7pmV5WAR?9ab7vq3J2`6>_ zUBI#ty%FxfJpx9=w+KukRaN}jKG(;9Eg%jMy1-#wRn^p$P4FRaDm-_d;AmpO4!rtu zoJ3``okyeb)cOA$SsVRDKC#gKMGA(`rLZg zU!2_NFx$S?ERyibX7=jq&JoLWN0qxG4O^l)26Qj=O4BfEB)<3NrfOhl;I_r%n}86Q z2x!B<*O7x*7+=+9!hfq>MHL(tHrwRE*-{8S=T)Eb@|Z{OuzzRRN*-~# zvYKweNCs&HKKV(u}^9*BF5rPKwJn zmq45ECzAbKT8d-T>D^aEfWDXj{t#7F?A}ytCgYpO+7Ybwk8z`m@7LOZO;P6X3+>95 zR^!?HEB$5O{Fv#Nk&Ok4uHTmc%8JuwN~Azp3$W%~#p{F^8EfSO-NJeq4AyPyDLvV_ z=B#acq10^Yd;!G~cmu?!A+JfQP#+hfinOQ`jM~GwCGYwht6P4ZCi^rmt_A!u>gVD< zum}2@VM17%ADgcB<#H8hpxZ4ASM~D;Y!wa#gTVSB#A#v(2p_C}74W*KQomUW1$3U+ z*x2ovFG}>uMnIVv^pVC7>mFg0%Dw}!@b)(d51rJ>9C!g>O?Z>3l#n}e11QpLz#c)Y zU#oAsGLWW!#Go^#yy*5ha{r}-#J#5?W@*O1%sy?W@_%;7Q6^V!q&X&jOi0*Q5j{pK z9wl#VAznaKP#lyYq3Y$J7*j|YR(O^4SdF^KuuiBlDVmMBSE5k7fXyN-E~HOK-X?hl zb4;0|E4d^Lv3FKMRU-GR0<+MIeAQJ|^Dr`FL3F<0LJ%Eslg(Zp-ut=F9t_R%Lw~T2 zq@)v;+F|BdVaDwp711Q0uHeSwds9ZGs$ERV;iY>HFFZ{aUF*gxgho~qC9As_x2@GG z9)vQ&8wrM9CmBa>%iEwBgZsD3k|__0F*+=O-azMga|}3C0b9(G_;bWh6PuWpj0|5r zQQ62UGFtDwg;%JKgDp1~eTuJX=_>b~lXQ?X`z5=1QbpJ+i(1LQrFDHqdJl&yc5{cw zn<$SV?>ukRfPkqp2txui6zUd$ue~=sqVjM%2C(kJKHLR9z2bL=6FD`x_1>;3Km`=j z6d}=7MwuYuc+X473y2AN<$jcme(tp}=#1LPcJ+RrM5p{&C~&}1*2uz-HEM$e2j)

%d#H=unG385&wf!b)nJw!0SK`^(;* z`S+6B3MR;|c0LtuGDY@z~GDd!>191UE%G3`Y$q@!+q!f)PM*BLaQ7a=7Ji z+)~_ugsaioOB$VQw)cQTZ2k$D{eaS`6OGyaJ~;k!EWb%u+tmsE-c`oD%A1@c(&MkT-O%X+J5fnc#_z+rMcGO()b9FpJy0>DQ{KOm1J^ z6S!R`3!)U(ld`~7SQLNtP<#{DZZNvJ*I8L0oEy+_dC^WtQhK@1KQnVb>xDH3{Zvv^ zG$vk;c8QpTS$!G~EOTs9iVxWEE$v zg1LWjMDa0~SGXCL(#T<|+g%jq(EmJWexum_WaRvJL(F$kkfvUO2mp7Br&o-0WD|iu zUVSg~eZdxssgOa$2i2kcHc2gXB77aGGw!ILPp$EB4R7G}#%O`b{K^Y_okbLzpJf60 z9ZS^@$>$|fuNL27?TcFZ^gO-Lg!b1LFGW?5+_3WfIJzo4dL8;bDPSEvm1em=Dr>2t z`7lcBLxH>v7|TuQyaR49c$+AULU7Wr43kh=hafVusjr_Dys&^Dh_iE;0_{X#V2w09 z@=?f0;;{p|r8{sy0;e=u%W#MWXe=@)lDvwb1=1>W_5P<}&*pTYo!O(u;mk?-k!{I)sfSF!IOwXM|gN_VAewidOqnf7Ne zRJ|*#hs_3XKG5fU<{1chVdf~{O#v1-OrBGQ_7wER6jwC;W0)A`h(D3fvKpby*}wkQ{$drTcDSW_h6;} zmaqND@!m3W?Ttp2%dT!YG7RUvsj9uvK(b_WMOexanfRRNw9m2;;!w+FHA7^hIq&EW z5Dvdm0-^v0Kor3JP~W$L#ep_KtcI+xq8ori6#2ean#p}M4j5uac#AMVs##CQe>V^+ zu2&GuI{0dvQHWTa7cqg+!W#qk6&{<#0yNeJ9Cs_rWC24>h??AzQtm3pEYAqY6$({X zy1a9gReRMmzD%aXs)`!u0WwsbMx`m-Ed4>;ki75dNl`E^?bYJ;VhDz3bm zMCYcK?ZfvAAua%2ui;JF9REvL=#$mpMHT$rXHi{H)0NjIy> zz#N3}QtFVaNW3^a-S6w|Wz=oVFMh={4DF-!Dv<)^IP$kSxG&V-6$pUD=`)kY$?TQD zQ(=8!W)s%4@SoE^5oo+tu%5~)0*d-ouZIeHAwdomY%Jng_5{xO{QM@M!c>Ig4> zH8eCrIZek1RTd|yEP!oJN%k9`n{4mOcQjVcIuv%FuDmlwHo`;n&f7bVj@}$d8qRWL zUztLH{%Tm`|5Fli~{7T7;yXo zL;zO^;AcF7L7DL%b$GflzkDC}*>a6JB%*77Y#&0St6Dg2Rz>udmNRWS-iHR^>XFPv zBdE!XwZDvv7@=qBRrP}ri95@!V)~b0P(6q970jqy2BQqQ42iTe`Gc@}nuP_l(<$b0 z*#{#=Pv0GWkIPjGgi)tXR>-s+BjHi3;^|cEc+ zf=bg-U{KkK9-2<*j=zJ)TD}E1F3rUtm6u2`-poU#l!7D(;siE`wE3rKkFs)$o>)GC zM;kZsz5Dkwg~j-bFVj@=lFsou1Ww!a7iigk9fEpQyxs+9ZWvAyW$6{8rOKXu&&+ym zaR1fMF^pG{wb{2rjB>Z0;)4x@6~Got_tC^S2iSGad~v*zbk|k>{Jwp}Dbna?iYZ-=d;aSc+$oiD75FOv z%Cm(SP(0xSa9m0N7$5!iFWz7EWCl34)tJ1S|^e-@o!|OK%UprNR_-P-F?tQ z;$3v&rd;Fw-u9x)0%(jRbK2onBb|by{6RjjL<2@4Wq1JLuFt%X4gdB))8w>#-!~=U zAu1}S!cKsy_~yt&JD1^mt((f2ZaDL(mg+q@MrJ^ zEBvj+pwEPlHUZpbo8OtQ7v`Z3Ig$`!>{+EtDg~RIwljU@v0w02Ke3H5l%ti4@~78U zK~#?oYsfV7SBYbw?7s!{Vsdp z-zz5IKlnBQOSGz>|M;5@htj&#()>A!P`IPoz=B~%u&9b;e=y^U65N&wr8obAc_T*9 zdiXhSGfQFw22>cu#6EvK06@?C8kV5%1(Hi#O>dl6CNthMt})z&L*Gt#4g<0n=)Z#j z7_y%H{WKr_Y*j_^3(k|C;yhy(OBrXyjD<#Bw!}%*pBl&9W;ZAW&T&;? zO~}9>>5mEFHn_b5e`zpvUv1_jCT)rVLV;H)#gtn(uqVDInb1eH$jq%mqwIw5a&c8Q z#5#-J*|ji`m*XR1sx|aV4W6!q9$XJmxG=YsXkI!z0vx9hJi;*eBXE73Zw%US@+8n__-3EqgB$S5%yJA6#pgXqd) ze=ej@5|dZZC~k|$b};Hf*V2f0F;r#jZExe1Wl;^tKX~7>t{S|KR%XGk9L$RGEV}ihyR?V>4iCks7rt-Nm zSr&@79Th71vgZ0Npy36NA^R2T*w_Fk8gU^A(hsnaej7gT{3`i0BtGO^=uc985^T`RTr`BP_$3aT9*^pal!5nt>32%O%U>HQYQ@|E|f?(G{@R)n4ZFxCj818vqdI}f~9K@t;`ZiV2<;Dd?EjOabMpZ zDJqs=FgI3wHDI5+cf>k} zxGngXjgZ%F2W+Cb%}Tn!RglU ziXI8lNC@|uhP#b7Mkm`{4i(o5U=VXl(odr?THXfCZ#3$|5WeeeE26cKg~~Mlg5HCReYM#@V3Yz0j9{0AhyPR8#~S^o7@B_B}c4UCtX1m!Ck#;m5gwC&>ed z-ESY@-;WVU60pO~tV}-*`E8{>F=99KqY01L8wD2h+oAsds~vmE$nXAP72GF$#n*RX z0d%N;qnk`1M99}DkSU}eFCf2Ks}X!VVWK4}(#ZCIP*I#t8_KA_*6$byyaE^Y-l3s* zH)-;MDA3v_0vIa8i{s5M9ESk_z06Jd3Y25MaCHDbXT^Tdd0Ywceg5h{zkUXby@C}E zjX0&lg<3(F7=D4iW~q&MEjpM^O{A}$R((ilCfgF zWs?#he<1$v=k7<<4ECJTI4{!wE&)O?$ZgQ=_U4+;Pdf5t^#2M~KPl2(K4AP`_JT1?wsoEzK0{T59ALl#Mh53>qswZ>%){`rajpZuHS@uR)l!8B%6A)=)B Q@Mn?~l@<9cr0@U#02qSE$p8QV literal 0 HcmV?d00001 diff --git a/docs/images/solution-baseline.png b/docs/images/solution-baseline.png new file mode 100644 index 0000000000000000000000000000000000000000..e7030c81dbfb175c224e8d9082eceda61910aafe GIT binary patch literal 75802 zcmZ_$1yoht_CF4vLn95+A>9qqAzjj`bcu9#cXvxD(jB5AU4noDBHba= zkduVe43lkvF9=o=$`TMreInY8DeP8MC;4Zt5C~=u>>sq>vCIMjxi40bmeBGt+Ra1G zdMI<%hZD}T5}e$kNh_xw@_-Q+JqoTl1m!_@N>2)HVdNX4dn8QsT*__YCvUS(s3frQ z37jNU=^QNJhq`|mX50&(Ob{p^eB~CdFWtBmUD6!UEodoizN-9K$)HUghK);*E8uK) zzB|Lb_LaFdt%0t)-Tzu`y++4%7Kt{f%CN;_Ypi(HoOaRgL|K#~p@5~>e(r<3djW4J zL@Bz+Li}(v3ois zB!nNQ+4x=JRdzO0LfXqEJrxwHFzi6?7w*V~QzZNnRL=RU7D9!`-Y>F<#rcQU! zhF$pT1;>v1>{T%9s)vKz9D7qz;mNGgy(UCDKS=gz=P<|9Qny}*VN0T*vUl8`=LpzN z)!;LX2{^CzN@0m2Y|Pr$rsH0l_=HMWjR+rzINRN0N{{4=elFK2elXS-Ppb5xtPI6k zOH0e^HMI3=b8~YzPdK(#I21}?oH>n;UiWS(!NJIDCA<&%L2BuHGT_}r|l%db2mlygN2cuXIuFi3dSjEau;TMn2Lr{kL4cbF5gC0Pfoa#Bj$ z(`&hL6y7BdU_{WC#eLH|wOo8$h^OC6J4tTwgCakH$?=!5D09I?mNfC2!$LEgGxdAZ z5p9}O5-g>cLi69HEJz>Q(5|Bqvge8iwqKr?EWWv%>JWI)qn%KN&YA16H!HZ)^E!h| zfvumm{HgSq#pYc}&v_*})k<#zIfswE4Po!#7q#jH-*u;bDp40_rkn<}1ix0_?-BmC zNo-<9X~t=lw(rXgu~|Z$yZ0un#MkL9ANFu1ELkh>h3FB-;q|$%)!by5fVSOknLn?Y_LrPyH3yO7MuOMcw_i=2rLYTu?3;Il7&GW*P{qbdJlkUG) z51#IYU`xuIjGnNC>0DF1nnkSVCMvy>!mTCE^xBED5F2QqB4UWMz3n=xn>5F}zus_J z!_IJo5pI(EDqh~}*uOuK$z5OY&3P-IyODilclT<@Y6`Vzh8}$vw@ZpUDtG~)R5L*R zNSg*Pqk5?3qA&hz(Z=}Vj5NMbF9F6JbCP#plldRenn=;DB z{pdOMJ*0I_jt7m7rB;(4TeGE((lyiJp`6j?I~ud(Z7~yrA&-)(0g6`m>wBQW?YHOp z0u_wX^2YRw^;_@m$$DeE-mDJxHFakSurs`>|GliBIuuY9?i_UD8hgfa=k$GGn9|94 zy@c+Ua_kip=}~w%V!v@VL8Klb*sgq zW6r0T{%&c)Eh6nh7h{%Nk5v#~2;`Do+7J@-KI~e|-ldx7a<2fqHkAZlb`(lSFm;N{#v9A(?qe3JE^ERc;S8Ke<=XEj2x{w$KqypJ*QGVbnH4ozJ{ zgF4~nu*CBV*!u)=ZIY--G+hFIGU4ptZ~e*yN5YzsRNR8PIyLVawH6fDB96`L+mRw* zTB#g#-M|f|Z9I67hW2)<%PUE{aOUm61PVa(E?VKY8`MCtuB;24J zDPj`^D7&e7B<$WTw5xzM+J0hrL6<@4@VmjMNP)kP*PABeGh6;xar29BuU1#rI0t;9 zW@WNy{1xQopIpSo#M13h0Y6~d%FhXwLxIyLt(d=_V!TclYb9 zolMqx1oDF>Rqu8vv~N@+VlVn8k)xhxYisMh&4kTFg|>}N1vB*r1oJ8pGc23&vS0}r zgrSeJD;w+URIiFNK!gJ>2KU()WK^xbXf9z>N$-)pK?mE_MhyAR@9=MsrG!^)<^1-d z5*tAXY#!arYM7N!4C$Y-dL7>=6H9P4F5JIb#DS?-XA&$>@?35Qk0C!`_UJ@xHOJ|L zXtAQAJm;6MZa2+hxm!Omg5Zc!yGZEJI6~skD$;9nc*jI>f|pZFS!LeyC@rw1m28PKNa49xCVtn zk*6KQ)*y0na#%#1Pb_bgqmcFd#4)|6G3r{EwJUWLCJ(~Ag1Yt>TGDmPQ|``#0tg%4 zhAlwg*EsIZHor^d0u!6$2%N*af)d=E$?4b!PqC9n60hRg&{ zhj^KQOJVRNo|Aler?#VgBCD5w(MxLmHj)mB$ln$x)$6PbVc<$K3)5*ROS)fyC=aE> zygxzS*|^PVMRF++X^0tq-pwlnz0R%HsLf|@_Wj9Q-xcA`*bb3dzn=p$e* zeSm@&P?kZod=uj2=a^`}n3tHGksPZu6Lwm;v?RmW^V{o!x`_0`pYaZ456hhv zSR|Nrs7mNgLtF}s4{b&qdG=$!b9?1y?Mj4-GX8y4oU`OIW_Hfx- zO9Yxqf3BI(2za}0X80-$9D{@S^S+d0tt=iWut+G}m~oQzacvhII(KL{bUKc5{aNpX4l zmz-6I?w5aGRQCuFh_Vhrh69YlbmClwTx0xU;bEP}PAlL0SdDJCAmzM@q;kWiE=l;C zEq&zESc;|fYmg6o4zzr3AhCGV*ehaa%L{I07V0Dya+~f56#wu_$i0I30VE&&FwRK- zko(2O^)pX4juyS#?vS+(yTMMu@7l7OElyHV$d~1xjVaR!uf~(?b2>-gmIzSq3G(AW&2)G(G7j^!quV9r^&n>GtX@IU_AiUPs`d^`t_ly4f;z z%}r^bHySTCCPqV3Gn3sgrlME6pFfI#!-(~L{_+5%$Enx6*guQ|J~vw@2x=r2h=oK{ zK28F19SkZ8la+d8^2|LnbI+&y?Zod7OJxKjey=k*r=)9<4!%$PW;JZgbPX-Zd;P)p zWi3CKy}sCSWCVrA@K6$AqEup=CCylCu31sDXWiOoI-!Y@QvR&ne)0?=lh#2ix!&R3 zleHp;oU@$>|I@1ga~tS7J_xCNTV1(qJTgn?Bu6AYR@OLlWct2=P zoK9!9=Wbl#?=;Gpm0>E$i+tl-bLbS*#`7MH-zg|mq6n+*5$X>pCfQ#Fypg^>`Ao6z zRDLcB6UhIO-F)fDZ5yaK}jgB3!-L1_rKFT*!rT* zW%b%$<>2~CacYzhowIpY1j;ZHF>aG8+63H4oUUNYt+8Bx9I239*imuMs1y-j_EIHk zGjgc8-5U?}80(D{?#{7H^}>EkVj%-h<=xKn_&ITJQoJnmb88M~;1^A5|I%p!l6D|w z(&cLXkBXBWVOtR7gd~i*qmN;?^1YZ&etxaTDs&d`tpT?_16PEqNKMbrQakGQJzumQ z<{zUYT(K^>{ZQT?w)%o}+5adottv2j8fAFM^;S3-ta&mltlD4_ja6othFbz(CA5a$ z5J@AK5M@qn0dI#;dz^RwtC1vC&ogOl$m+Pc>(yi1G~L0C*MT*y8sqGWs7ugx*B1}2 z7A^E7f8Z#Ly}wurG$x}74-X$?kpY_qMEusl^>W!tj*CP0-Wtl}So-8@Wn*Kbpb$;* z+?rlYt%VU$d(ahE7M|70{jcDm+mc zXP-AIfDVgzkZB)Of2$D}kT4QTs!toIvmq?b@}Tg5N8iSX33Qp^+9FS?m;Pii^wwhq zsS@33hwF&V99LLXNmhK_J#=-|Y+dj=T8T)vaWM1kA6c04c7BtiUqNs5WmI?#{c1G# z!5;4sWt7CT8I2aFP~Zz658;-PHcCxWzpVsm>OKX}AzLG*qR)g_4S0UW=mD0u-o!{f*J z2^EyEsvTA~p-^S4hT#bSef-cO=h>|MY(D*UhX3{Xmi-DPOF;PH)n?dqWU*2vH7KeL))mu!#?%Lds|&+GB!o>BH3y_LHVf~Lzdwms^Zl{@d+jxCP#Gk^G)@` zI<@4MQ$f~W<4T@SW5&HcmX|hybc#(u`mB3#`(K^~RP-eAlMRa|=5q&}%^7>0BU&wB z`-fY!sJPz1>*)1B_^8eOwfj@c`6fc9g3`G#vj()#=DHw!W+ZN0**~<=gu5Y_=0u&c zPeUDeC%O|3Nl?rMykx0Q?Ed*g)cHXs(1UxI@jjg=D4sLOUP;jOjW{CPv&t^WIoZK* zAIJx9fdR_bSs9<9QQ8OMxj+Bx zzWz0Z%D`X{6$_NLLx!o`Ix@KM2A(S@{t~zIN(76T8=;&l=#k%{gaFDioBjv+fvRWk za?>*?UEuYwjWZt-5X6|{JguEa_`3lRi+Vq;#;}O}{)s1WbwEBOzM(^d3$JI+Djsmt zbX)~)b8QtSjl_CHPzsXdj1(QL=uOB6F+hFv*LPiz54_-$DP%6A!F04-lVSZCDN@}A z;OdYye3NlUKy@E2$==Cc2DUx;@ue3Z0jpk+vm4`$JXXQR?@%%}AHsdJPEgpNchD}aXV@m9wdcRXAaR!O@_rFsQl!sf@iCA>zit{Ma z(5`;|*Z^{D_1kB3-7E`$J{D4l@Y7vO_jebQ!=^1BRj>R%?p?mFKw|>iJi4{xeY8b% zg<5O*MJr&z-PFm4xHBUHx+|FM%}3OUnX%`^MO%V-K=GG^*g$x%2@VdbZ)_qjSWkz? zR~CAJh9K41A8`hC+`7)N{Yb69#Ifb0X5ymJMb@W$O1>!iml*`AaF@3jh`1kJb*=ml z{f8UQW{!E$UZ4xw14L*aq=Opo@F>3=s^exhY^^QCszqtdM(OM@mw%>jU>ltH&wFTt z=yddjmr!?)LMcq&@tJg+Q;5 z>{|b4H#sz9@cYR9Lh^#DI zRvSSgHQWJ&m)}r3j{-jp{(eZq2yKAQQ}^N2xQc&~WTy7}wkiFaR;P`BcI3J6GdV-M zv1Wj7C3NB40+{7i~O(S7+o04@+4EVvY@fTj~bXTK04< zkX(NMh-LhkTVbZGNLcsWv8~K@@3;Ln>r35@36KBfuE3Z7Xdy79>4tx(ChAOv!038} zqyy7hz^86V0s>Hg^+u>j)k1U9KRjJqo8iPz9t9H92WtIvgZCwjF+oU&Y#--CP64hq z)8fg2$qNve8pD?SbAx9DFs&8|x<5I+PZHyLNjg?_3RmPk^I4V`777 z#|bnt=jqnY{jo{_D)Z!hr{=gm2J8ZF6f(KH)N#KW%TWVE)*9r#4h^M>56k4f_FVK< zS64suJ>3k}y2fKr2{V_j04XtZOFHkHORk`)8pU4kpwJ!DjSFI$+1gAO;+I4b0e+bW z$!2MvN5oc%cGgcWZa~i6FiobdL7mgY1c-3!TYaVr#n?-Qj@$Q>N-ZM9Yxho#1A zX|H#p=4vx~RS!PN*ua|#oQ7xKVAPiGw1pl#`W@BEnP;?gG|UlrS!hS~yc^jFD_@Mt zNyu%p&V6T+XGBD^fyA0JIL2)#gUt!MY?ZPi`Pa{%lrknt5JG%>HltRe@GpvAX@VrG zpTDOKQcpkvsZ98RC-9Dm9GCKB#L3}2o@*r2mkJHJ2x*kc)2VFlgEx2|wrN-AJNK%vtNT?=DVU@vx-z=imXiN(|S!UF`Gza~e)w zqKPi9WwZt-7*WZ)h_f}bAyM>SPQJ5?LS#s77mz|}vM z!sFG9ya1{N|3k3tqvvY?mX`Wnw7^P$@4y{Kt<+q^KdptTRY+Mh@K z`64k%oqw7{UfKK&f_I(eH7%y`!c-JuIe{nWZDh!ao z)rY^z0C}5CGzA10hTJrv4TFp*_mtX z`qAcnujYt6>i(D0GE=|Lzpsjkn;5X^GGX5dq4pqntY%k%ha*e?46P1!UiKF1`tASYR!V zN=k0^>&1>ZWa`O1GBQcz*8{*hwq&ACORl^hk-iJqJTGo`b8KG?DxbS;pmCi!-2b$H z+LPrklbfl$6VK$mu*$#%EhKy2r~LF^o>O$n=5V`TW!0A&^MCelGlNqdopnkA^dRpOM-ao z@04SeNDOvM1Q=dZ!kv%v42nMXyFZ{L(vRCI_(qM9K2Aj<+B zChd|mIx5M(;r8gru8ay*X!}s*?kF)ZO?=fdx(On-Q}r&*>Pg;^#$-Q!W1K=r^*x<2 zJ|faT>`jEW0z8q*^~dvg1v^e&E*2K0bR=+*0r&y!{C+@BXu|>gP_ z635&twaAMMdGzImFXeIGkMX4FN`=`fy+&iLBe`Q&3Y}(2q1h22M+hxs`S%sFJ!FB5 zEoWGAyfxQ!wdJmSuUj`cA82Z56vYRs|1~GUbX!jXvs^cHG>wM!i?uP?>33FG^oB); zGA>XO8f8`0$ZY~bLYmLC)$hjo!a!uMQ(gZQ0))*0op*{_!?M)x6{vx}bUZr3bIl-2 zw(ZOddrQX6(r$p;G$-V-n^n{VanqU20$YdKyE}Uxug7rL9))6aKx>b+z-gKxQ>(V&f`nQ>f{d`tmDuf{Y%37aDd{f>#r0kg zPirygR$zp%k;Qey_u+;loa`UOzy;~?1@(LWe&U_ZY-mvZ;aG(YiyMT-lL+xhzMn*})LI~*k%*W9U7Wa6bHe_KiAjdDu6cVZ+9+J|$6GKf~Us$u2# zGItXtL^+`p5=#Fn{tCh585X?wNjM7Xh|`(T`z$*b9_$F)=Z)YVL+5<>o-W zQm0xi>(wit*_L()PIe|Yyg*$2^76M=kCpDQ*e*GPP?@lP_p1RrU|?Y2;22@-hET_K zgVI2%teER>o%f%L@1IY&cOti^KaTgL-^2{6sc5@3u5FNvzVX;gHNHDX*nr}LDp0!^ zU*ZG4oFYoJ!0E&RI`UMqn#w1HI>Jr=$*N~R7Vd_S^}Rn*QJqV4zwvSJPvk@aE$Vbh zO1jm}l=rq%uN0GF;FM6kPQLNewRoSrV9Ms~I^t0xNsGe6$1nD`l0q)A10`T``WoD> z=2Q$#`IcVEdDt}Gr@^i-$uxAn=QCD2F3xe!%ps(X1+e0CU4yz@;Mn(km?T`wTB{t- z!u^KdVP5Y+ume4&#vakTE4>sFa!EVHFi+4{UmY?dN@$;Vf3mGb<_X;`V_(oAGYkoO zC};k)+mmS-L|AR_o<)Q_{Qj(o{uEs3Gf6uXc86$F{{kABsF=_)0^}{wkavUM+qB0C z#_jpCnxM;%U78E&+pD>hU*|YqdDF%|6>Cv+HTeYg?mh?~sC6TjaXk$#RkYNtIiEn> zb%Y@mEOfZ$c|z`z%(z1SS8j5#nsYtQ#GNF!;VuI<>zPbaei99A&l^YE1^TLv^fVav zQ2(#AqWp>8HHQk5zP}P`d9^L#Y)0bwr_z1uE;i4Fk@Z~*&CK)Z8mLD@^ay?g5i7r^ z^yuf^m(^s8vP^c~=2XL&j-ne3RtPOndLHenGcmY|okP`AT?;K#fH#7llR?9%^oMF&l+FCX+;7v3x*el9wEBM(A>n!?5FR`w6N%U&7RY_AT^tY_`x;>os=*iEpWk1u(SJ!D40Sy)ccyPws!@UjB&|5A@a)qD3fS5PxR&ErcF zRYjCE5z*~VP0KzgVzs<+*bv z;JesFg;aV&r!K@*!iNu|PA=CHV+#cV=$f{HFeL?sFKj0&ijQe&8y%M@m~=WoL79S! z_dJ)&1Y~d_9bQLf<#o8RMErA(|#98N%xO z!G4Z@rePCD5<+^PmlY;l0-2<_Ui3YnntoAbhFiHcRuQl#q(kzjW zM*Z5fYSQT9;<7BS*qzt#fR)v_ovd6T7^e}p)AwTkbKX0FVIN$Le7|FbB4Ecd&p)}W z1B{0N(~=K&6nc}{aU+wfUMNciA_a>ETWjz3*Vp3rS39)_Gw({HIJ++wJeSg$&CKH- zuxg%zsSLnxKLL#V+ZA+k zZHX!@S=-HyF;2U>nnYR}7Grnp#ei%3b)4%EiNK9HyP~=T&6Fgu<1aKC?=1#We9pG1 z!k6mJ`$5|$nu7_ItI50XB}PsreW+)@xMos~o~N|7Fg($k=Kx(Mt3eYslOnDCR88Ky z{4mkm>x=$39Vf=*oe2szGZP85m!NmA2!Tg$Hb9ck%Sll?=@*mh#eDsG(aj64>V&s{ z**(9eJtZVr3KAM{d$k3j=UWR#rs#araduYE-|ja3N?GtrK-+E8IJE;^P^ETd3B2B5 zF%*?+!Rk?8kb72&--UVPGB4SyH-P54y$f@qt3dyxvuWRKr<&JEZ<*!woYp%pOK5OO zF_d2l(ktL6qnIDInkz@{MNT|wG{(pI!)0C8xg#@>bmwEQqLnl~cvaqmC+XMqP`C6w z#`R>DVk@Y9B%?U8R=4GiW{bj!E7y!B=@?XUpIl8G9Au=C-Ba#sqncQlmOTf(EYJ`- zj&*TYQc$(sJ`BS)9>u|boj+$6w#MUT_E9wVbnnr@&hw5OArY!xLfTv#3@5+W1u{>~ zG-SDULl4Mbe-6IQxJtzo1V#JbJ7^8``)F%s(oR-><=*Q%ODl@qe$tD~9az=i^KbRV z*)L0`&Uuv=G-o}p+cUx@5}{R_p&srsK1i=ohN+5jgxEWl@Dl^K=KhX%ry?2k;;Ueo zyWNR8rm_Pkt0c-&@+_BZ?burrg}6zRzVG}& zp`AnmKVn?+XD6tow*BPN=snNlRC*10$->yp^h$*e8`Iw*MXdaKwq#gnv(}EKyPNAt zdBR^F{znE9a$2EO36Ow|IrF8yU(^wFZiwa((3T4;OVC)8)ScR9E?ONrVaxi5S0<14 zefEas)>^(PVy8lCS}tWZA=g9WE2vcMCNV7bs@1QJu8a9lK=!5lzWuKVo;q7L$YfhJ|sVn@DyOA=U9`Wt2$3RPE1_Of@Egh|fw=p8VMYXHZ)gk=96++P$jh zSy_?jfOjABA0-0Iv`KVZ8~352E`X+Dk3;tUWnwpI{ECy^7)0gg`)`1gC|O;Cck|s{iK=3 zZr&Q~Lf2EieNm7N^DXP-pC_Tiqyp?raJ>Ma%0ax)-?bJ|Oh}#PE8$p4n=&XrSq6CM2(5T5*K< zLl6VFpU5=OW%~$bU@WPLevBf3m7+PahOL1TiMatFS*Q9r{_v-be0|WEukE;l#S2i+ zq|HWk^YX$pXceswubF>;b4BVma8p5#E_zlhPwvLG#RKwUc-goPKnt^3&F=X)mOOtz z?#=g|>-hEc93j7`%aajtVYf}Eo$8Mje#MGl80OPT1nH<}dLnJyS_seK;JCmiB-)9H z?{Y$55~@$q-u3NA>RU_Ai26D|%}9p!HzU6^r9M(As}APyjB?=;wfeUtZqI`0FlXAJ z_;Q3pmLIDbBTMkeq=R}Hj@yS+X4mI?gRaK__}`oRsA`FBVQEQKtz9`eR;;wbYSikr z+8e#9L~!T5k;m|iM%3r@NsMNJCQBjmXQKK_qZ#3r`;kh*-3dCM{XRXKboI%1d<}ti z;?&yHSnOF`{ElhNp1=V8Bwl|r&kkRu43cqN^8V$?gp>OHGheBOcB=$piXfZ79&vJ;@16lw2YZ*lfClO^ z1}Pte_--c@K^!WvxC zmVo$_b5DA@QP642gu+-Sz%Pu;$Jjj2bScUiU6@~!ewRH^GodUJm?I=rgbV3Xf32uuznK!5-zb$-7^lsTLmwNvryGxv7K#--2KI{0uE!Zl2R*-hA)`{r|d# z_SMTBg4sT;(|h*65FkUliBKu<2bb7fwv(*uGdPQqcsr|vh#x#5cxJ9;AoL(Hb5fbc z886Ddg~Q7^1aksXW`78Bq?r*85?*wEeJs-RuS+R$eLxzh)U&_Pob%w;0o)e(akTU^ z0L*?A+TPdjzxwmTUc#pvZu5J(ORlQ56r%g>J2i}&=^zM+Ngoo-Mp?h4*yN{VuCThG z@RVDanb2A{hPe$Bkpj(TzLxvDfQ!Yc3o!F{FuwIGU-W>S$3x`nbRP(##?ED(0sR8( zp-5PJa~Huz7mJ=33m5Y<)p9$PIq~HDrei;?`}pp-VH~`&8*Xa*K59y%7Scq{e^)?^ zB~Fcj-I+M=9`EprBQ-Db0dpOWC}U4#VhjZ@0^O3|$)3Gov%^w7-rZ+H;}D|BKiDPS zQj+HH>2Hta;wfn8z}(at8|d6yYZG zA2p6TwEK6@9xP;^s4YH0JQwDsHV1_bIS;`Mm=GYd-WdKUz(O9kr=^OO%SY^4v_l~N z={EHv*X9?qdCN_n)blmPq>TNidwG{3xr33iLevCc9+#Xc#N+ceRx!dyjjK3PHU^%( zoy->|Hx`tiu{-50SxsdSy;sv7&wL`hSsO$eJb?o8h?A{($Sw>({UYTvW0_#QLB{PS zFve#;D>i?`ZB4U==Iig|3!tN8ULE(KT^c+sF zFPw3am$#3|%2fGxXY@H_@W~X;E)`wXy{1*Be55FAe|9B>oW zRU(;5zY-(Szm3}C-`a%YIV=?9jFB}<3evL6kgZ%K5ZR>=@TY-gz*2`^DQ4Xo13kT$ zSN`UI69HW^=DH=Wzb&PKY^!|lIs1$^E(2Z)>Sm%Lu^5+^t!?`12HPKjfS-8>1p~NTo??^En*6WQYv^MWj&E7NOd@mX^Y83 zeHvFrr48VF?$STFtSf4hK*m?{s`TnZ9`zg}xwB`o&eWUBwMit5+7qW<{=Ck1GC4!~ zY;O7Xxd51P2i^C!!L*^jgT=u4f@b8~2vAm6nIkI8(VEVKQgrkDWsCv5=!?>_Z4Cg% z0q}+O4&W{934~=po7Fa422%Flk-?w;R{aDLI7TK3m(?GST-9BcQp+pC@enw%jU^vS z#`h6vF)^{Sy4m@yBKSE6z55Li@P9~9$}o65hSO&AHCLX3Yc5Z6Wt?O}`dPJ6ZgzNFh?dxYoGQNYlMjTphXV3WO2y7Y_ zpFJllArYysURBuUJuZC%3$=MebV{5mO{M{dpBiyaeysQ%y_1UJ%4O7LyNLk_s0oBL z8Am}$sYh=_KQ&!nZOl)ryL01*1}5Pe$rGxK-m$(87=w~Dy$_cq;(kmNsw|3$9#)y} z_xcd6bFbMX`QP7OsK-X@D6urGdPND{lzZK-NUWl9Q)AsTpqVe&Np}zXBfb{Y)tER~ zNPHO1X>E@1QMXT`-_Q*VP7QiRz*2CSe64ZT^zNVdaxwS2a1k(Uqq}`McwCsOg~k9T zt@_odf>sC#36&tEk8B5Am5Q!pho0HVlIQC|h?Bz*`e~j{AD_)MAt5kXi=4h`#5q1W z;W}-|dj9T3{KL0g8lN~%+Zvf{C1D;wC{rn8clN!~A9O~xElteNsM*4Ece>!YKyrT~ zH|G;Lq8+0>G%&F>=o%^{QV+3_&tMfys-gB+?T*`WBy$kz#u@u2o6j8zkHBW80jFo4 zJ4k2t(Q(P)Ylh6<7pOFxkgrX-4i|*Krf9R5<-VB@iCG}>P58T7u3OF)(M7vz>ABCU zT5MMDq&cpr-_$7DU%hzt0O3Q^rB`|R#T8ue-v<<#xDDWuG0D`5p-=w4Ktc{i`D0qx z|4C}!D?GH;Isn`yDc@8_DTKH6NxQOV`Npd0iibopY9jl)fAukXtJA&MI*w#jTp9cF zEf3WSG8|EX{?1#1!+P|H(5eI*$(v}uB6>1)@;X{qpU)p0iQ?y8OxLM6CLA0GSJcsN zhec3Tz=vJpNY@^tHyIG&Nj}`jwf*#Z?Rw-8@5;F`Ei89nq_tc`k~Q4;PU~|brH>`u z*NY@D(Uk=G0USw#st)LNrOTnzbEB9PfnI_n@Rt+dRLVN&ItLMg(a`WP>>X+-D`il! z%qg5S2Ygo2VZ?zxu9-UJ2Y^qi5%a5sq1H13KlS7l_>-Ags@vLtE?aQKPEtUDkH0=0 zfo+BZpwbU^Gt|&3bM1^S#;=2e+usW#F-?}|qyFD;+5e~M@ZXpp6>ay+xet50Gw(o^ zq~9iVpP2wd0y#lffB0V1O`=o<|FXsEy|PRIm|_8Kz)C-B1u!_OEe6>I#Zm~~S09e+hIsDVivixgtf zHUG&qp3bT07L=AYtO&5mvPx}OqFD4C{T>vEaydVL{sg(MVzp8uq|%Q`BkFm&X9y4F zI)pT2F7p{ghAYEFJjMTNtMp7mU-9&TUc+)Da(E0)>{QjU<*Byj z@Po^5{Hk}Ur=Sl~tY(C+F`)8V%#cK0=QZ7YmwdFv8B1~wQCXpdyYE2A@w@#i_t{%K zz&RdDPVRIxjWX`yXV6Lu^McDxOM)x}BKgS$nwLkxTBz%f9zKy)rlU8g2?88v34ekw z7+klN$@VgNGNv;4g%W-eZliORMP60{i`XG8+S6Oyj$GKn^+MdXq=bP_I8^l;Uk0;I znx>>bm#CGaV-a)8M%2^JQKa2b32GByJX8sS60B;l4>@}`q7@OyPrAKDaFf_Mf}MlF zJqqS!jICg|f5Yw2&{+(DAHW4W7N;b#Vdn^KMP_;_$Xie`BoD`{u&%^q&5%@O!umb& z4U3Q3@+U}(w76-8rxD`XQ0H?^lCoAje@7^B#&b$Kdgd#X=M8V{I%sV}65sq6@bDi9 z;(tCl!;U<1Jhgw?9jRVft4TpsY8K;wr3}757>1whey9Zwwq=E3p^L0yYP$Kpv@yrk zaiXXrD0$Ot8P?+?K2fl~M~>9$qnQ4Eyf2-3!p<+?_MDL97M!>csI;-P7^!mUYa4l- zer$VvLylR&ULBZU$b>5NcezBpu>$8D%!N{qO`+kHESvY`@p(=ZD1-u7dT8d-54SOOvCr!PYQ;Ddh?7I=yUeA?IJ`H2*Z)^l%v7 z6uJSmrK%4rvZ|0&ghxme&XEbW z=5b4-r9Il@Q^m3w!sZTCmo{tx>^>mpx5Qy`FyjfyNyk!t!c>4=j>ypPt;WEsbkNa{KR=hrm< zt0Pycf7vLM3r>!i0)1|l!}1kzTFmb5~O^1Bt&Vn zJS1rK{~qTHh9mQ1?4-s1yBc4YEtei_8W7f>Xy|o?T|u~?-2}kM+4clD_vzMy!Cyv= z`%nODI_yET1Q^2qepe7l5yC;Lcy)6!((`HaZ7f;*m#wl+0D%j z0J+SWoA#S`-WC>u0RwcpzHm{;MUjap1l(bGzAikjEHQtfR}%n!UC;XxQSz;}?fiKf zIyh^y9?c>mxUml-8R2xc47hJ+=Of>Pw$fvjM#z%O5L?BU0na7>%#@V3+_VrNHPrRk z;GtO@8re5DV@e$19JXeRjatMs}67+2i!D-9OFd$h*WAwQ9eaOln=i6WE5O?rh@>;`S!in1(+V=(xY7oeR zLWM2MC+bUSly$8K_ZjAoisgVEB`l-V9c|+;xrugMEcy}}c@C8UI_7apj=7{ca)~bfCHL}U@$r88eZ;k{BwX&;z@-c6Bid3W5+@JH37Q;irga9 zXEmCkC?kvUJ~tB!KCy{L01mxP#O3Fg^SwC`GMTk2ii(PEj`|}Tpicjz z18h*7monfV?gPtnF0}FKr3Hh)NOPc3sJo6qX{J6=rcRU2P$~(f52k!FB@3E^PM<;P zVz7?U=&(Sdy$m+4D}&?#vOsYanHw~~-k@c5qfelIL0~iX-FuwQ(L-#No1Bqx4ywI! z5$D9njd?nl+$ar5l`>enCTy=1AT*%jkq!m@UgdQKlVuSTyjJW`(6TkH6y9%wzZ);t zT={4}r&AHd;N?9081}G<5h(!FRCIJQ=AC**zJ0s>mE%|xg8+pYA|V&$q6FA((3I@Y z&iW6i50{sv{{Mf?j|%oH47x;nz=?{`%Eu~+P5(6Ez499f@}qx#0*9F86|;nR3C_Cx zraB)vTI;VAVS-1@LV(*bAO(liFI`+EK`ttn`BVI2-nD|)>@4ddr0^o!)83ulT2_!d|6yn2R3CuFTeh!rk)%|!p|ED?^nJM9?M3)`1l&`R z&z{h`pOwl}aUQ+X-~=eR+-w- zoD%yJDH~0CeieU=Qs-|q^St3%-&V(jw%b^YHLO=oAgNrP?dXCTYJ_~0x*~9HKJDT6 zDDC$*Qw2ppgc3r5cYZ}Hn1CfA*g!6Mo<^};Oi^wqhb{NtGRmeVofS3EN+E^kC+;2P z_PLI_paRFld;uTUn&1#7QUUt6h{nGVtOc#dXsy~N4rZA;do}$YE+gItRL`K`zhFo3 zD#10Ge&ao3L&|WK7>1)%a&2t>MM39!7S$H!-op-OFRl9*K^F$v;o&dF9(&$ zhEz&m&0&tQvyzf>AJP zM0$UFFxwouinBWvr_yR;A(?85`c0OWmkbx{o_XV1t-7CH#hYeoUA$!){Q>9E!zDx_ zbaJ7_I}`Vj>)m4-Rn~6?r@z&`F9>Z{4+6OE=RCq->eu;Ts;`@4V$W--RMS`7`{k-F z@$tUnUNG`&?_b0dU-s34qDwDjg2KVv!~U$N-360JBREb_@hT0bDnU~bGJn*n5Q2wQ z00iA>+d~a|LMDKCFF?;-0Wyg6YG*x9L3#(!D(%10&_VpzB&P zB=xdluKll#{9U*{4YEPFPR+(7h`AN7{G2ee0Tr$at>fy}@L|ogplXF!l&`jyV9Rop z{8^v>qqm$i44>?%6mQJ_XBpHmNZ{2DDe=s+w%0=AURi6TD0Q+uxYr{7j6D21sm6pk zB&5|%MJU#Sa5qQ$GSr=`gam|NbiTw)zEwkLdLuBR7lqrAwf(_SEl7cK`MoI)BhZB# zx=t34yPeL0Roi}g1JT)|P=PvISq3A~?V^DxsHtuu7$5VKH0yJ2vnR^_|2}Ii=lZ6K zy1ouH-}D29`iCDc?R`x({*2pVV`27USiSAHH3r%YmS2OX;dnfcbKylD=xioGG_ZE*PKlO$-#vzwX3ka$ID(10j2_~y~~YotuLlR91?{B zVhqtG(Q%K9ZNzo z3Dtr^?C~XyKo5q_(HD)p_$12ibNqj^nmQDikp|!_SG9gEp;2KE%&hv>=inGfTfCCw zC6ME}!B3o@l%|eu9KfbiHE`JbwUxKpK0ha;uW>gFKD)zPgRZ?Qx*n6Sjf+*^G(E1O zDNR{J21>tzSc?(+1WKR3N(fEJ_mv-F_F&{CdF8a6B<0psdlasB7lZXGQL*ga%xz)H z;pOjb%4&PlKg9FAX^rUtnbE)U5y1)l@!)VOkd>J0K+U!kvU4wCgOfVF_x<6~L2mXK zH+}teYRh)3hg6>K>}0?v`@PKB@%*I{*l&&?Xp}-7G@d&SoBY)9Yt6S)Wx*4&bSYb# zwfI1L42jMO{9cViQ4?JD!`Z)F$dBr>M$}{F;eW4kI^gMD0Fc|$O*!$Oer;@?Tb?s| zXM6!Ss8J55a(If0ica&SO9>S%hq3~mtXt8?ca@KL_p8b9+S0~*E`m&xEcQ0}eLfep zU0#v)d71Wo`>gfRKNRjM9Q0ql$UzBWW5o-^G5x=7_C`qIni(lL3e8h}eIDdI>f`u+ z=~d&qx~4;~0p|k*qJ)hD=+MW=&s^=_SxPMRS!$~C6Mq&f>E!&cKC*YN6VZM+SF*45 zzWoq?BrZrHJ>sYdek+D&>FuJOsa%?PT}4NE8{+@Uzw4oiy}#~7=&2Bb*LMtm9dVNi zcWbeB35^?BGY7bLUDaQX*(ar<7-NG$~K6z0;EWLoJHE_Vg zFaS%handq<;D2f7fP!Bs5~VbQkm{*_@uF7&wIU^9Kb0>{Qq@>|3Av!I;^U# zeHVrWivF03e%rTzv#C_in@*}?!>ba}k0oP6JhgQOL>xt_0m}~3p(Cg>9 z%UoM87g8tn1)rmK_DLMurq=JpORW5l_dHL1{N%BbZo<*Z_)K}7&{lP(%Nrb$@SCw( zeF|(-_OBZ_3Zltp5>2RaaLNh*ePlIVR+#&!IVkk(2P}!kJ5JUWRMcky6K4a!IMy`e zZK~m{MF}k?F-HEfAMH4Utf$aD32@*`(#dK?i{e)t_s4j=CQZ}!yOEE6zw14J@>cD2 zxwrSTE3Pw435oLs*CMyC;)feda&9|p`j;z}D6f|1PY-mZp6myxMAxP_HuRTl+qR4r z9DBcO8ZDR$ASb>#j2{I-izF#-hz{w^3JyaLt z=tl;4-*K+}V!nHM@3acQVfc>&gAQin>VsV;{YLRwB`HLZ(Wt!4Rcy)JUlDIVRAy0~^WM@lH7 zSu6eDUfLrzm}$`z`tux|-eq%L-~>u?70h1C_dZ|7PmNZpzeyQQ>~{%4;BrB8Dko|m zcEzUd7jYRlprz}2N3;-=y77WV7n!a6-(zQnnO1tK2)5Y91I2@mp748QILd2hr`h1_ z10LX$zGQX87iE4SKBBk#jM%*wkyNx{v;tt|-nUH9`4!v)C-#@i%4Diaix~W}&T?h{ z%~IPET^tc15oNzvAWFEe&nGb1y}me;Z~tri#otG(i2QHUL%4wP-^VK0a!b&EZ`-~9 zi(3-(cZmMmj5miO?qmPn(R8T)^0xm^|J}soHl*FB-}rz+`lvS+>GQWcr-C9tC5STz zZY{ipR^dSba6SvTE1QZOLa;#`;1AKK{cD2~E@Y)_yjX)r7JtkFnk{0PRtd4;4ImZY z06^$R*VVRx{(kZ02%fxSP}Gf#j-Cfqa!_MsXJoJfZk9#yboMKV9RY+Drh*gxa;r5{ zIcQ}XaAQ6z3LEM}d;tL}>0QTd+rDoAH&P&TMdKqm=e!RDhwgIRRl&Hj1vVM48B%I= zCbXIcK-BXve|$ys``e8kKQlajq=58<(aWQ;qN1YW0A#2w#@_dxR3p*``huS*8XjlC zw&?PwtL7x+CHhZHTAiJ_t*vgXEEZ2W=fii6G-k0W35`?zO7uMpO+ImU-cvuI-l>6N zYwo?4oVGYVbKW;=;dr?}@cl|CT6}M$f0t*@YbW|F&gZTJokbWg-a=DDR9liJnY>UA zrnq`8!VQ%d@}ss4I|{OUjo#N(Pw`^#6LW=2Z_FBU{U_k-r8(^M(ZQ z#J(383Z@xyE6sTu%KP|M-@^%J7`{sZ5%=&-nUXXHqHMy}Eic#|x~?fXilMa{xOsjQ zaKQSg5aKv1T*Exncr>oZJm26}&Z`UXEW@X#r;1!n&j3soG{Eht&rVwxXh-eEG=gGSXxH~b1z@NZJrqxdiXW--Th*7l$fR-s;6Q1$;GVQE&nWE zT+&`wnJ-_GdcV0k-sos0p*JoHl(pOM_UasmiBVTC7tu~zoVbYwiT`NC`KshPL%3;$ z^>#h$Xh^e8@izBPW#YDj+U;l-kZ4fZL*k;RM+(co_vrfiq$L;AGW|L6CCIkV%*tucxBMT%WuC2qm9r`MI@QeWA5=#WJtFv+xoaN2Jv>FP{l7)DoqIY=mj# z6vWxPZtJouLMcM?mA!XO8<~%k3n!^Xs(wVx3oRb+3V0_s?k|5A_;Gn>r3cmJIXQYC z_r0-gbt|JMX~+_LQPdJ;0E~V|DcNalkXCT5?Omo1h9#)d z$`6QNY!R_(!=~WKY=SN^1dc>V2T@p|42dcGrWyjY(;p<~7)oMYrt-EhNad08_GaJv zEy{c4BIMq}MnyFIV1-vzwd$0eW{VVCVs<;~^xOU?)rWrAF}J80fnY~&f7i1)zY(<6 zpKjQu_^>ve-t68%LBKP8<@A*%!l?-%`18lC@ciIgM5Q>;t|??r?V{D;^S^ zeUorgX>^S{MfDLm9v!r|e(mA497Xrx&hzPrwyn|}xt$9&i6~<7S?l%|>Z$kDP>;`e z`A*`VFIz+f?}LhLzyY3W0~XfMd%yEwX+Z2m{|t8=XlMCBIYH5gqYC69-RxXhi8i?Q-@24VA}&*D zWJtK%?2`cb2-MJS4!cYH`F1%!AeCrKyu(vFWWg@UeO(3k7J_J&W)<|qa?f?S9GMif zDUyp)I0YUH;k0ujxt|t-qXBlSPIk=ko4Pp!2ReCYVOS@FO_ao#^hM3hL~eaR##SRA_WcpqU@%y`%7{nlG^w&58QJh{(EaCgtS9@i3 z8JC&f4oJDtn&N>$@}C)EGB?r0SOmV6afH-F*FQMDsk}l%)(X&s)Rv|+ZyDQY^TQ&W zIq!ZYT3R5&(8Z}2{RED^EU@PFpMrfKgsh82#y+uQ?P1|wYOQw`8mPZ7{2{MWI=sJE zYRXET^Q%)dmXY`#yH{MjlzgtL9O3wgRe^23K1S>qtqm{mGGNz#yo?ona`So71FiyY zkBjwr=U6YP862j39V};HvDZfE!n$S5hMp%kt0k85Yw&3Jwa1dmWy8m%&KXoiEoMNf z{9NQ2JR?)z5A@8qkSFKhC(BBvD*^FiM1oC%HrsZ>JGBD9-h-mV9aTTJr(lZJXc770 zDS77AZ)gV?@YsP;7@i#c6EoC9K#DSOBA7+V&Jk|y;N=1}zt<$1@_rAl`52Ue)>OP4 zyMoY!Y?cm;p&M)m2#6&9e0vKzxfJhee|lmo&+z>}U%mf{_7-C2-20Iqa>d7h|CAZ_ ztKPuM0J{TH6O>>UypipBz^AQph(0-2XL#mdoUKG4mq^>luViBC_2+cHk60E1KU%PC5WA3eSPDslfZ4E8&?9A?d+AJ$vr?h!X7MS$ck3f=(J4oU2 z=Ey2ypf3QVxiwT@%+Bs`j|Hd0{T+-7WwbN|m91}$yH{79^AmMIt8W&d+5nkWu5XW? zl2i6o(TIi8ThZsk^_5-k*&TjQ=k@1=Tk)%b5TzNl{u;bCqCG#vp{9$O`4YEGL$Te? zruy!^bhj5RtNomX=|k1C@zy*A1MW|?lfM1F#BYa)7B=#UpOV*o?G;iEDh}0n%Z;D- z%bnfvD^3b#5fSqBbhl#PAaZfj(d5eI&rzuesZ&iKt(nGoJyyiwxvi`;+uUW!6BaJ+ z3(yQocka5)@uZfF=S4tq zi6)E1m-^SspHq*K+O?M`xHyM{R`e`}eVj3Vy2Oajo=G*u0ZOj>byT0um$eI^6v)tE z_r$RXXh@O#UZvj1%Ay^@5DFe)i|)b2_TXsEu>@8zkpDe9I|EvXe)s^@fXJvgC-ITF z{BiYk>pMU5GU7aPUvuUaWn7vLB?iex1`!!I0nG|EMufz>B< z26x`61fJn)e*myu_2>QR@H&xGzeLNo{3Jt^RgjE+T)FYUG9PK}Zs1fsL1o$(UJdjd zd5VrNKq@2W7ymf1$Cyn=$T=)!O+-lu~L*}I$u*qsSdgXQzDROf|)#jG9#cB>Nw zpGr!nAnV#F3xl9wk6y)PtR1m@MM>eeJF~TAAZWt>u6h>bys;GtRf~YQ$KT5o;jX=2_-<8>&lOFTC-y zI=c7KVSp~L-X%tg=h3gF4Kk0f96uJ=PKcOH7`QcQ@`0j;$nMPoEH>A40qeFf#XV7O zPx++j23a{Af3dl1NRvl>^0TjU|1b}ddWiGSbqMeH!*}Am4PbNv+E+H%omcy-nO?_R zonit70S>cXht1_&mnLGA)SVb2ab-ccU_+|^%t&Fg4DKQf90fSu_gWw8P5@i5( z@vf}r{G8fOXYQvNQLuxQAoQ|bqJqY_76_97e~NPJcCe7icO&{JBuOyyZRl3&12bg%IVjbsq1Ufrp}ev6Ma zN)>v8j}&pxFe&5k4}in$)b1-dl6?iXl%KED=R1@+zcMp0Ow|n3J&nPR2f(~aT|A)B zD0t-d2x8{^Gtz6n@&QURnPxZckZ@q+B3CtHfB~{@hGMRCT~CivoIu4Of)|4NO3)wE z2h%DPE$22!+S#h)tWjb#v4LB+Y}ZGKcYwTE4c?n_+LWzIy=(q=BQgbT+`Uoe(Hf# z{jXE+5F;7w<@sSxcX$2iQg|h{I4FW?S|h*@csJ7uz*N+K@HvXI0YnyBc>4rI_zST2 zevThYd?&odxClHT(8(6*TcmF7@6}kBaRgaH(o%RViHV4=K;4^>krDCb&nPo{g5EGh zhZ%qZQ;T9lx0|k2*20AkA!GB>TqJJyr0q6WBh!>i7Q#gkP_hdo;H(q(q;X1xu3pDR zpQ@iu9(-K+ApH>1+jKIQnx1|IoSj?L;-Xw!zzSLv(;@tsTXmq5`4B@J*4fz!7>-(C zdciTQe0q1U0#Y&~Q9G$)3&HW0+)QR&5!RCx*{rt^C~Q*~)UiMb z%V%vcy8%?>lZ;Ycu)FI`V-V_k~s8UobD|u zgx|AhOo`3NxCY9uC1Kz%8nA$*IB+2QzkHDnd#ss&f&jM{GFX0}sQM~^Y^jRRmvRX~ zAfhQ9R|A9_o#flzN&EnlT{ONUSsGRixQnBsn)^X*ZBhUs3%~=i9z=3~-B#I8;&QK@ z{;^O~rYI=kc;NeGSL}|2R7F{UcPMMWfKqq_YX2Q_TcPyc*JrDn9)B6O0 zV6D&A+NVC1dB(UA79I{D#Vi;c>!ALu7T{v-x;1%Mu$#_LrcsS2QeWfzzy&UWXbG5#QX%wyp&8aV-Uok8E*z=3_} z`|2v&fEchv{>>m656X2SrSvc(JGR6%2T}Ic+dE7iFV131T#Pc9bg2fL6ibe+wT7wn&b6*dF?t-lO!Zw6h9e!HIRRG zDTf$U^+IkAlzXFADBXhRUQ5J;dKGc)whSGIZV%_@+U?o<1~(~7Sak$E^WtgAhU`8gi^b02L+JG~T3(g> z#nTC_>00?VR|Op9b_-v1t_IzHCzC^DkOg-TcG@Th>YPHFxbm4u{s|utTCw}d>&RwLB6W+1bIL$PD)jf9E0*O|nGGRVU0?;Cp{kX9QP~wXlofg$Kim zs>NLh*HLKuLfx~uy^X@uTEVM06>+ucBt)AqP2X-QOm1%U3Y2eHh>2*ZlhRU}{7E?a zg~AWshrq`fRr0$MR}qSU)S)>m$+TNRj-x#JJPTLRBn4UhUiie#8%yse!Cg=&)W%Gw zo2gjZvjzc^bD0EOP_{xb8A;5b`^S!rnDk%K*lGMKFifIE?3T82CDEpQlo@-ypC9j* z_xadN#kw_%9UsU&7T(yv5=~A?DXMjb=19#C&^_U|n^bK^P-Isi@8k-T$&$; z5wVS0l*OPLLriPE5yz`oAM*xBzityWh&S3q{}a~1j^4qiGncJ-N^R0g*QGjEp#q>A z#;D&giHkiHJIVNW^sf_@+(XgteczWDi@>x35Oek*Z`~RKjy7p4+VLv?h;;snFNdTT5Blp~0G$^)2}>Pe)0VIqk2hGCdSg`Yo7pQ(YF>V8_nJLW z7a(J35fS5lfZR%v(rr4?y{!84P1yt0CnYWbO#K8moyF(r($2SE2^{(t0GA-2<%eQ4 zzyByfhzKPEeFy#l znbvrzCG1~`|FBYHLsV$m##-+m46NHb$*M7D-+!1AY_473tge zczCqZKX(TM_>8v405_HVF@mNzccEE)YdGgc0EK2Irv^|r550;lnrAEyvh2Q-q1X+; z(dGmR0*&|yI9MB`WoQK`eRwP|pnsk{TKZU{;2Ka;3UeMNj05q`ps^Ai9T_T`RCi9= zb7od*_U~Chyk6Ubspa~-B)-IZA+}x38J(6v%75pRCZuo~fU*Z`?e!AehJg^Mby`}Q zD9}U!+M${~K3SWSNHMIE4=V`vD`(=B7q|>QX$m0p{|Gz4DV~;-kx{4yA{6??Gqhio zATvM?l10aZX)VYn)(N=1MXV27EAxosU)nE#DoOkk!Qw(fmWg);!U6PT)lR*;Izksr zfw-p6pQ4}+Cabgr6skP1|e%9PTbDOIK7Xs!`^^fZ`WH zyugl%$MotH2Trgs%JMtTvDd)yZ>;xXl9HHeVCG261W=YC$ksi*M=g&Ue8HS+TebzF z+g$VS9ge7!*f8lvDhM$kR(pdhGup;eulbJ^&A6& zasfJ;*9l!ynUB@}jvETXrXx#isXRRfW64E02@9iptUBFeJa@NueFnt!n-kV2DrmF} zSRmji$QX<)+_-0he3+|tHL6)fg~2_TL-&~GvGEsA7u98#TD#Zi)(dYyCbd}Bd=S&L zXQKf%NhQO7)J77jo^#Kfto}>l2v<*HHGKGs$l5twpEH2Or+>Vn_FVIrJT38HSE)Qi zz2{*94I>eeu?jY8pS0vdP{r&*f$P>_>R%lo^TxSE#&4eV^8tDKaZIZ;Sr%z(KJ}b= z?IgQ1$X1zA##Sa+WcjUx(Lx58?`17q5sX@9tuFI;W2BmF;!9hQC&OQ1sH@W=66aCK zzi17BvhEU2R2jWxEor@fsKS-hmm|@`ICTJ>G-wf>k^qQq#a!xqfM`@li+H?D-u{Hf zfO8#q_6}b#)N`P^Ts4fC444s*2XI*BU++TLn+<(!zs8lu0=58rQQ06qv`O^&s7O{X zO;)d=uR$#Ty3tPM7|dtKurZWf!vjCQ|lG_3O zHi#(dbfnelZ0>HVT*$iP&16@02%#0P%jcg4(KC`QTNe1OmzIegp~~hB+9IN??1H!R zM-2FXvMc`&pP95Jk5M*O07QLmPQ3%+u*Yhg$8KJQc1AfaK1Eup|8;c8W?R<7i$OSPGcF3 zKcSEcea;O%<)Eh+zbL}yaNOly7}F?@DByq5=HVBEVHQ@1jil1^X~ejuG7gU6zX+&a zfXg7k;rmh8jW$Wz@8_@EfG86CA}VK*SD7H7%@{{Gj4Dkd~6J!I_$8Do>LlZmG6Qwvud$ z^M*a@+oazSg>6USXuoQbJ3oKIm_?EGl!7si5O&&y^WW+AHDEp;bSaRdZIV(L|-bAo(X2h$V-?+h)v=UvD8w#*!F>E&)rhZj8fHJn!p!}8)Zc zlSWc5-BYSm($22*6R$!AOi0HvGf+ey5v*4f&n!`VcK7~mfc3>);v;;u(C+F$%?u9D z@@!IM>I~Nl_|@HSb<&*frA?y(pQyu@8Ko7#o5hih{6Jd605CZLoq87Zxhf9D#m{i( zKmpKM>2rX9S4?!rhE9qjYxsc@p`AVe(4MV6pD-Yyq`bvMPyb*HPpS=XQMK{2dwiVb~v50~HT~ z_mt`|AZk+e15EmNE9pmaLH$CNypp&JK-g(9?BjOrG(!m#uCEpQ1`!D*tF>mrknJ&` z!YzCxy#%0QfW+5K(SYg-K3f|ENiMD6`p-XIAC*4353L7Y8AvDWHl1Yjim$HBss90};S)AD zvPCdf0bYf)1?aW_q-V45nNK#q6nSv~5_`ps27BMX#O%@Bhfoj^ja6H!BsfFFx}Lj& z#C2gKL1{E51_l83=N~+x0@QIhc`d@LH(IRX+4?X@*cW2aG9ZAvncJ!bK+AgL?|B?N zjTZn>(n@-Ki)D720z&FQ@?E6e311rdm8 zjt7w#GB+zCX7Je(f5TMIG2>!llGBRUUR(=(kO%e-Bn(?PiPSbXH#fU=2`CYKJ(!&b zd8llC4FAfPQ)2YP^F9~bfU%eZ&=60UhvE<(0!H<|!ug{3JAn`VKmwrj;E^|;v7est zSx2s#Gw_s=$&l0kusotDMZ~X9&Hh}(5(?PcU|}&*e{?w${6-ZohQpv=*-gK;(wDS3 z|BB-Vj~Al!RLis+$laXNN6AtHlZ4I-h&`3#3^2et(rfX#if7Ttljg4FYOwUc(WO|v zP|c|U{0gZauq+rh)J#1YEXaE^ZsV7Mu>o%^BQq0pfC6Hv5yh^chFgV|b-7a!N zV0!#fB;SeZklHFkM1c^HAfN;RC`!B(jYKE?e6rKWRTpt6$4^)Q6f;I!;|n+F%oMkHa}1z9jp> z*j^b6<5M?GaF*dAOK-*Q~=MUn)Tl`yt4 zSpLr*rcC)4psj%F3LtqJRT%^7z7@h05hZ+eNBM2mifwsQ!GSqQSn~`I5Br?2GbJ$Xz<)@CR)I@S?sd9na`o6M8>sEq zCx5Tj35==8l`;jgi2@%oxlICklYkk?q>%8@zkVB(MvPJM674O*E7yJ@CXoJea9Lda z@xuogN_81Dbh$=cq%+Im|EAig`fWephUymT)p2rWfii|R$4{Nhz?i)@+q>>pM-!Fy zQ)Aph9E8RfxZdC#oJ0we+*=f?!vS_8bV3y?0B7LiOF&tWY792T_5S~Nii+YPT@Oe3 zf@ej=s32Ovl4%0~dsw^CD<&!_#M<9?aDu{Ff5-k1JE-7;SneP;ILW6LClP-Eo#2k$ z-1!AK^70M>ac!4Ri1xq|xfyW#sTZkmX*%K$-bIPJsUO6Snw<^?G#)nM1%7VgJhH-z zd#0?^YdOyZIK)b0G6_>!_o9hGGpVj2pT59i19hFR%7ZpmzshigeQZ+@SLXYXlyaWE^w1f*h{ernaBicjDN@VYp&cs~x*L|M5{*jzl z;bB{q)e3kL=W8R5V<_*2!Da`n8*n5t5S}(e01i?+4zDK}wTvqL6vk6j_f>xUMc?7~ z5W~LvL3w4;m~Uz+W*ES&6^e8_dxal4fHrVC(E&58w!)%|H3LyO z6g7^%sk!`uL?YttqN;m6ckF~g0Vorn8{cqVOU!eqlE+tpEl^mF z<^;-CvD;%MB$q*$B>kBZ)Kj3F12Vj~gKFrylmMC=c-NrN-+10WkQ)6^%1YA8F>TH1 zgMk>U$>QomKQ9SaO?%Z|a>qu0!bN06*#%meJlYR24|#Kn0b{RFDh&1jS-VJm1vpGD z4$5fzx^4zI(~oN5oyp z43vhSYx;EWlMFhQN>1kRg6`!r*e`=fn!G=S<* zRbPrgpP*IM?Nt9~@Hzm$+Uvhh2NO_H*%95&n-%3aD~{c0K0PWAA43Vm4PU&#AQkw$ zO#I^m)h5~t_5c%riRYzGp=oXu(`ORC4(DoCC6^ZcJ=G|c&R*mRJ{MH?ya$#;9Fmit zUAb!w11;!sCLU*|k|)z7ppu%r1g<#az8v}fb8*N3sD^w2(mkUWDV|jIHcyw1=c_d} zfVm*ZCNka|J6W=NBHPme^?*CH$?8q57NW-qbDTO6uhF^ z?iFbH7N(i&1jrG?bGo6UT>rfCD?X^Z-qJVamLDg{>3i*tP(6ZXzOgV#7H8TGb_U|d z-Nl7>2sQIGKh$Oks?_=Ow}(9H`(cHEI(z~2OxqBx$}xvv;DW!OMKYOrKujO;f<&cX z2py;N4y+~r`fw=m66L}2itqi{wwtw(Mk5Vh-Wp!A@ha+F!)HLf)9PIxQ3YB>so&xQ zTQz1qB4U!PiuycxZCK$wV!EU7B5?v!E^s8x8J&{=V_y-4K;3=$d~sFiIcm|(qQK)O zN%ZjM?kUjR#Ja@k?wF;3Q!t$g#-CSJN&|}-^KDMIj-M-$OE>*QVNYe`ayW2E={XJm zTd6JRfBrDM`HQUqYh?xKsrTkySmgNAQpd#@GfB1q0vsmVL70v{FCRAFIg)eEjI+3CuAln-_pXOqe($u?kxY!5D%1RncE_{&#JO0@&lrmAkW8#8;;0 z2b8Xn7^~T@d8xWDZxtzI&uVRd4IP>1ksUu=F`jy;`B+Cw6NCrw3Hr0l(B30Be!Ee4 zMZ2F4wIeVwFFh-=I@hH4CF2&+FElVZ zZ;YfCwKSTGfAwm%x>?C6kHD~W)_8fR!;pqGVU-F#wqF}diVb#wvs*^+QWcqWX@hWz zVEX!t0EW}ZB6`eIK>#JuX_>1z?z6WYa9plPn&#IV-M`xAB7NRp_17#)8ngampGsZR zmgX!69icFOc;?ekx0?2#Ld3G*{+rH_nh0mjnv4ti*&u<@Gq(oNsE81;?B~O2;quxC zCgoZuw&||c&E7cfzy!rYy?B6y7mk4M|h6aRN*rUtiB>-7|-B^F3P!70a0N zvWWfNVi7^*s}MWhp}s2hV$4469=Di|R=lI6mw<&x`V`(o%ADkBj|A24b?hIin&IW{ zyKV%uBN#+00ub8Aw+)`(@UXo9%!=6j!fxqM0Nc%r*BopXpuVM&SA*c@>x(cv*5Whg z`&CzP2bomM=ZuP*ozpx{aK#SF6IQl=Qr)4MISacT7RZtjil_s-?u>JN)P@v}#=Qkh zl1`+ik%EoboywuXz~i_#oP(7=QNRv{xHR>VizpOcw!kIxDI zVo9~V8|B4~8MV&!WV_1IHdzUM>NojDB<{5Sx=H=kKNn7$qF*P~^>Oz4yb?E2q8a%4 zDd^!EMB4Z6t#YO>aUA^jALMM#(Rlwd*GsQS;m-zlUMZlHBn0MsHt+e}v~~eHC{p~+8j`NM$0&5mK%4qCw_T!=>^kW?!BgM zS2w?;s?eVKeRCwZtNzDVc$JS#de4Gz;X_&ny~q0S7iJjuS$$l*4uDac7Y{i6ipR>} zyz@zM-Br8~@})(C{P5wxU9!KiR}6a`L=bUS?2>LUVKh6)f@>LLE)ZhuFKe@3P1_a5ON&w~V=#Q5i#46ar%fIdB^ zS!V!8YX8TL!hT!i&=Bx1&+7L_@DS`zz}Z>_x$G$~UnQox=sfUs{r>U4egaSf*FU23 zU*8r=FSx@;8LR20+SbP9azZ(0ZbJ|SK<*g-9JnI--lwLL8~HD`C#p+hxo-dlVpQ8r z^^%7z>W;;gwGFi){1^Z5<6g-iyN~#*)a~{F%kT7cO(So@>IjADi#rbwpU7Oj37i1W z_RpyU4hZn4nmjeV`$!p^K=(_>lvwYQ)-p;r_25nHM@qfI*>$(*0aFBO$^4&(#DJuw zq|QC-=m~NJ;s^C@5r@g?IkqOReps;2-_>yhIJQ3z60oQFZzEM6&BnW7yPR!-*t}V( zfm}aSak_irnXrG@@Wn{p|*5<1XNFQSxutZdOLA7>q?m-Mo<732MTDS>HW> z%27Hr*b#4rdiY}zC=HY?z$2D_?0bE`iQS8=o=|r+ZFxt~g818Y1O3U;1@;<{j7J|X zHCI~NL4rP+ytMk$PERR_n}v)0cs&b#EqGSp&w2^e_lt|#u@}_ju&$NZ?~O-`}U(!niIrG7PAuw{HL3K$1C7UFbK7;W|Z*D&Pa}Rc5RC<*RHcfM;bQi>;sE3Iw<9F+=0LB5`SE)J z=IepHHrt2QgR`GrjV{%9u5~aaW)Lo*+3x7~JdZzxF=$d!B<8)ieCBq zM@=IS+Ueoc5m5s?qfq4qpyObJI383r4Op8mSakTU3PiyIvip~z@OfcgYqg;q7f0;v zatO=L5oi_#psC9xuISEfF6kah9*y8H%3fFDX7dgMsW*#5vaT704X0h-eolwB?za4p zU0t2@HH90lqQlmwB4mKDFThKQRnhM8K<+sDq ze@ZthK8&(Ol%+-1Q=jT^)tD{QTnBBMnac^`yXn8&4^Eg+kld!U2=D1}h=vR1Lze@^ z1B{0@bH1wHldo=)j%Rx5oh)9SaE0!~L^Ca(U!0zlZ#K@NORhnCC(itcM` zyf6Pb$l=!pTb=DizuqU#^0DMNiQq8{DfI!NkTS~D$)ijb9PuR9ph#*RNW|vux4KglN@-ol&; z)yA<^Z<7j@jj*W6RzSpfe_L?0744&fz%+`zvs0CdcFfgKLA4lbVmkbi(y$^Yl(XxRt^x8{-l za~y*BAIAG&ACDKEk+@fI z6SXAhUDW_cilF)!+e>f)VCsPH4HR-fw)vNbVi~wQ5!0JVrd_808U+M6BkxoOdV2#T z%zFv7bc%+n%=-X3>uf+swp3#5{1ij^7xU}*)}|3rhXz3Ifl$9eqkj}(c6;O-Lj^NYC-vJC|YDm%?*L6st=!j9Yt zNsjfLt$i9x3;!U4|B!YZyB_+|wr|i(G9L{bvD?R9$W7KG1)#|LnhVlsJHgo#0Lr^S zTuf{dkYF<8&CJY7G>dXJ=*ZnxM}$v*4Y%rDNT>Mm!azaW!FKM%$uc7)7pH%8ed1IW zg_rr0nGc239bDZaoY$bQdjaDQg=}BANwc2RQj%<}uok;sTxwABw&j-X<#8H+Y)M2+JO^awDTSZif1;}D2l(Ry zX+qVYyh1J@>=`M>u3O=^i;KxQ@GgIEpXhCy^z>YPF1oJ;wRrr^x*n=isLH%3&5E-G zh3t!|Mv3Q&xmTypi?I!r&UFZjI?3`K(}HP-*Kj_3K0s|n(DPyJ?N-VyFl_a7W#5%y z&ul?`tVB48T%>eocv1wuMo>auR>UJLx8YlE7fYMH7-RE-cW1RYxw&15nolw0(-dzT z7kV#|X4uJ*t*(9VInrsBba3X&4m^MEQMzH=C-tw!^%mC<^b`g+tPcCZrfRn^dLZn+ zmvw+;g_x)uMJ49>BgL|99jx2YZ;zMXg32z?DbN8($$(4%oC^Yg=Erc>Iv{h=jdA4` z7Rpn!YH~)ZggZ0o)!MZ-1;9oA#(eyoJj6MIqRR&f?^L`Cc6d+`o|*9-cR>iL?jb*0 z6GF(VYR40`*AL@TFc>$?IoO(cxf95tq>DJHv79yCOQE3i#Z!X#P&+TUw2jGCNr3x9 zC**p-8iY4^&@3Pv$rsx1SFW$%WsW60J{Ya{m1x`qO(bN37*ksW8G`P_`T6%$5yb{C z#3sj*>+a|4OZC@7WV`LJMWJNukkWP>JzDnsNNi0m@OF8&4$(dV)=vMvC^Wt=q{8p% z0FZ0|wqbqiO9~I=^D;kV)6Pbdc9hc`IBg#;I2**r@Oa{FGM{Z?C~y7M@pSCQSLZ>{ zuy6AXsD!E<2fGxOEBIwMEIoKBuTl z5cZS#^Rn4rOCk+y=;H`#|_5 za+7L?);@giBL~Y1SY(prAd}Lv2+R+B-f~|}nSVF8eeR%3H*rc^szy1UE4@U*xF@0i zfqKQvVGfPj#TMT2zd@;)$p(_e7iJa!RN*a5I27^#(hm|!M|5p+}AM& z=+rvRjg~8HUM6gYE;Dw$E1a1vr$?(_@C3?ySEuGJPf8}=(Y+Kvm#0=f`WSCB#^nvP z<5|ts>}{$ai>6t8Un}19HhmkTUCU~WFtfpb`ll6 z;tiuxR7AAZ@X);_{VwvpLv}&V6x}CT#PM@`eB(S_lRjZx>`0PVU35*li1g5|_l+oC z+8J6!gKS$qH}WHK+#1PPEzSnovi?dCZYvIC_ZPl$-QnDQw|KExw_jQ0tQ1IbDLuc? zyd!%37PP}S*MaWGXTA3uC{`bPmVUcjQC}U?a;NKk;bfNzMW&3jiBFl?5sUqy$bEFL zsusQN(|BzVLRJ&V@6P0tH_VBUic@$jMCcI+o8r7K4ca7{76!JV= z%aVL`ezJ=rardqohIyQkkF$4U89R{Nj<*O<8|%0MEE8$T>}F?YXQznhEVV+Yt}TaO zlvFX$Impj%Agah)Utf2kft};J*T{c_aApE|32Sd<@&uvh`0nOU^leP%K--ya2JVj+ zKMtRMW=g3<=@g|ICU4-pDq-A@KD>7JRQWu?7?BzRsDR|KDrpZYvp?41bhyRnb*fPU zi;Ph`x%fjWuRhuI$n{1^sZF4k;BYDU*p?%)Qx`eB1#XAk*nM=9H(XhggFSIlO7EHP zg?rDrBt2LSh;X(!1@K?6vb__WwIwIv{RQonAZhXO43Ft8)113PsC^3Wy<=sL8GA3! zQWc~A-b1&|wC+9?w(YZzc62V7P?^`=6};|;Lvkzk_p+PKp`@wuNKFt~@ zlGQfP^|xrQ1slRx?zu_p`S3Ph{i*vB=saDaK8o^N(Ugx}KkueFU7V1MD|Ly|ILfqw z1}z)>f)d|wFyjnuGrTTZ3O)yNYpqhT=@WHGAlpOriC?Ob<)S@|*y<4BUq2YQcEz*-04cKe!-Pv<7y3d-8QHMEj4CCimQuVvlJT2e>@nE0 z{&={n!^#zEtO`0nPJzm!E(Smd9e?2&fW;cMZOk=^Xtyr_Y0d}Z`3Sz?v>P-|crCBA z%EO>p;*))#ua>P7I{o#>#9ZFJN6-8kj|XN6b~V*K7wXJZZp{X}&yc$j3N*)kvX?b^ zp*=)@a#bjS#}WK-X+C;u{4pBP=;nQ<+J~BVRCgBXkHf<^vU10jsnQU$!uV1U4Jud# z3QQ{XR~K*I5VSk$>ga3Il^C+ys4>>OA^WNufWUqAMs~{hz8CtV+?2N|J`E4r@VOp9 zu!_7BhY!bmiV{kZE1aJLn&H;%4CHLSRt4>hxS?D*MFn+rc-22o71JVvBj@TKNqa-q+XGzPwg*1wZ1jxOZ_+Y1>(}~%zy6Z6u%*P**WanHOxVRs zATqwYT+IR`J+6ObCTiuDD?Y_}Ggjjs1bN!LHKYYR6)vL(7-#fWi~Ss z7S#bhaOB+pgYBCm=A3n~}U6P`ZSg?evY;6Pp_cMJEzm~ z`7kOdHN)OjbL(%S2`zJ3lTtITGnPJO^yyIgTJfHj#~}wvDXF^&Le{5T8Babh!~egS zd&{sYx2|n?(OoLtAR---!UB5vDf1sfB{t(BevOsEv} zqr-DFO4hObB6f}VHNE~{3|-|uzQpv1V67SIXJy=W=~x8RzG@S$J#U5~KHY2`2@E;ELS;sM@z`#cad;$K@q$u2HX3#--#4}{tyYms^cVYRg*(t|P=sh#+XSWM~ zlVb)ewgz5>y`Ch0iwy$3H`DeGo{UV%;|O!y9V_6(AHOG^vhIU2fl|yUNCe+J`ZiAM z-RY!V38~L)s8gSH-@_}W1cA=Ke_g->eEl7MkLj=45BJditd7V{Ac=y9UVv3_b(#W! z<@)~pnKQI2s73(Ay5o_)?|!@FFHeA_0+W-ExB!5`PY)5q-Gy+ag!(VRAT_|vB|4>u zA7PXZTr`g41h~DU#Xv7WnjH!82j3?Cl~0X#_hvp;A{Tq8kMG-A?8LQ!=s)HwM3Ug1 zK1L(>g9h~FM_)lO4^T=%k&s67UU)ZQ)S}2jfPMkF0bHSX55N@YDMe83f@2B7Tq;jI zeNQZg>KV6sP&*A9*UsI|-e9m`*fap)1R=*u$O9J2z|hsI{B8#M-Y}~-u_2#YoC*vy z$-%}&jSZ>%>3>-ew-0(0B7TT+SXVz`;|czH8yCO9fvLH2$nk35A;1S(oj#OMav23~ zhv|;xGdxta`Yo#q7Ay|5*8oFg7CSmFtFh;=-Dj@%I1{uR(NUOmo0#0}*#bByP0FOY zX=h|&AmFb*TYmHlUa+#0TR{Qpc2HjBc3j0yD!ct+V34XBWP)J(*a#B%Pp;vhZc`se z2S~^qP1Tg*0lJfEFKmzQa|$C_qu&h=s=8crb2|nmKvx~k5(|6Xv0x2!{nm&2f?|bw z@0BTlZ!%OR&-Na9+QszPZENG*7%_DGc`T3N*(XVW^kF^G(E-!J0JRlnCaS3#fjo&7 z{x^V@Sgw;$Y{KLQfDZEDdE#U93H5nZ0svQaqd_pxcfK}=QaXS2ny$r}K*3@Sf+9|R znRI=X*sG{TU&uD;SGn5Cv5iR+P`mNIY5s-%aL0rb$ zsKd9h*O~w1J-|DES@SXZ6X`XsO{=NkMVs4peu#BLMxUYBd&-J+k^#>g^EJmCF$P_! zh7zp+%ENthCp4>4F%X_ZlwBiT&XB1lJ7*C}Q_0rZ9Ga`7D5GUNlEs$DBp zfCg=L)cywR7lxXXt$Y4`6EcX#ix5fv*eyv!pHeZ&LxXcrjYFp5`Bn zpb1*C;ORHQ2`-1aG8CF5KLHLm)ghIv0|0Eeys%JD%x7l@$lRXIR>?$tqy>a-z2$c~ z$i)jzYvQQHM`yzDRfOWA5GfY{!CNLq38aatChYs$2z@_P@!Zy_Sl6E-KJa9^JJ6X{ zo4kM$X2f#2$;zy;ifd4Aw@Z0l?hc@OKu>}Mz_EDVD$FHiAp&wCK>~Q9J8S;BoE*|* z8|d{eN5e8+nunSSB8fjhcO`a!lN8;KabQS$1ECVz*v+{_hZjuuD0wwc|2c_WK}9)> z%j4_-02J(^NZ=4G$6mXA)4PV7d(UG%j#ZJHIu$_lREA51^!qbus-|-D?snPt`SIUy zUs{3cHdg8T{r9Qpe$Ek%)F9>(-Yh*H7aq(|9HF{Syl7p{BH^)0SxXS81GV+D30|Hh zpdA3Shb~&X@Z}H*z|>*(|G~+#B_`WAVWuhYgKBs-KVC|kbbC^P2A)|nFCCTn^KRBc z&lHAIe$_oOR}Ce{Ed!8d{D`_!OiUkb3FF~rWhJ;S?6jD=_MX@CNOWF{r55#*p*9;x z+XQ8x`*y9J2Xf6FtSZzAdxNM7D3VX*hMVEhgW#>Ihwxo^KBRK(&|CnB9!&XnJHa$4G2%csOA)wqm8yz6VJ8iRi9(<)%UmebPH5)iYDIH)p?cu?JL*zuG`X-Z6KGNVO zD-bMd7Q!v@*c}@CTpGahTVqiUy}FL_=`ijpLAlHc1F(bZ{QbP?(7eGbK*x!6clHx>fhAm2#ZyV>(VP3H-mmD{0qidVxE`c{e&4EOlAfA@gSF{`GE|+- zcNh|%14Xm3(Al1;dc5A!VP_`xeUrTzN5lp z>V;^MoBON$#vwDu{feVU1nemdZYQ_`+^5c2CBJ&pN?Vf_y7Vwv)|b2m-5 z=H@8kBNDBRm#lvQMZdwdH2|tZx6V^E-);MRzE}8o<2ME|1?Ow$vMnf8&5qyU>kA51 z=h%Zz(ZB3Dn<`&Yg9{?`B+8Nxl+Fp>aq0RKnb-xAYby_>vi2fBoW@7xcY+-^XD{T_1z^wE+j)pMuk> zd7k%D#g9VBck8GOTVo*>pNqF4v!?3)5={HZc zp}3Ni?S0IR=_6d-chY9<&2S88KX}g!QmOp@Ak*%B*qaVAgG97N4?(vblLE>_FClU;1?Po!r*aj!7WN;q3;6LE{bz0FMd{+Hnb4(jt2ZKsKD ztY$(i3T{{Cch6g=GT>NW_c?q#cjfO#80RIxUzkNtoX$prx0;-z$FqNi1Kee+xveCH zk9Pu^^1md-Nd$0tCjrf^qU(P36yyR=P#?q3rhheX7)uh~8_xdLyhB-A6=}8uY>C7t zvLF`-@0>ci@bE6p`~&<#z^mymVD4jQpxdLipC(<9OcwfiM^Z0DoD?eDc8}VYJ{d9d zWV9WyI=($&e{dV>#and@u1a802pD;1iuhtG_++ap4-?rxlq3+LW70p}L6Uw=>YwLs zbqA{eg-QiLI^DRw`=aVR^}wZS6Baiw_S-gp^kxDvx52O#dZ139boKEUUVSQcP7`Hq z7xKS=m?XrUeiu)fQBjUY#EQkK{NLY}QwbBz(Z6?-Qh;{fhKs zj{5|jia%=`i}d}4U&stL;i+%kA&NZt8Gm^jCo<}+))M)o59H$RJTkdhU>Go&-dL~O zUB_p4Pc9FH&;x9^=j&jSU-XT1!pStmrLqRzfUJpQti-@=v$xkl0 zd;}o1+t?@iH zR#}6+Z^j933bet2wyan3P77_JY0u@ans|1}K-d5eDu=={HHscQkmUnZow0n2^|$W= zjzDKe2jjzs>b33_j5ObgqFP|ZsG3$Ts3!2AX_Dle_ zO3*UXO6ip&J$dsWrh0%aJ6tyH38N4P-W_N?;{`VmiufrsK0j$!m`J(e0qZBD5_&pv z{;nO8;Jokg4QMwSx;r8{S-}ywWHJLFHm5*pX7xY{qV78AV{&Gc#aW2j=WM~L#)+LQWyf*$kbK(*;N4uPKZC>O0)FBaz-#)1 zzq2#j1WLa_k|j6`ofmd(d5yjU7I$|@O;5*Z(sAeuSWpcW-0?mD#R|J?>rgR8}W zfB>)^Nv(|$KDn~&6&iFK%p@6E`4$|1Ak}4)Z^psB!$5yO z=;ed~Z)Gd0Wvp`mFs~yBx*g)3Sf<-&xDbD>0wjOJo);&xO@6lrofno2O5a)p2 zvWM4}&(>T&5h<5hW2GZsgAM}9tmX7~`0PCkfJX{oWi1u;JeK+Uy%3CP(1A6}Xg-`1 z%V-Z!h-i_AtZ@piMnWbUG8-(*;XmUl_sSz#MhbbHZA@12($j+*SZLND7Px+fov0E^ z!n|+S=XKZFx;1A&Y`#hB0C1&aX06ffqP@nEHo+F^hM`?(c&iviY$2iWIonjqE!6lW^Qb~o$Jor3DaG*j5z(vlaw2^a7=LO@S6({=c>d6Ss`)|<4$XY%FAIU~25$l24OlyXtFfyO%elqJP zS!nMn?2@s|PFEu#YA~ISV@%$y3#diM>ao;Ww|^H&x@+gpp#lKhQ$RqgoJQk*^xL#! zAAQJLs|gk{=Rg4b=Xy9L=+FAh+YKp*GQ2~IEp>9xT3uewH#7zXw+C}MdUZm;&2;|; z@G4#K7vgPe6%y4l63-z(7pqHq7k$Q`;D405irPAvr=I)`?N~T`3{aCyAgw>;THz2_ zRF|tA>%!Oc{`m1@s@Qx9#EtyNyPH*hjo64^Pyp`zoUY=PY=5;o+Y*P5>OX4?-!>g4 ze;r6MLVB;x%3}%$-B45>F5HK+s2byA)`XfwC|B09P(7$d_!l@CGLelvieU5yndfb6 z%oWE0$}2x??0&vSaoU>5yn+1UiO9hw6(hd@ulokKmRo>yNK3ucy-U<=Z6hIweSP= zZ-})%^uwXqOay|Ww>}jaXbU0q}Z429i`6#&^(ifk(ClJd?D2@^=(kVh2yjtT7@q3hL@1DYf5 zq^lUWp=XY4o`5VEu+^#o;HwTs#*g523$RiDEQF8Up5>5f^8#$bwG^n7O zy0kso&;Ih(0-#)Ai7Fo^eLI*9rX~A^rFH1R z^M;j7$n~NdO@brS)zUeJd=OpOVl}Ya)WS7;kCEc+moL ztq}f`a_^m;*$9@tD{}>?6X+2t1V~~qU4rFD7U92k_DovNv)%u6mGDT&TV_+;Mk?A* zN%wLgW1q%}s$2j0ho82i1tOb`dl^6?LCFsy@K-4Hz6Ay;@?5Ll-`?gS4Sk`mY`=Zi zkHVdGb6FVOE}}$W57y1ayv^snW2)H4olMTsC-f$GoZf&y?9aB+z$ZjS4QCv0iaOSy^27emn- z@!9=dxRst(O;NQvuw}3*PJB!^#!oi-_0e4KBEjvUQa8exjP$@%o8q`10+4%H)|3Em zrW~Ps-?2LLebtx;gSZs&&ts^g+0I9E0DYEN&MrEayG(Z2o^y*p2cIu4+rJ&~8AZ(C z?OLhBTdgaS=-{sb`j`8w^GwP=W*64;>xL5W@h()CKhB>0b&_{2D?-raes1@& z^3F_Jlgd0n@?D|p3x^WI&ScjuWLPkT2j_Vrtnqj!F38Ru)pA*=)gistvt#opBd})e z__2^ZrX5=jsc=QtF14xC*B^X)AF$YKyOTqExm-_F^F}gkl0Fw@DKFBR(cA{&Dpq{_ zpcni-wX{+`MdZolmiva_e$0HVgU&LFNIgPtn$TtKyo367Znw>9HHP01Wx`kC9R&ih zvcV40n#RXXrb1km(u)Z1*D7#k#2JHq1i&J3Y#QPNu0jU?e${)d#DMeW)&@tr@h20!zQ-Vbsjzs|(xTl8`D0av&o%Ipb?&;H~P1N)wB`h!RMCYp|y^Ywu#upHF<1h^X zH({26)Y|SYmP!+(z99u9K7-28Ck~Q6NJA- zd3cN2YWCu{DTfQaI$zX!45|@N#-p7D?N{9^DJ*rAFonfCfojeC4i8jjv=r+D?gEMT z9T7c5qc5(x{7wqP3riBg4kI{!>7bqcERIO@g0x+3RG zHkXz%|6k=_zdo_vKv$4N!)K{~C*_TF`W^mV2i~0AYyXa0q&_gh;)nk}LO?Q5^dmp5 zgd+Q8^*tF2VrQdTm$Z7)wu)!p zT2&3h2)trzUj9TciDmlK>x;{>nD2d9saj};NhxB6*|%p26>*HmC4H;@0(c6ee33ah zIb(g&@8hxMofqeGjz^4zOE%gBrYYP`lf#pSa~|#J1LtgV0Jm3uuPeyS8`P9T*%bG}`-&7kYMl$GmfwoU#^lxhg(N=O&37bdT6m7`1C_0Cg7`X$HP)Jk&fi zPv9|c(Gd&LFBGH>{rIMw?^yD2nGe|FpRa^FzJMOX!3t}?Lf-rM_F!`Wa0+4BK1_Ft>?JhOT zu(|t^7Le=w86~WmRk;l92j0Dzz(4y2TBsk~zdu-+A%6a$ui#_$kg9mI*56g2Qa*M7 z77-Arv-^D+Uz?r;5N=JUYlZ}ha-a%~p`yJ0 zTX87$6tOj``O&}L81%OJr8pD(ry-Vuq4G7P5UqzCSz_L^O5RL}e~NDP5Q#?M&^Xry z@Am#;(!4~~^mGv=EmXv&WI)s+=x7!N6{$n1Qtyy@DjuKTp8#FyqomQ4R#EA{jPZyc z_=M<#p5mjH)U-XzXFegnKo>XVO|QUQN=^>w5r0-v70E&Ea{}6Jy8UBM`!!UuvM^u4 z=W<`(GX?|R%Vbhf&g=&)_j~hd5};_t9aMWL5kGxK1!N>3oxtmxpgrT}7q6DDoN`z^ zS#0iEwiya4>LXBQVHGjXQC<&F*vm^a6bp-DtClj-k_=0D`msR`#(c1Dr0BfmnBA$&Gks8>9^4R~s7gUS)yNC_Ci(sbb_!r=Z*LDx z!h*5aGun&af=H%v0>DFA02dsUlAivt$hec>6#$6S1l!8e2cHlf4Xb7hxJ%y?$DIxE zn!dLJ?53Y%guaFFE%q9kiJ;h}|4ycZB><2jx~KmV(8wcX0&vi)?u& zo+4ggtOfqy9_$Z5&MBo1i&^g?GWWGibi6=3 z3?XhonJN{aICxvEIp#7Ol)MAfJ6@#fpbjrruxi*htveJjy8k!2_5YJU8c2frqbOFL zIKVyCi0|U!0t$(9b!#du$k_*Xr89M^O0!TgR}|)3f>c_(AGpb{B%r}n+v?FziYP%T}Hnto)`o0waQ30WgbMKr_f2mD5( zqw4%(uU{84a#N|qJyVB}ws&@}kCkMP`3Ak1Wb(y;L6L#2puwoYz84lAo&+#l0N&;^ zYk(=A{QWz?+|16BX(*H?IU#rSR)qh7%Hi0$2v21F2?oun;zOE-v;%_KfS+}Eue|^g z=tDIY%r3yny!n$g<8$Q8%F03g=zkj|tb%<=5n!Hnjsf1XD=)WWI12F!ZaZzf7gmHA zb1G~!I*)k}Q%8Ti&!D1$hWrwF((x0XQJ3_k=!;)C7(7pYB)_|dkDrSAFr(P&j_Xr! zlBwP4U8X-ZZEO_QbX|ER_?ezry)ikiHl6-WUh{%s+6^@|L1@^Yyx1mS4I&!4ZIEMHLzoqBo9MFLphAvLiDrv7!W*9W>ISR1qw0PeirFwL{x}{8bC`T5?|eCJIqa7 zcw2Eb5-$B)Rhs%jW|5=Q!DbvQTQ-4@qu~9V8XUkP&535>upDNF&Oh6W(3B%}+Upu6 z@Y$2qWPNPsG>B0k8AP(U2ytTVVUbX&ktrbLEsDXnn1#q29|% zM;cmFCG0uQso2YeZwF_!G_s4nw~GKqg9^8;qiE8vTUKLI{%XF(H9&ET#W|DI>-y3O zTIIuad04IJDv|k%puR(a5Z6_gVZRA=K##GvZt*z7n0C)<{WskQOycj$r(HFLkF!7> zBl&rn*qrSBEw!T4`*>SW~*^xrKU1^jVdN8QC1YhsL1|Ha`3Gv%zE4ie*&xRBk%#2iU z%<}BgTtin{3EC?rha8(1z0sRtTq?XiP1ls&>gjg+3ajMFIrdr*Hzp)iMq0h{A#)htlP4-n2W> zBsAI(j82vEM)wF~M`($UAU;T(a`Q|JQ7$? zeB0?6+_9Il7AfOvmp8Zg_q!8P6Y+FA_&a%teESfXnd3V;JBw-(hW5x645s{lDap97 zJ)LlyHA)8paeOwcn0lfGkz8U^0YgZ#Yeb7$^xYlN5aRf;NtOsuloNHHb{ImFeeS&l zI~mbRwH&sYI`6nP*k>8VDa;a;XPAj|_(^f^#CU$MtCP92D0hWh#zna5A}-rmShqpg z6Sfl~Ke&ElVLU%bp{*cH+KLY&ks>)zRF)~3zq;(5_PU>X0$$>BAia)l^^Mvv3$JgPXM z>3J*@lcdOy!jj#YNjJ`mZbe)`rpN!)B5JPI7lxvuQ%Z#+n&{FwQE-K-3Tb6JX@%(* zDs{!j(wAwZlR_bk;`yr*W4ongK0iVxT`DIu*%Ufzx(eJu^%)~B*qtq;fSEXt|I920qi?3tp!-*^in04 z)ks~Mq%+81vVE#sPzQDF$3lA6VVcaAwdDm-q@Y|$-rTA|CvB*(bS`Ur7+v?smn=uD zseDPSX+1`*;UbxGJuWxpHE)W@jb}Wp6K+|P?VN=g^QTEb&s@|#1W23B*OR5i!qPhj{NmilcV4M)SMn7nc;$Ld%<94m{=tE178VZ-r)Ue*l zr*7)5x|Y;!ih5DhHOSZk?F(qXIzW;747| z=Ivx2)2uxXELHe8>L!z%g~o>xEZeSKk0;$$bSPHK?OWf(j{SnVNrgS)jB**Zd-Emy zyxMJ7gIg|aR;h<^+QB58L@hDTgn{uo^eDva&oq|&WeuPFqJTX8cm&b!4U77;!kuh< zsmk-ahb#;`-pS16`ScuJ{KSik-v!2SGAlZjujJL1j2=|e3+|gXLH;~MLadO33M>+} z*zIyiJmW07lvTSk&O)p8#$z>!K|(j;?c?Xyv0D_b(QnqYJ+M{yRku!4+V=9_^e zTvC|y+zO1`Fc&h!^ed3QN6Yuoqzsqx4`M{a61q4g^NELzKfOqz)FSv*g@`M6Ul4M| zNpFcpg5Zq+_GKg1AuQ1HjKH*3OM*bJ!br+q5=+Y7395QQ8&ZhZX+j4R-9OT*`K|?m zyZwJFtT&Q7D|Wje*p@3g@TiZ*5WJx>n&>wl=wUO|*mG1D-M;fArVorkqhImn17GM4 z(8q)abR$>}&J_Pun)aymLlMlltVFFu}Y95H+tTC$PKCsmk zpRjFC)l^~Xz50m{usb^2+CpYQ(^n^_g*cT@qer6KQ`jbwAaEh$KA08it$~S`gKUq$ z)d{GzAXZXVzE7IO?_d(isB-Yh@H6q_*BP>0%hi^76%yTdKp$!!KzaW+q*enKB zdF|S}pnRb%BM}Z!OzuAPo(4>czjNq+tI^%H&FmKtBhm zO}U8tA*?60Fol1a+jdI3doKl)(F0K62)eq{rFei*xsvHE{7$$RLG*T_rDm8^&J9V+ za44fx1=Bu2a!*9LIVxPpG{1>Z%jwu1mSkNZ1chG!x9=()uG<39>No$Jcef@u z0gmI9U0SeF;=sYFVJ-muw%K`=((=$a7~W(M?^H-hGXjpF9Y8T%BYrU78XR5m;{_y| zP0tflg0J-{LR$IwC0rjC-tH<@SkPODCVbc>fD@MdeGVYx4^!?~C~-!^& zF$p8-@|3Ydpu-fZD;<_33+_puj&-nz7Y#uh* zzYc`xPS9IG`0#?Dh@CKRiv^Bh4A5g8et!^UKN5AOJ`qiahj0J-;6*b`AMrm}b~zK5 zjNAtplXOks2J;+XNgKZgt$`MKKS7lZodakVd=1s@XiYG^UjQh`65ouO1Jqovwaj!Q zE;cq$oci8vOes?)OXE-}ywp-7WBDySUrRejzi6oVt)!VWa3L!nTFU*BSkCQ}vNT5< z_2k^DjbbUO3L@p*vmD}(!49DaTQ#+bFCeW*|F#wN%rhECM{CM-oe3`R{H`hjrZEE z9LR(uq;px1Da#*H#lZ8;6w`O9JtqY<0;KZo!SH;nZT~*zP&RKYFin8Y+6eC1Whl%! zWb&es2IURN9ohWx>7O4*SRt7pOnWcBmFFD-)B*vo^x@^ZCgc2&||;#VLV+@iOr-Y%fMF= z1nxei02w$k!haW#A}UgZQsH;Ta=*7IecjIRn3K;`vR5+SnB8e+=6~$^a1N%d$fEWE ze;Ox{+`_N2^3u}%^Q~mGrxIyK#>SvDMMmC!0MslPcSg0pRE!3_ltX~=Ep#?{Nsw+v z?=4SQQrF?tA|?7vIaTX39B~4e4`_xtik`*+d1&9Wa=RZuzlM|Sf#VlZ0gI49PULVf z?3_5kc`g7vcWB;5j(;&iAz5N^VWF+EgFeaKe%6;Tt77?!Q{t;>7ki2en&nRvfy^)) z+eZZSBYUWW#I1fzG_+5g1!^>b-QcN+$Xn(Lpr(%3zcvFHW+oSf!er*~#Kgpe1noR+ ze&}|LAqcTA!8Kq<4nPBk_3F4X&ZK+y?o+{R^v4!R5S))x{j$r>jS6R4o&@0Ugha4*uxRn{^ShiMZ5D=tMk3d1$mm9#WnQSD zT9L_f=#~mBf5xmqQ#A^zi)R|%gS+^AQ;5(<)gIbglc)vEShd-W(_61Qu_#ys!=MHY z65+W4zl%(HEe4nn_f0?=6v5(S02e2zLE-ruQ!3(3?X?wqIfjpIgnV?rN zTmS0Iepei?tq>pw|81|J&_^3AIOlooDsNMJYU0>>3X~P-K3~av^6Pu#{=b<%%^)}t zm|>1dFa^aIWuTg;Q@$W1qF47)d;H{a{}Dh$ck86b3n>ZK73xLIn`xW6R=(kRx$ z^=I@wJuW8Ex~{9gzKOzs!&~37|7)>sy^wG|ejI;xbbZJ!%=H5v%9AeYWjL72LIw5* z0`pM(`TxH-JPGAJE;vFb`k-D4K&>+({CK4OQ^{uFej?nu?G~Ymr%gJBWScr2ac9MA#i~oq3hST&plrDt$ry& zlMF!)DF8l}$-R2-@Sl!7^P+k_xWE(h0sb00_+;y3UzZi!=nS&bPDl~Eujm-Pg z{Sgg}6VYi{U1l}cd}A#!&d{L@a&*OFG8gt5!{g-!P;uRSa1osNK+_ihgx+N6X?!_$6;TZsqyIi#P zG<)jFd)wpd^`loM#PJ-5MH?R&uH8+gdG@^YmUTh3rR#vCs3BDpSX?=R5q`FrvUEub z7o%p&(-PNpANt=idlRJ78&x)4JW?hW-`%*jqK7TST(1e8+Pyh`n6QnMmtlXRfNI)~ zy$7XAH9AiVCjkeQRgX&xb6>588}|>uTMA-_a1EgP5;$a@D<^|80wo!AsqYrw*-If_ zH+vfJn+VrI=29)uuL^9F$T zl=D+PIJP&_EWq)Qf4rK~8+9@1T+cHs>-nR|>ju8RAunyfN zR~5yt(k6`Ju`;Ss`i!x$`$+}lk=9WRN=qPgD6(qM)@gDAn&XZtJP)!M@DZmpLzHTk z@m%UsAXCIo;FuMXO~>3m9!b*-3$SzSRZ;LRS{^rB8@b5bZ5+rjuCz;nulsvgW$COs zm9oyqPl9wMwYQZg5H^(H5;$hs;MJenFW+^*c0OCh*heW2Rt+{>Gg%U3Z$+-c<^2LlkGH+&aG;zuxgY&C%=4^rK$cegXE~YzPnH1krza zH;#vwZpja+>p8bGVqzKm`q~3X8ChcD-n$Y=R9~l`jaH>q$y7t=M0m)n)bQw+LrP&q zpZXDQwC2s+o0H;8b%gHMcr!l9YpIdHL28=ziw4AaI^3ar~&zp86iIX3M&;cQie z$goVFU6C_+jem8ryT9+)vv$|ca8OW&NGI(uC4K5fGy z+{XBb@I{B3Dy1zw}@>BDJ9N%;=YfxTVMsz2R)ct&Ox-oe8^UF z(2tQskT}gU^lZKVz8Qz$<(dMqlk6y~O5WzRTJ(jU@Tvcr8V{$n=PW26P38@9miVJ= zyoC=~)bb`kD2npDGk2KwaPIA6NA5K>vasHTYm9Jh!lPG1>+4~fWvcnghOU?Wq!iEE z0tsKslb?=V0WYjtHkQ+Mz+dZTJ74l3M}V4L)1PrcWrD@&ZxqLVrZVXI4wkV<8Woe) zU533i@AgZbzS=qJL?)EyVx!F-T-&rbHe1Ti<~?zgL2c&zqu!BDk6wei^@~dC<+-=x zB=;#pyHd71-fT@tIUHffR-XTG;-vIsQo574SoP#Ed>{^cvUIOE2iE3qi~Z^|Ow~&- zw0QReN9U9Y!}8#D?0`p8MDcY^v|}g2I7Muv%Sf_-?x|qaCGB`>s6I0l#>m>D=82X_ zG^@_RhZ)qrbl#q*o-wF<9;ju`v8cR|+S6za5Xh?(p3jpkb8@z7Ey%3W3hFX5I-9Z` zveDAhdvO9hqt)m2cz?|0T>1Rxb5Ppv*E{OvSvL3pp+EjWJ{qXD=a!@SC!IBSO5Cc9 zMq^Y`E!Sdr_h9TkBYyAyI$FxM@}T8TcXwP5O20g~^5e8|Dt`UJZ@;cEq_f)c?dZt5 zB*1s?YRrdpYq0BIfO}$CE&pWNMoGN=u#0~Q>EiVU@QHOmpMWbjAI2X$_Vn-@8`Yl? z0&4VIH(z<{v0}*(i%Rb+)B*=mJs|~m3OFFn zB7cOkekzff?(S^LuU0;)zZ3<84Gfz8<9b508MMC0%gyt;A%QMlNN}y$=Rdrn2C5;Q z7895nFBC>E`MaXmIJVsMnkRYKF^SKsDgO6qRnguXd;)y;>MeYRPx}~0o$wZeg_+;K zi%N7`3G^Kgl^I?Acy&y|O?LQ?;~+exm^oGzW)hz{N*HVuMI!zkA8mEkR`?{Nab+ZO za#J9L+o@Yd^21xDS9Jx!*~^av2y;57)zG`ri*>^5_H{2-LXuXU(-3`$pf4A3(=v<(z`6cIWLGr`s`X#v?(^N zwtUA-ej(sbjGqtm)_o+YU*{trDkEtepN*tMPw=OHM_AL;y*7CQ0Gx$t2HX&Zx&5bs>{>m!b;p#`pUgD zFO5b5>#*CJjuxsMPl4Tdex@~|qLA%0wxjl_Y}u-0BM#MN#gGj5#z%uAMerv4OS4Gf z?DRjU%pFsIQD$p#x2#dcm(hE~>PQuGT`W}h#x@wK!0MYp-Qctmj{EsBN>q*C{RR0^ zC#v=25~ZMgESU_p-8io-3z;R#wPEQNZ#Tq(C|R|-Hty*IT3xX02EU_8RD!*_OSij& z8wyi4r?da##Fkv8UTXD3ottjt4vlM;#EDwWV^pS>Qp74`t62oiQ8wFi%f4#2%bjA4 zJxOva(~ViEbpGnd;xM=p$#b=7H8NGl?1rOsQLw0_;y!l8T09uu5^Nqz>s``?;hbcK zcI=Tk4N*eMWqf4}H%?dM^N*)}#?71X>hcPCatv5Wx|H_)0hhYI4=h%lHHY2B# ziliy!o-CIi?DC}qEDz;Mtfa*c6Z51fmu#krdM+T1B#9kr+dotEEMw_ObjnNb2B~xz z==KkIPDUTxIU_=WCAJ%->suf*K)K%r9~*F28^I%BZdI>eXwF{^Putl{k(45<$2xVP1!2%**Md zi446n{HA`+BeI%63>Ec2iJ_m2!4rGGIX4-S%b0wd{eKWxoZK^XOf4yoy_@%g}OMR)|5Y{qcs} zfT!!A^L??5|9EO7esBTRevZB2G55X;!!<>h>uWitD$SNNQC^$Tfd?YT|S z@H)KnPz!=ZN7&$ES?`w0WeHR2uiUfhJh$9kccoY@kCsRllc|f9?+PuVH19C~QI&}t zL9*3L7mGG7{cK61RRtKHPj+PJ>W%n%Xe$n9h0LCL92)menGQMIJ?(bVRPngl!_cj+ zp3NTdI6W@&+BjS9OAXJivG-L#>&$7%*RA6^T;5iS9*G~5mN0hF$R5~mW5MYpFH(wqn1Ql#_R!5zdB^!?x>W$Z_{YPUBjtO3-h8L@bntIiU>M>$8eF4NGPr8cyx3SSG<3i zZ!c$FcIvflisfbOv^n+jvVIA^{Jffa&y<4k@xj^JG0r^#FR1fSsbficNzT<-sumi~{6#!MGZ=lNW# zoN3uEJ2bD-qhFvAJ6o^tf+Ds1A4ZOek!)2*gI`|Bize&1no z1GhMdJ>oJ?9+P@3yqo585-5!gPgC+n!0~kMq{J_HuRR#Ni09=|YZUc$FoxpIc5H z9BP$rk7qt0Xs>G{sK2B5WXW$TxQ!-*&g9OLvVJj-6;|q zVK1Yuy#>gIOw+DwmP#Pv{-gZBySWuc$QlGTBpqAsSgfz*>oAmT2@mB560%0G9_pDx zPd<8X=eUm*(%{MdqdxjrLZLr?G9AW|J6&A@-x9!?{B zKPN+`QPf#%-8Dwk6$n(FgyU>Mli+v=%C<2JTiHd)|AlM&KvDvBF5Eix_BE$OElf%R zG0$tGqW`O}3YzZH{N90JuNjt7#9M)j`rfnG^t0Ay(6RQl+6RFPM&qCzOO_#ooJ5*r z6!kRDHX%`t?A@Gd3RrUM#&J6MlL)yFtchAu)XjH@QYZp#yPL z@NX)EJ|alhp9$Zje8InObHjgD{wDCd83-&w{38IZezmC7;NP(1us1REHj9k0A*$E8 zjeHjy!z1QW5(k#=yYU(R5}MKR@KjwIGJ-Sq^WgunAmAsB&;x@b%Ky0yK5pL2w$A(8 zlXb8-Wo*P{lTvD%ont-->6G(qBrj4g&#S4B9WVEn-*2z~xne-{b$3SGGBrqfY^Sd` zBr8}xq^qqbj#1HFeup!cXj?&=RMPtVEa{@JHdo--wp+VyCf|vLl$wV4AIHfCbdC(K zM=^`(er2u;PEE-%NNxi z)OD2%+l5>I?yog7z}eZDPL%JRDrYURn~9jjQqtyjP37M566+gJfkhh8(IH4(ot%p>1zeeRUiAnMO10 zU+JL@5<|62=P#Co&PSs(oCgrb*A5*uR3k#L5~sVlU39Nnpq9)6bd7lbJac7ey}X>%t+h3Zdun`?7FDr18EuoN7HXUm>CR@ zYP-y#oXG}(tt`sht8o=G_1glTg*QP~(}pli3}MMIh%PuvH+l4s14yqHuXXE`huUOP z->)*g%CL+nTF2eEYVkU$?gWe2MFx_Tlu0g|^F)u|4kOh@E|6LOQ_?9xb!V!!q?tV@ z+Lk@)-){B)Vc7i$)h6RO+synkQaISVwLT2bovc&L z7doGv!peKEvNh={5 zj)YTvK8VT8A$dPxk(ynIwwGN}vN7ec6L}i}@r24B+Px$lDH*KabYFW&Kb4tHD`ZYM zV>dVrdA15w6yhZm3PR^UuqYiwjpYj0%5;p1#r=01R$|+mA1tuSm?3jog!+*-yQOA3 zi?KjA;D1RrK|~Fh?eeLAWLeW~3bkeKnUK!s;-!evFY9`otP~NQ#ah20 z%QfKO%B>&uQ;fEJO)p5d?CTP2yk`*@n z-ZZ3~DMq{2CroX~H*;_5mfc4CsMsF}>KBwM|Kl1-5agVv&UQ3_k|_2-w_Ik~Y@GX{ ze12)t{CxPz(~S=@dA&Y^2Tc(f&4;}|ODNFVlcOcV-4@0TC8-kpv(wqK@08aEtn5!&2fu<)KEP8cs8a09 z>Q|DL(`?*Km3Xb`O7$z|+4ew(KjRAT`4Ax#x#Q=kNw5u7^*iO_N7bJ{qXRhti@MRC zMXKWM{*4aJr<$g!*6}+*!^YX_maAoU4|ZH43=mgCQ7+l(zuv)wz#??yARAkYcXq;G z3oq{h{XG(RxR%{%`FJxTfiV<>ynOW^i>&G3_D=i_L@J|v|A~hAH_(WN@gcOE+rC7y;|c>dBI9|>WYwu0UzgY)!MB~&SURl z@S!kZ8*h1mDd6^Y_?9>trT5<`A(ca@l@k4n;EJYYK|%YE|M@TFbqMF=*^fTb#pfkh zf_mW5Qyf!X>kvl_S`@M4Z(k|8>&ZWG{ahoZtxGF#P-s0Tc~7%zv!PpS%I1r_k`tyL zWgjA(s*AVuR-VXN4a6WjqUJ(nTHp`uY6-}&$t)-met%qxHk|XrgBR=mGJi4})|~(9 z5oVKnMHTg{=g{%_gr4G_LxuW!cZgf8p)N?p>t`Vg88MbR#w7$k6Z;VR|KmHvRsN)7 zME2X`Au?f{fK{1Wx!CbNuWU1FSM3e`^FumwtzjIR&J-2&Z(f_( zy!Jys_Dw^==o@MY#xC8KSJoe>41eOn&jDGu2YSdIu_!GWZJ^p2Y{TsJDa$~ZZzFR$ zM+XL@Z=Gv;$ym|zhIcY` z?s_k&YgUt$RuyO3Emj;UMYW~7;mHx3KbbyqXVoiSl(t>&nyJXk@n3D{G)}&kR~dfY z(f@1vm#J&w&fYa{y_ZFz7G;JMlr$8qHW2#i81t!UFiAe$AQxVw(ALmPPIi;d44}D# zev#D7aIFtTa&mGus;%pf5ARp+6YygAptIf8ySzY-Nw~xQ4GnP%*q;ERg7HL+Bxl>! zjZ$_k8|iuKvmt~-+{GOB81rdsQm2Um6>-ZAr@ss^>5t?KyY}W>O$TnEH_NWU^Oc-I zTGNa}g-P1QEA_&%8FVCJHFY|{GNwL{_0Sg^TO4ijlOyOaDov!7Gnq&z1|BMq2PRiX zG5VT(3A?O8!lm+k96023EQ>~+`+)sr7K)l72QUJ(N7 zdjOqO`;-KIWPzU8^2dOHkPxGTjXb`>Qq7$uoz;c*BoJ3f5`EbYJ*+QYya2Yaf?o(H zmD{$TY`emEu+im(y66~`-Mb}&!T22;%!zA>3bYzfnT*gH{t~$U)+o40ymxDUw?B-J zpiXV?J;n37rOjxwUY(Ftd8f68iA!1g)>;Gd7EABLSB487hViWk*<0!z98`Y@_VS?< zc%4k(Y*5Ni7Np@#X?XCgBf_O=yOu2IrlHD!KuN>>pi+H1iZ1=ezW&~oaaTR9nc<5e zB8Xq9$;hjmM=tX0^RexNQy?I$(a;VZ0~s)mDWuH*-Qi9bvONG=L>p9l8bKl=TXYe5 z=yLHAzkLq504z??vmR&^wgyL#&Vc^DLD91Vc0K4{kaJCnkZR~t1$o|NVR^VwX}72u zAMnAz`!PePkEC+$hi`_#K1{|41#8hfG)<7&+j}}ul4pldBxO|>N2ip#?q>S3_D;kt zEZQ|4U-ZZPEevkxj`)p;uD(u#K5J@mx{Po0$^%#XN5^w#$IiQ~M$I-B$@+vI+RjXx zKT}wo^IOlWM_^`as6L-sDr-4d|3%|kr=DwRx{+clBw78D&n6Hd!4{ikz&S}Tp_IoE zbkBP(?t&aCUzmqNa;GV#8slVRKGEhtbnEKC*BvS3iWE16LQ|j= zT$ySyQV&6NteEc`44##8A$MSof8@}6!%Q=Pn2bcRIWO^TAjJY0poyD_nOQwY+g(-a z4{e4g)NCA8xe+9za6}cqo$erlJJu$^SAa^zdvoN)x#`pwikZ4_U zhKWXLul*X-DJAKiu@Zmh%50LYnRqv$(_vt&wxDXy;Egwdp=_#xmp*N*A?5#!DhS(f zO(?yig19P=OQlwQwqXY5e}x?SyqiF_?=h!J4%pjIc#z|aeV)1^7K3K%7k>+jYFZu##{wlXdBfM9QT&KZm`T+GZTH+VQ@n_d{a<Hl7420mWzrE zr-Zb0u*z&#wh)jq*J%ml_YF%{dahsc!Pi8*7o|?+_(^tMW${c}&WCT>YPM;T9s6l| zH{!z;{C=p*lFK!bWV4GV{!ysz%pzA#T4j94wBh=j@27L_1{L39nLAkGpJ!qmvz)mi zW}&P%)VFFvweHC-wPQhmq`=I7g8M~M?+(M6&T~Jk!9 zbsrYP3cr9m_n=dL6FGwqHJYB3U~itgy{vrl$^{F_0{f~gxlNq+2&44xU!3Z4sK;hA z+cOT9`Wj-MSjlggnX9~YAcHy_Y4>%dN&GxtN&;(WM_g?9Vgs zdUm%#n@!){9TU6369{Snsy`!536dY3j+%)L8WuX-Ym*&KJUyuMh=9hVe0g9w`@C0_V8bxN2Gtm*e)=A#1$~l*rm>P=<7lG~Gm8}IG zS^HRcn{m;G9_L_VOfQqPqQj3cOwC=ow6FQ_7dsYZ!jPRRi1!o@eI5 z$>}g!Vnh7CvP=uVsC5HS!H_R)R4Gn6K54=Wo8^2J_TZoF840t!yVn)3H7i8`NNS2?v1SIcHQX8N)e6qn^`q7oX!Wm zsfXN_`%UjC`h=5Qc?!4k9?&?43-7zVTJ9yBcvs7&lR_Q5v{plA+`jkm0^0zxmcNTo ztUp}qeAT#2En{ryb4hBwLGg(|U0J_Z?)-rc*AuS0L1)VK`~j!chg>_^uKOm?>cuJY zhu%d2kf5mC-%z`#FgU~l9rc=;Q9m)AIuw+AV6@_gdgsl4rDW+#TwR+O#TsR{d0wEs zE8xo>gXWu1Y=TSKQ8L-(1eTF$R<7n(mYYEEPMZQV8=Ri6@)KNfsAwpFIh5~NF#^*< znYfCu1z&}Q;AvCjZ4U`?eDQtj=m$<2UD@D%jGpyGS4<8qHr~*1C3T$_bDr8gSan6N z3@v8!FYTr!mTfMu9{jMI>;K9t!x@)*X_L%ymLSi2n~{M=-eI?HwSTh1{FZ}+IHN}@ z^)r5BN*fbG%Fv9eFnOLe8^> zfYl}`^6g&dm&Q0K;0G6jXL&4kCbrJX37r`^oe1G>Dz0S~vrOn+)*~ z%qe=k`-6mWZV)+m-3$zx(NaD;HQ(9(T?6vnar#33De65)d;&&lB zNU7eW9Oy9FdW>Asm7pfPiN!C{$AtRFjX#`w?ett(9k&;}B>UMmM+enc$Mpal&`-vgGtJ2!)!^T+E(thuI)~F~%RYIw6d}Sb@kf< zEwtR+o6xqWR0C>P2hh0Fo^w@={dS<7oE$XlWxeT#7Dfc>G-ZyCjO@=)xgkLEj`0E8 z(i2N%^f?@+JaxsGY`_70Y zZ)x)KU&`7))gxEiBMB}zy}z}a=tCGFpSMY@wf?-Fz!_<;m(IK-?nkq5@yV;@Ppsx^ znhS!92bFAX9p9m+OMz_q3+H}{+Kr9$q_RnRwlXTd#Mc9ETlqH`vLCkUlN9!1nI z?*P;C!s#5|g|cyvb);c^4UtLkJi_qD45?@q!g7p!`*3?0Sx;PV+S0%IaqO(hP^5&b z0QX!`eT|FS!FS*CzIzu#c`aW)2w~ou`!O;N+lFDF8e+-*qM|#7&T9>Zja1*Sn9tTu zhR?{U+3dfI#<+rpMt>l2|E_!huTBcnizb!3EKd`%qAP#t9E`kI9cY`Z94Io>?BahW zAyk{f`%X2OcP7ajy-_#-sj<%+jOPG6znaUoW-p(2@=iv}ETuo-ZJ-yjuKbdzHjxEtcLS_-b(5L1Q$y(bt!aB`J#Qc zYqKBTzK1AP=Uml0(##0x#klWQ)7%2iHNx&}f#VOdw{1`cK~aPC9-|RSqwXdK3ra0K z9wcNIWFC>x?iJ==4ou&d=OhqR>%Y$G$HfWuaelbj3p{gs-et9SmYAns z9#s@hLBS;Vy4zV45B9%TCLAVwa#dQGdgAi&etWn1pKA(dV@qdYA2k*Y5H#uj$~UI$6NQ{jGM1jEFEz(ix)n zmD-pHoI2%>(amP*eD>zhrQ^%Eu@&RVj%}}>+R-dR5j}V5vdmwevPU(!T!6~aHJt_o zan+|ObW!m~Xkr01QNld9fBqpjH=r93=A$KHlU=!sK1a&x{^lj*&fs!zx31d$p>6h3 z--}B7)v41tz=0puK7J{fI)q6P-{^k)&%b$cGp(53?_$0tC-^y#DPkLXYcX|Gub{x)z411ocsI>*R$~K6voN zawfEMvd^`{^`JK`z~I99Ae8&pO-7rzC4TaxyRR{nPy4u&;W~$&7=neizm@Z7+2TTX zmd37{S;c;^npxnJ-VNVg^UCRiU?a6)TEMlTW(H=->i$WDfc@SJ44#0vT}stfF>Rj-26Gw1pKT2rZXip4Ao-68YxpU(I!(=~&zM@RqKCmRB93Z8b455T~u;$-M2 zCxwXnlS$Wrz-qsI!1k~H=QW|hV(NczElfDjI1UKJnRh+j99a3$F&S!fKK95rxDl~I zn#6&lee!-Jnxta(jd2SSInE{iwRLw1Hw#k-D;!&zBNykphokT>E%+DMP4-{%YSY{; z^KPQ&l{xAOGH!5!a$Je+UWbv;bGy~^G-7x#9%Crx3LVE*kC}D!82lr+^JR-qv9^T?Kc6Q%?`g4c!xARW_4iKoOp z_BaAxQz&L=>DK2v%KT`{sRSJDW(V>O(mQQtJIq_h;(E=^yE4`M_<9HFddxmozh0Tt zH#FF{>%*L1EZ=pGiftOH6lhmqaI&CJ5G7IM78p zE|fSJ1eRC}eGsArUG?sqVebCtQXamM3#DbF6&ht->4QV%hw_Mtp2@-Rb(b1ZpY_(n z=G*C3hX4ub}nmjVL51XR5K5$#J%E3)ze!f(>4H7iuB1i_BI! zR^xq6x9eT!dB3T?i{tZ-FfVRB3iXZ_EytpIv#)$*LBx> zwzjrj!7LxS%Hm)S6=*WeXs)C#L8EbcaO$JjYA?AYfvq2>xQrjCE6(pOQfyS=b+8ar z324qE#y^*9OCmpcUEVEl41+-s&ZLg3#BC2}B|3xm)u!4KHwK04~;#!_Qv>s!tD-FV1sQLElN|*#yP=nXZ4W&X)@>@oEB5l-M zR-p|^OhRF)hCLJftItP2r+g=W+Fc&!KJ)3ZHAc!5Dfd&-pn-G@f zQyv|nBe`fB(Dw8!dz?k~>y~0N-h%}``#NXgH~7e<>QaZtfx8Q3GE}xYIy$hi))EMz zYdau9>C-b6E(hpg<=@x}Tx>Z=arDzP_7s6q-k^%r=&KJVGpbZC9aH9X6X<#^4&}Q; zbtR-HWGq;u8gii2`czh7azadGm&HN8%WH~_hgGf~o zH-qDcTkRcYR6I>|=@r^aWhTg?!1+~S&+NATD2CbGU(|cvew{8vGl}$eD`;d~_uGaZ z{f<#Olv%?#A;C=l?m|75t>=;XXzty0Av9zzCfAI#blV3o21WIZwi`|Ge2_WWTQd& zs;H5g%VABq(Cw>12-m!odC`X5K!3?RdT9Rw5c6 z1V_+cx!7JFm%yagE-;x&Y1os^J1J48nrEUXvM%1#o6Emt;o(xN_i}IeC~5qsn~R0a z_XeusE(N?UnH4boCIE2T2$j8eyu9wTm~*$}%(c%LXKz z`D`kc4hD-Y;~xy>47V%@IM~d03KRCK1Tm!-GRmj?$WU2b=y$Z8v+!%0>Dc(StQ2M3 z!*%}GqZTKg-@y>jdEhARX2UN#Rl3=+?re_RC#iS&ld!<{=i_8SSyy>>wf9qn@Hjus zs)o~Y;)GWStZVV(_-_;Au1_a=bm*n{TuTjq(!4vz--2 zaEQ$KZU>M$ZakQ=p7a@2+iT2|yGj}Br-(|0n*6kTfY)*qH9=MB$`jn@aUAEEVhNN_ z;!(wA3oqAxXW>RT#ZBs+V6uA_jAZC{TqCy~eCx80y*?H8q1C(!x&--5f2I5J?}R3B zhXu9Mm)O;7tQb6Mje50%#*-=|8SQKOtq6`B(({!NkQ~RH1^TZxJZj!2OOGTR2^sdWBqyY#rw6%9w4y5+2pQ+6 zo4WGtBqa|&u-Xaq%{*VN)w?x7-ETQgf8)?@UL~tg=p%AN55LgcAdarw{K|1<9>B|g zdWzszYI;Meji5!>z9&ywbR6GaYR$)|$j)}dwIItb7Xr$Wdqx_(JW{pd$BAL*S zmT{ek*egjW%_aLDR;7bWfqzv~ylAa#d@U#E~cRuJ4L{M`Ul5TYVce5sC`cjY?7c zfKC$3LI$c=U1WGBDA*NM?A&v}m_7=ucfY0zR8^kxLDv(x_(-Sc9`RAV`Z$U&W^#LJ zgz&;;K5&%UKkri_KklnTlFV`W7=ZS}IEj06<drdL*MrN4PgH2g6`C3`7IKpR8dkj zg@viFsk&c1lcS)SKRF`qKxm1~OS>qEl1JHNYlf;U%)XfgFc5!b|IC@9qpbDM z^{v-wxiaz?FdHS9(Ug5(TDlqJ-l0bB9~sF~1CV=tLjx$z5QW^qlqMSGm0=vlzjV8< z_s2qi833gSKx7U85Qo?SZdoyL%2RQpNA(6nUJzdy<4icgW_w^hJYR_jT{RN3)@ zHfgnjCqH4TwKUK0DFL=eC%6`eCR#MCAZrK{6H_`!y%+C_S^%1mOl$DEJ_ID2VCWRbg-A*VDU+vM=E1kAMiuDqY zMNCYLHfqLI2wO`_3mYTj4h-Cs;Tbnz$|8{kK*5#HIdq(~T&KxZ~s+G1D@zgZ?Kv(j+9fcNWgh7`F6_5phO zWqXOZ$xlS&j`kwO-kA##V}S66!FZy93P;!k_CP&=LmMQcWPU|E*v6cY8gxIQ2`+yj zLXBVqC~mH|NM_&-6=?Jd(oxwDxtEuh&&|z&-qW;*`x3B%Vr^8?KPC`v46w{~5sAa0 zca`?^(sa;4|7zvC(Wu>+&vYzn38AFjyI7D)qx%YMbmti~M!{QynXljDM7+|#h_X=X z1C<9mvq1b%NR<_VZqt)sD3rx*d_M)oE(z@|y<}!SfSD;IbRgJJXumuHH9go&;+~Pc zmoF*I(U-&W7nS5s>a`)?9~vmj*o^+ZAB)#SGG@It?<0m$#a|gu1V=CMVKVQ%{(}v< zXXWBj8Sy?YCnqQ7gE!nm_FY>H(1ATh@pOeIBf(;oG;9=PWo4rUoyQKrW#Eyq4<&|Z zf`mx0CdOL{s2Fg3Txz~Oxpy!cMZLeKCg>7AbaL^rkUQMP318tIISg?B99*mtBRkDb z*aKFl22$kPtCMV+fh=k4u{f~>%R%QerO5n^@67wxec&o0UILJpFwotWMVM)&2@?4Z&u(|lo9QexuwXd$7x zUfQ~$qI%RJEdo7g7^-%@=Iva|& zAd*w^tS8Um?BVpL2+s$>rr283_g8g?*;=37aP+w^At}l-~hbE#>U3YZ3ib09R2qp zoQCaeUwxMT)OLFM$$2%#uLZL1b|4Dxm~UOFX&KW<#Am!|+aWD!!g(3heX{UV|BmYX z0AZGy%>0(2fs3teGdLO0;E6AK&EAG0-_7#N{Ix@VlHE~;>wn}lXh6xHect=xP5=8+ zGJ?P|0UC~;UFSj1;CLYgvckcy9cQnoP|Fx;QS6d=(J^N6gYXY75hhR)G@fpmpul)p9)Fs7koLlcY2q{%dsl z5(1mR739AhTo2eM`Ri@LfPlCc8&1buUrVNd2^ix0ZwvtX2yP8w7JC1R2{(K8Fc{A6 zL%@DI?)Ug-R02$WQBexz@}m?(Y<#}9j~4X5W@lsYHpK$u;L#{>BY%g0loC*^v}s=Q zu61Lk2IwT#?|DHzHE4~Y0C+LY*9pJQ`;U!&iiOf%RYUC-gnbPc8PKf?Qy{lOp4F;rQOOc%@LxV+|%FTaE* zErK5ia)1!)K_&VtC`DF$wu!M6QW`xt)K3-N_dFad$c*8ME#Xyu)NZtr1#JyGW8eDD zXD@ux|6;B>6y^Es>w>m$%{$?sn7JO)<3K^ovjt;2Ndm=E84TA zf=i7lZ0U}wwJ}uQ{EsKvQB`O?x4p+8d-=P7|K}>Y)w-d+|K!;%tXG1q?jRY0LeF68 z9g80_S}uN4h(R(AjMZbXfw@`RwsOnhu)QU(BZs!U=)#;_%^RSTKc=64%XV5}aoAxv za9=cQC6aEE;Ul-X;#uKw0xl(%R6&wpT3VcxzW5{Mv&y zmu$dHjiqLL9&S>Y#f?Y?zkOT#*5Lb~8=)4Z^-a6*XMf@BlYP*913+a*rHKO*(PVCM z-jQ!qChL;>4kA(f6JYxCv;L(B$K?=Le3DC4%QFiLGK=N5&uc+j$!R11Ag!gk1EDk! zqM2_rBWzA(8PwjCZJkEFU1GjJwx&ka9bCe_2~l&GA6je-fprRmzo^|g*O}Q13=}}C zuAlZ^vQw@TT{~BkSS`TZf~0&MN{%`K#?S9uRb^XZk44Gg4w3DMV|$|D9;+yw?UnI- zq|l?x-Sc6oARooI`#m&TE}+D-kSxk_lBO5AV{;h4s6TOuIebH9%y==ekaidLjgd&>?u;Y}>un^=^>)YCMDCHcwzQ-0(QkwbNsU7%M8IUmjBMVQyV=s;Ux zrZrn}z{*-u=G-`Z@oCvXiA>~5bDVQc(mZsAH$zLfYTh;T>0f)F*%qZ8Y?EoL36L!c zjeDRU0+Mle8_xI2KTei$C`z`~H!_+P8@jQsM+hhFM6~ZCA{L={xen{bh2ERg!R#00 z;=M@Og2l32imxsly?WkuP?YU zdYUv_naHJ2-&=$Zq8xJ8>%0rq@Tr)ugHY4Fb!A(Rv}l=NVZMW(d}HRJe7lsi`K>R7 zG`mX#LVGdP?k%^>2OVF$dRaxEinw8fOl+ z;2;8V%NypO#pA>@x*|-3QB{b`u8|<#2-qsd5#OEt2icENvypU^BlNwsJ^PzS-lrowdCOl*>(TJOXNNLe%XAfQNXxAMZUCgieZY3k*E++yq_* z__nK6I+OH1=unPyD6}^_SwYd?D+qd@yJlJg!)jFG*^%;2^?`I68vHpts~Vg+TNHQk z0yG|aBydxNvzBC9B1cq;1dE-wm>l)pu^3o;Z9r3`%)0th`Ad(AWNI1tU)Sp;4<8}E z?iv+Uj`82nLrA=;_1SIeFPjKIEnI7LFzR$_Q!P`Ov#5nDa&#D!&6q`kwr|5d552$&aThwr@qjsN|x90O9 zHFJf=Uu$H$23f<#Np=o)9`@(veRQQawV%(XXGa)f|DU@{fepsB-*P=UI6@c-Vin z{C&{`kQ1vzvX{t9j5K zzN+7kQ(7ikUOEdh$e_FXZs2w`Ae>~tfH8BC6)+hXGH)g(@#fx15s1 z$Gtcnq6Ki?93ByqULc_g|6F%g-OH#Xkn8AO^8`uTgn9bkzO^D%6yAr0)#EMDi2m)K z0MXrw`~3sh9uOR30Sy4v>@mc@BQ8dP5`KA*c6z@Ndzb8DxBRKZkjaxgQoQUO!* zf}#O?#pic#{?>K_O03>ziYq^#OH~?o6<7#+j5cf8qJOdU1rgrAZe@5vDbZnqL4l9^ zBdtLW)u!Yp3A~6$(2n$X{rNLR$%{QV;j#`1V~*txbqNg@eL7myQ=h=oI=rD!=`TN> z8l3#RrJcs&_P^Bas~emo@`sLL9hEzu%vI2&|20S;l4oxx({R=9 zji00q+bYwKcP}^bsu1=Xm>VZQ8k<7i9g^3%iJ`23blT;PU8+RqA;>nVqRPg*931qw z)?C3Mk;o^_{E2Dp=~0ynj=I~PFSDbWI)RMv6XMFK)4aEbNcby^n|ZFQ*>Yi{`$8s!^OE;QHW zzm^&G6}}GWG$?>7ICMWvE$jI{g&gMY-Orgt18Lvf@BI7AL5@e&P^iCHcIW7DI<-pB z&v)mJ!sv3S;M8(15E;-w@n9F~JNlAkw)5>+9l^=ck&IHF({0J=9R?0&W?2tZAWq7q z;UWCzofXPm98dW?b->kCE4v1zqx$DEL{J_vQE9Z*gQ@-!*pJbmR#nB@k*$<*ccyw`T`*mRrv6rVrHhhZVB2(h~ zh$NX4{0SC^mo7L78mZ~{`eA3pMw~G%s5HCW1T-n!YL?bz8u<>$@IT{}9rBp4gnR8}YE@e}VDVydaZqFs4PH7SLSD*bU|l2cC^E@~p#|NnnZR!tm-1RrMn V(nfK_v{An=AtHT0>z?+D{{t%j%F_S< literal 0 HcmV?d00001 diff --git a/docs/images/solution-overview.png b/docs/images/solution-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..239f5cd793965d2a9136bb66262ad9ecd074c306 GIT binary patch literal 97582 zcmZ6z1yq$y)HZzRZs`(GO1itdIe>t)NOyyDqm(Em-6bgv(gI3@NJ=V+e3E5^vw;lx*s8<%B>fwcla$?J5u> z!7I2FASyWGLs5fI`yLirpYuh!DLZy-{3^fD9}SpdE8cAz6c zn%+lhklY5{Ren_Bs`)G|3yYkO#i%9GPvvKxZ1iNXj{Yz9vlkdgh{MU4h!C z*5%Kq`l*#!i!Np%J+_D?zCFo?smsT59kbVH*w3tulU!_k@-njxoR`%0U{zI z+P(4o$Yyzn6*VcT&$N+OLX}V`8ofzU*t1kqgN}-YFW1|j>Io>Y{pG16uqo#n9Og5H zz0UU+geh9qXn4AaqW_#to5(k4=F4eQMNEFujc+?1V956C&cd~vtTaj&1?!YPK5HmI zqyFPkWH|AJxpXlHaU_iNqd)=FgX|H(LQBWMyji7(z0B@#{M;f2QTm%;R$b35%67^Fu#*?(PnJWWeXT+b|iBlc=gD=`!k z{wy;F3|v=iU}fe@C*%CiX>0S_n(Wqvno?RbAHf1U3!8f&t~arqoNr6b8}fSnRKAdv zcgR(<;_0?E&3HUN{eS=d{i4wki@|b>h#78@*uwG=lkVoo4Be_xDlRd>Zo&~8Vt#I> zDGer*JVE(B-|5#)@J}jgL&tQ+W?k&D{wyOf<@GJq+G!m+>x!Cv=JMk|rN+TaoIydm7DF ztq%#?4m*Z+hvePny_e!m(WCF0sQsXg1{Nw+M;7B_%Ws3-68IyBRKX^Wv$X|{UhCuBjO~?gKC@CYn2IJPLh25pHST%ekfu5WL`D8>`Z1gsJ5zp*^B;as{?Zq z<%)^HxE}E*@3r5@&bmBWaW&m{ooU$ViB&z zaJX?Xi_4UNe3O@>orsk;a$N$VUrr~KZ%()J+ucOH_6-HF$auyA`rB=PmPHe@x9_*? z2pn*Oy)>~2d^#(QBUpS4U9ZLhzg}Cw;kH{p{fEoB2YEsDSM$iO0DG*`8$o z@L5{5mFZuLd{964zg#gkYW2MEN5l;F3>kka@Nn6$8cW7o^gb5D*obTD+Absfs(JFm z2j9jvj|bJYXSI70@mpDW;|aMv5j|T=fy)fa$y$nXzmN3W$o8&WIicGueXshAbU#!L zH6TdmWiAWvTZcANaJ?e={9|6XP^;fP4;TA2Y9xJAQ^`Bq@4y~I0$twNL4DNX0U@6U zIJ(xxG&+-2{Sqm-=0fqt)ecY3b;T|;2F*0-jJLgG&v#wA0{Gp!FAmfqq^Cc#McLp8 ziDWKU^mm-jc@t;l8`0;|neUg0-p{^?GIuyIU5pK?8Oy3H?rUu#QBP6i(W$EXW-B=1 z$|9()ngtHZAw?vPzA)So2z*dk^1PN?8)Xz|Q%{^VnWv+L!k>;4(9ll_Uj;mE4 zYzA<3`rbC=qZ978T^OHBVXdQ`%<(XTk-x&*s_db>N#pmylydHdW7O8gE zz`Y``GYH=EpmxaG_^UfB-ZZ4x=Kgwt7%dCA0CmSJ_U`!WG>-)sjA#e7q z)g_z-O3~MeqSTsrzw+gAUI!*FN*Gm}cGZ~oFPqA|Yp|ci$7;X7w!0i55p;dxn;TJ7 zrl|e$N6FXpEFn_PjuWqbzB1Unqc!pa9>%#j3|gE<=*Y>*2T7M0nV4w$uWQp6rF;+Z z7BYN5Xjxhw@kZK%Q)Y1$c}zzaD#}n$PykLr@j{I`%IB!LU%w0!wM(HEwoJUXlbefe zzJ*Fj)&!AbZ={fRus+{YZGBd4)|Pto=u!0s>S8$BINOI3_RSlDs^Z6@DVg$5zxKtF z&$=NVbYpBAr1xcYT(+JSuh*Q$$*?U6~`Aink|C zBs8(HWV2~h@y&Yy^${WA9XQZ7Ha4l2G02X-B(bXLd@l9ubQ%>e>N(B(1kx?Twx%i! z@Y1ZEe}6^$T2q^B5mGC&>nNJ4E6f$pXJw^*alSVn>7S>2m}%4t3z+`jq~yo!+#7bH zzbpJi?L;Z+h#Y^Dr|Tt>8C7@o15F$L&{=^%g{kvt`5wnX_nL;u*oEh6Io&(I^*nOI zYh^F4%fS@({`$(cUp~ZHAu|@Eg6|AkzCyA}62g&tRHw^l>d_rE!l(;miMfEYDfK861Fsf+4@^eVek-{Y;3{O-lQ9>19o}^bz zE07Bu&yzjPMCK)M-qbo+`;h+||6Tsg8r9WYqGz519csv9)Q~`u=9W+{HgxnJ-K9Qb zg>g@D5&6z`UkADeLFXFr4)HBqI(28g*r08Lj@hYw-Y+*V+;P-$aGYk)up9BlJlL&C zJVQbwB;_7!kQDgRh+utOHq~gS7s#p7+wx>1U?HHud=+vRLCPS&#r4^wBOv35pC-%n z_)g$YQM#$niq5C!no|d1-XUFGp(wMpcV!rY+eQlNoC}Bud+hff2aA~*)Pc9->?ZA4 zcZdY{*TXE_Y8*`r-=1 zBNAARW?8U9!U96>qQN9`!dyc<#NTfczY6gnu?V5dBbYtG`k;2W*SNIkT41r#9iDMP z9g%8!%1S#o;pYu+;ont2@QZftC1n^M(Ius@r_9@p?Pj-)>w9iJxsS_z(gfBarM@29 zlQut!Vu=^H4-nZe&+ThVFz6&6Itl3sZo$BOq?SJWtA+#DA&Lx_5#)~6B5RkGqLBgz z;+ULW0bdIUC6y)05-@Ovtu1K?l^l0C^4=Ma}1uoQ=xYPfoZ zl~2+rkzDdVJ4fteN4^XUG;O~4@j>Z*t+s*KGg^0`K0)w2~Nfi7Hvm|B@CJGizg}0$+Zwx_Lk7pC!jYw^O$3{P zCcPdhZPQ5i`5ZoM-6fd>iyl=&QKyfDup))G&Sf4!-4WeH?2PD#-M?SqJ5U>u@AZ>L zq?=l>@sJRuHzj1G@!HvLbg~oddes%@3is?h99R}i5`V57eC=Um*%sMsQ%7w@;uW&M zGZ)%@NwACi4?)08`ftx(g@j7_WSh49+*Mz!bQInv-;*?(K#S84_-eNbmdHpDQ!LMN zVh~eTN9Hk3mZJi;Zb-OGCp%TCC{x5M#%l#pTIFy*|7ti7ztVqK=HsB>3lzK9^Bj&d z#ML!^%BUYBM=h+1174P2Fc?T)hLB^m9V$Jqc=)9{csuN37hiZFUuK+SSR2SkFCZ%X z=JPB5X*(%lnrf7D`&5e%VU-s#+`4Y%^zrnm6SpBR3$mSlstX=;qXFL@B7kokC7%j> zF6&_H+2V-~ECZW)pUC(fsi#qgZvBaF(GgpQX(HOxTpiY`$5S@2CmaWez92<2khp}1 z0@NTyuNO%}KNAn!4MCofK>+s~;?6rXY-Fyk0O`L!@>WQX1=v3C(Fe8aN-Zi<@M`3= z6ECY1{N;?>K9HpQ$ki zX+#SOO=dHC%N*T%jg^OlL8MV3+{E3*L2+bgL%s3?(W+qGJS1gQ=q;L@QvD0C$24CF zRKA=ho_+TBom@OxXvinJ|4KDw698~@VxczaK+OVudGeU^RZM9Wj*;V7vl=a`2M@#Pz3ne%*`wl> z!|%;J?*zIW=IXkPTKsR$6`C0#i9TeL#2F$aJ{Lrn^V1CZH&I4qP&qBTcC>SV(+W2| zsY3PzyNA)vN4$yEV4L&DKjbb-kwL)u8=^rofV~M5UX@+l;TXBY+7mgF2*)ck%oU$u zDEN4x;EVd-(7DDZN`^IsBDucu2xX$;XR7meXhpNd5ttJgs=kFB@+pMezdYdxoJvg;5uxfBNkXEUCX=cIdGU?9_s zKXvDU zp7ghRhEQ!osG+uDNv08i&QB#%moG29&)NEFrQ=Dtk!uZVEmcQ+k%qg9RMQ!EOunR> z4r%=;35&8;{-l{acQW@gYrbKm`@hOH1@U#r!Iu*?tJSU9+(eHISREFF z$wI&*V#e*2kDpQ;@H=cUl_&!%s14%w7|9sb9Z`!Tv> zR3jOJ(s6O0n&#;^^{zH2nR0aEHsPoC>Q_9JP4dIAm&Mei;P$pSdm~ zzdis;u?@sU#E-WW#-TIiA?xLwNKNODQgC20jc5^qg$~cp%8He% z;#4^1%K!tf|NCb}zzIR^&p=XUn|txx%xd`;6~MYxv-;0C-FWV>TWK~wM$M{m--6l& z%}ICy7hHq5+Vk$tyv-C+aG`rFpQQJR7dvBQL>{)BS5dZ1YG`JFkK*~GW| zk5g|YySaa;@>!^#pWw2;fsk=u=E=_<;Nx3c5t%hrpHomj#{ASIdV)cu5_Z?&^FWsH1 zrPwXnSq6UQs{+JsT(xDYNmhV%I_OQp_a9I(dFe_x0=f^yvhV`-oCorBM%m4X)kJ8^ z)xEML8cAk!hVV1~U&vzzlAck6(wK{jOH?#7T1D0ebNNNREi8F+6LZ^5m3Q3jIc0D{ z@H_Wc$E{iPKNr00eY@8G^(!W&P@P zS?lNb4-DVY;45*Sz?JvrkHeU2D4B7=_L4y5V?b(2(deQ8uU{Dhl~fclrJdz z^){0L>M{WK8s8^NwH%Tna&D11B-@!m?0DS&cyq)s5-dDkF5hiqWbO7O`{DLjdB;Sh zti#J)bX_iA1i4zuz}V1mC@bL|#FP3!ZuNBQ$C`|c42jZZ1gRXZEom8GdJk z9I)AwpdO+dj?KwZ0^h#>z9(}bG^uBcvC6-wYY2#^kud;|`BB6Qx~-co8pzak#ou$p z?_kCtRwoG|5OA2`HQ0pmQacVqe}crFz-K)Mu}SB1U<9l@+r&;~>&P7KI|WELvMn{X~!Le!o<;#^ZTM?yf^62ss&==>=1EJ`J3`#q6S&)Js3_4cMCyoo9Y8 zL5$CCS{2;TM{F?KxW&G>JqcQn2d?A79TX_?c{fW@QSbI0pl7WWM>V^MP*;0|<^Qe{ zYhe;``0{X(7S0}N^>TLI@9li*&yRKumhAJ1vQ?H{bv+fkpi-HoXykkP3@!wd`svMw zMt>J)D)a;>`Gl>GZ2e4FgRCGnB_3XlSxr`St4HWKD1GAzCoG(oW#G=@ z9Dn1D$2t8D;QXJVsYK2&RH@4Kwc*QG2CGg@E>07(AB=pBTW`eoX01F-_L@>kJGIql zRdnpmN8?m~l{AD)%g4WYL(@-n!#xdUqaSscw644beA2pd zE_vF+7GD2TU9&Ly|rzY)0;sI1KLj z<1HR`=WO&qMmSg68h7G>@-o%as`JGvy&}(w${GGPEb~I!umHofgCsL_`tRmZDKEvN z%A|Jb*ErErc`+&HV`fu4z;=xa$yuZ%eDPA%?U16=8$B*Ic zt%0)^`^80o>1T2v19u#O9RYs%o&YSBuPoyD`2VeqP&0Wbzuw{Cu6r)7{?^YusSG{r zXeE;lZddFc9VQG=2)Tt$Y3n@x?{&D!!fFB1F7v*9Om!%+_D&3)bu=3$U;_R3L-io$ z9w|e-0&R4DG8E0Fzvb^Br(QI85>)XxnKrLtg{+pIfa%R4D;>+JV!05;y!rx%v8<8( zZ@II5kv8rycc+JkGnOrHNasQ634o~^T}22kEhK76 zql4d{4X`sBH$Fw1owbYaS`YNeu;s2rM~_xwDQ$KhGo2d9=O}Zv_Vad1fHZ{Y&3!yH ze>OjV5oJ)IYX98IOgOFgey49TYe|jVmvPdvja%q?kCOUvKNUr|s61>nE@iBPF7im*(e%Jq3#sn)z2tqz}^913?Q4+&_*J)-4G&lRj@Hv_e^%6*LR|5;r!8 z=w#19O&cApwCL;kFdGA3pCk?Tn;{3TTp-}~vVPjwpzXEC<+=RO@Nl8{h-`*#DYQ^N zp6U?PEpggc=KXQu`Qc@tFqLU7P~VEK#mU=oXkU*leC@fQla zUjdCz0-&Rm+T^G`xve5xhi^(Ywi2~p`}U)h;96ByOst%(mBI!80EZI@7Dk@47Ku6< z2UsAzg+D?{8!o|4q=mEg&*Ri^8 zy}2lt#%1?ZkcZB#nBpsM^`F5fWQd86RRz*y>w`IjuEiZI<>QJIcOcJADL5SzOemXf znvI2vE|$VojlFMm$zFOa-Ep5|u0t(y>()9qJwIVyn9kAcf~uZ_hh(wqHJxsH4>+ls zR*OgX{olra53aQxc4O4JiAXU#dBWrPD|O&a;O#2)?ne}GkXI^jO@%lfo8Q-aSkK3r z_`Byl?jX^Z;FiT+$Nq6W{^>z$)S@hyYlthxlC;#bWrbSU`TXRnf6FZPzoR^9jSL)d zWIVGu&Xv>0an%V2D3 z{k=z&`s%cnu!K_VYD_~Mu;?Qp|o@;w7wgel-%6h z2kng~p*W&3yG=&~^r`{I^+)-$*!!X&@mMpa^VlF~lgVR~Xt!PeY#2a~7o9S|YW%aT z=8EoGo6zo3rIS~K_Dw+qn|kYe+dTS-+Y%`LZMV9uAJ}p}xPaJ9B#wBPsq>wknOW-k z>j!@mpFI!eE`pC~yhH4QOC7?b#1fuz+i$0(~V=g|5ir2I=r}$d7-EU*5u4fMxn*YKQoaO!lyGXm`_sVRfJOb!+ zOT|v&h6w)I+f|udf5S8Yt@HWhKQ><1$5LIL*0nlKChgb*J#KZXbz`u%32Sv<`S%z5 z-}y~XoeU3|9XlE2D?AA@K`hE25gBu$a}d2R963gJbAJlMAmYw$JQ;1V1?g#43@T)T zWxF0)xRr3ziLDSZ&)w9!hx(m99_zkHuQ;I-iH<1Ql}9O#cY&%iZ67|EUZ^OAP~&A@ zTG^JP3&N5f^`!t=ODS5~(TYPIP%wc5Ww^w<5_f`*6mtIS>!Xw$yfuiUS@dEH!m640 z{&yObr;V8Lx-_~)goMPlchbcHTKc}LJvRuoCIde|>2_=trB`;`zaD9ohxm$G4w0OV zHnQF838I#8bRHv*&zFBZ>t)u$4TBU-H{vPxKJAVASt~e&b0puE9&y^opiWyM+UT(7 zvsd0xA>2Bwm5n1kDEcNn;okM7D5hB6BO6IgvGX9;fvmwnfyB>0g5d1esOzO6OU^x5 zVA??_)qCPo{LY$D8fFk6TEo=95fZAskMpbj-?i{`oZjh#J+%EE3Y>a%pMRiu8MPQ} z64U!J2xQCBWEcYbXV@qUL|0eW_wU~etb#aYDpLhDm5rwoMaRI{+Q~ok0)^%hw!`e_ zr~b|EAe^1*KnpO4*$SAMBSWPO`P&n^}&5AQqsA?r+NLVn)lK;soR9BmR^G(>p!{M>%?b8skp`q;Ha z9%W+3?H~4Co8=l7iy!y5MB{I&RXa9ZG)~mr{kOvoJ zk}rp~!$-(0;WQp3fuN)romiq>il3$f*qJ#yNmgIJ-Qsr<_&*{9?YLNiJr#IjKi_Z7 z0$QXLt7bw@QnKs%Z0F?oSVK9s6cG!i?6LT{Y5-HdoIj-n;jpV=SFk%Cjsh7L3l`<_ zNWXs@>UI~u<$92Q%E(mH&b8#78cv3rJsVUDqfV~DbI?jIG7?Ot1ZWxHwu!t`&Ks_An5 zo~($O8NGANH?y>&Kw$%}083*4C{kjXv~8=fxFjcU{Dy!dJxqN~V5x zILzuaJKl*2d++W|L7T?724e)#Vg?r@$TGm!R&9L=faBX zDhM_g0B)Sayb@0Ic&BWMHzZeEoxI36%Ok_(P7=3XC1+3Or{n9A!efqogip$+o3bGO z$$GMh=nU$s14xzZhU!NLiy?1zxN%mV-negIu!kEO)YJPCr=XAmRD~3OF-0Da=F-N2 zwz4MaQB|THo6(pZl}=zIsxHG->Z0H*N+yy+A11sUcI-3H#fi{AY|1|QLB8fKM}#yj zMfWDspA%>@gpV}E#AOA;n#OL?zJJopNA{1)yeE8fe^CM$Nk$b# zMW&Ndc8SDMWqm+UkT@R~sbwlXL=yQu2}0^3JnixbIpq@;7Ut*Iebx=3jp+tteaUDI z#{c1i{+Af~-w#(GIn9y%h4GvdDA~$r=_bK5_yAY5k}7 z?$6fc*YiiL~q{a#d~$8k+&4gT_5FTWIDT8-=HJWsEk!` zqYadGTxsiQt4T$&B-cA1GDbfd567?g1@~tjg+oC}DGzy$J#k}w-PQn#(2~0euQxw2 zHL(4IDM6jl#DMYtqVxXSwbb;4Pr?GVE1#S&bXP|(_}owakzy_z!*Pz?b$zU1&hq>R zFII^{%*Ityr<%fX_czy#?0!jcu<)o;UY~?v8FXXEkYO5k2g={4D5boo3nS%O*d`kz z;{|+J)0l|pRUB`d)lB%uP|eUmX5)#&eY=&hH6vPX5>IP}wiU{$Ive>BC_*!G3gLK(`yT7>o{kQbk@5J2@-c=9BW{w+o9cPSYWK0bDt?_9#6>|Og z{tBi>+-}OD`uBzwrviPfh&_3k+?g&IG{44&Tfu})_1p=HcpYiNgj;;3$r}HM5&U0j z5PYDy{m5nUcCXQ56mV8i(P6I0mJbc{j>4{Ms2fPN)lN%dFt1R{6tZ|mG=Kqcd)q<# z&GO@*gXRq?0r#hup?h?j|bNn%d|_ov#<kWq3yes~ZHakh5^!blvrOkZ@c<5Bez%!kNKGdWWs94H*Jy}IZay#5e#8C$ zi(0|CU^9m0pG;Gx9^T&^xFu+e=rumAm#r)OqcFJHENJ*7H6%Stsk4sBPT*@dp;R;6 zheZ}Tph7)IOA2-b?Y)R6f0wjSue%tAubt?O>^nYQnKa{g4{|XzFNIreIY3 zK}3fY5VF8+6jlHVs?n`7UejU@BM}iXZ-^LDDg5605dNy*8DG|ltpUXRzOr?{1pp0N zS+qi+LDu}lC@OYZYEM0l4}`+mx4r48nY$(olwpCQn+7Qs4?;>Ev}h(N;%en~$MkJt zBk4Q$)2uVUw9+5xLOI+}O~;DWsR=u!QjgX?sK1;LzrO#~?Nrg=X~6e+>42%|#aC7& zXJ9iOh?XEPu9QlnO);k#WtQse%|b2bZ5*{_^xR+7H=}N;*AM02{kDE!3N*$muA_H7 z`t?h^{fI+3vcRt!76hQo!U$Q$XlX(RSQ$z3 z5Xs0e$tHbuYFp_GDEA+uw&z5D;%+WoT`#Iye8^%J47?Ya0NuWzWy^c`e)du4w{DI4 z?F%n+0{(Tv^^)YoWE($k(9bO?tL^tf?St9W=vczSh2ih`NO7%tSbb#*an~XMO58M~Q--DBAS*NkRQKBDy`JOahVG70H_6 zu9Fk=+6waE%q>)#_3)5q+G{i~3zO)@Fd$FOns72&%b70SU%5<|PK{1P)+V+n-C$l~m7p@v!KPxRB|kE$qLQa#^9TNjW-sDFAX!*mO} zskdeo^*GFZ%0EpYacoW$V)%+lbgA>h8#X~U(7qMB+V+r#@q!d-Rh0-bXzd1STEhA2 zr9mNRhYsoT!fV8iF$vhF^zYzXN)98|SUO zS#-jiIlPPfqZL(nB=H5*%_7qoa=^JG`VxbKLw3HKLuz9dZOoEiVki7CBu(VWC}uL8 z2CU0|3Ey{=zAyzTvbmvahuEB~pegLI*qXRaQ7Y7j?z00s^D$m-$w}}d&D(psg_amL zbNM=kZp}EmBUJ{2PHYd9zMhsg@3$0>7WwA#G6klK{8Kb{gn3{y=1FXf6H{!QkHRAo zk?oP({0$E+C3Cl6o9g5c5&2YXr|s~@pz-C>5CLhR25U^5G0<;a!z7$H$7s$CL$(AG z-bX7s@$kSQ7jL-lTuiv0D8Kv7^kSH-yrc=Y)=(Q7g^`oPNKFv?NUlDycXESp-SnLE zz*ZVmYZ&c0a^5PH0I#!5ojHZ-k^ zXySZA7@#75FkRuI{KJ1a?4Aj;i0UhDcB;l#228OcLZ*c(P32jzqfejq9=c3Nj7OaJ zMH2f-#;&&@P>UVD^qVe4d-bTZp14lB49hf-l-k(&_*SKA)O_V_H-IucO-Fs?J_so} zPkkMwxhG7c3cpt)>eZHn74_J&o2mI0F7EVHq;!4p3~i@a@;cM9y?)$Ls*lFJ0}5m$ zld;&+&(nOWWmlDFQ_+vwE2q3xd(wje(xPSViy_U&;*e-(1!6CLV^9Cqr>YxaXj($m zgWjZV;u+I!QX#1mi?~*xhEz|AiOqwVds`u;*NgLQ_!)M&jmMW^jw&9W8xxK#_^?g6 zl5O_ZF0!&D8Q2sog{$Wt3!h1qTq=eucq!=PQvA^*XJrJ3s&Rtc;> z)%mD%+uu~gZaiAAj*ygcL98#M4TIJjz3)Rr%E`$1NOT*}>WPY;okkz6C?`P9k;#<4 zF-@3n-K^}Y{!$4?6-T5Zq%D*JhHjS%Z)y@a55xV(_q0DkGj$q^wE=u7I|<|;wOObD z@eOrU3AF?#cx{;}Q#3mPLKK$s+5g59@en3&9+2aAhTz)E1S53;;V+V8VFJc35MxS! zUVIu(F+V2!&c&SQXL_CiMU_u*VLu6Cs|>O9yM#UoYLg__uBuvoD7F(5IkJXUHVhp6 zeSohdV&qp2uR{`GeofvYR`Y8J^cH3pj}@AjH(9+u^U|TsBM201HV)5#9YPWap=SQl z(u=b6&7xP#IauO~Zzx@tJ!c2CiW<`y(VC3#r? zTMt}K-F@K0)3(B;-u}wE1f$mg8K=V|L!#{!T{mQ3zS`n8lY)rwSj5;rs;cl;{ zo*Zj7IV$7RQs@I)umjqWoH1iAUOniRk1`n-&p~p|Hj%eNw93fV^L1wYoH@}Bi+%A+ zoCu)cVC3g78CIbQN}=XDr*RzcqZ|2Z{go^J&YL-JlfUNvf7q6)GyZ+tP6V+)Z`R<{F(i<&dMx%mJ5 z#(muPX69<{1t72fd2?^lNRb0cl|SC#zeoNr-y_zAQSl&?x)6d)`gdCY3v3fc9UE;c|eM!*R?cY!o8+HHOt}8E1uR= zS=V>()#1v^n@SMf6depnsKp*F{{nZRcFCQ-NUHU|C8QbA#)h>?Gm3idlKdtTQ~F$6 zi#O`5{i}Pn#ypa_%S`}^zC4`HPbd4%fQ2Ygi*T=BW3xzZHd_6*C|y43X(V(=0IDK{ z2^SM>wHO_V+ZknLyE}Aw7~*9Z(3_-S!^pY+fod^WLq&O@*Y8!qdcUKJ@BO%m$H|6# zzFU-jOZddRu-Ku@CL-JRfAe){UDdzsm11f{^*C*RDw#y)6>6SQ;hWgJEh=fNR$CijH z#8V2~T#wBVQ#}ycs-$p{_&)Gy{{eDQp$;xOlg_C2i)>7FlY#zNebL}&t38oXz0t&U zpf-7U`-q{#ewJ%N+T(E!1e;#h|cKPdzjt76RTg*YAN%A`y!Y@qB zG(0`U?^k(J{;!yOUXy5*`s3LF_g*U@=UGpoCmWAbMiMgu&)27)0N2RL^;^Gp5Drud z|M31DchMjpqE59l!Yo2VwWkvBkbjuSK7`vuQUEy=6@ZLni17tz9hgC<^hpD$hq%ka>(m zGc0CF#MG83E2+#hETCdUXuF;C4_*9CmWc1UsB_W&=lpK|<<~);bF>9{4glYOIx$AD z%p-x@@qh7w7P!Gc)?+x9)HqlCPDkeI$Eh*k!|{C?Zq_FJTMZ3TG} z6n-c}+zI3vl`7*l#-RC`RAH}uk7*-$>X|aV8d~V91gIaBd;ltFJ(dHP^tgB3mO;Vc zj7EYgwATcm(iAtvKpgFFTFQD?1S4=n`WFTN@5Msr)}AWZ*V zsL-{^Fm*ta?!`Wi=%p!+Q4&bd)U*TNqxBr~($g4jPIt!hts3`n&b)J?D+fCMMjqju zQBRxWuLS;7KIIo}7x$mEid%ip{a&q=6r0WQ-E)Cx!^i-)8xD^{BGBY@z~C!-JOge{ z-AYXFSETWi^1dDd`cS>Ytp#iflh$hYx}jat5xIyi!q;0?^x_vHyb=)chhAl6-q>*(Xe%pS9$w{u@W^DXvqYgDQHQ#FGCo_f%N8!UB^uex7?ad+kNOV%b zCtJ(OVN;9IzJBzXz2lZ^0#v>I&uuKUi-l}|CysxiTWDJTDQd#jS3no-Q(QW#AQJa1 z_=OPnVk->)kILxN5xMV7VKIex_HKJyh?6uhTN0w5*_Q$-GxDMEQx?M>U!ucxQ?dVgXO!rxTA>_3FQq; zq*W$}nR)(aB<7r&c}GWqwsmwxk7TlYB5~+^=N-H-{SMi9?Ko7} zkc!hx7&-PQQH_{2(aS4|tSp6_mdSY%-_5caQAv1SKdUkubJ^*AVLkTF%(nc_(fw3| z2E9Do5|J3f7W%2Og;|V-#&a|aE;Qm4GzWvraX%BUfI1SwUNRoj(kM7Ft}Y(hdpKq` z_2(VvIMBl(UbFlm&6O|(v?3kG7Su@f!1bgr5{K6oDz{{W zvlX>DuGy0xi5W|_Sk(xAzo;F#8*w4EyD!ey^S0Ka!$E@GdR*Iv0fzt?WmGfqr(+&3 za7ky%PKMFjW^hx?$P9F7P;-2_f875~Q(autKBsbQQN@ypB%f z4$WS!_~qfOF;0{gnn&v(AE00oARSty7Aorp^t$f?R8iC5@@l*(aB;P~(%f^m7EO-RU^}Kglu$q&I^@WVu zx8{`y^2(O&GH$C8=f4wR&8v7#4(mE$4RsZl9X0hiFys#(qa4E(iA|&2dxvXg_ zxEw28(g-=d+>$PXBg`X(pP8e#C@z~Ir4}9FTC83C5$F)APs8r~YxN6mzWO`cde#u$D&Yaskbdp94 zud{e0a}t?DrxY1<(L_Ij^J8O2D605LmKh2G3?$d9Z@3Da8?l{r(9NY9DQqfqfFjf$ zx^zY~;Zy>#!}NsG?tMQcBp3DIKmMYCH^gDh-Yp=y;r`%Z4tk!7V?`6~Y8OetSR7e^ zmFq)a#^PHhc@mV{37QtK_FE4GOyX(+!y^6=6Rid}+h|K=^FxyzhVmk<4E4^$>>rl= zpSi>B5`)^CkoorZWYqg@QAaw4Xj95^&Br|A5(t&BC(G^7&=5c_s{`ONpYRhr?6P-K zz_4GoRaR8=IcU468(yCA+=4@0wySD8A9(iXj=vuc)AwlghiX3l zYy`v9P5J8j2jN3hQIAyv@3+r~VWBfuDtx6U@?ft16Y!x799B+F&M32QlolRv8JU@} zhtjX0kbGP*+pU!;veu2!eB9x8vV^qF)z$CrEno2FcYHjL8f0r7dRbHcqW&;a;_EbDzvPh_JexHbsLghD=Kbd#5qs-^{>suDx zZ zFDK6y$H}HJwF|)eAEUkc{=P_aYV(g^t)y9o&pAtDwaW-{I9uV(`JS{n;SkT6yW+dh z#Im|W0LGSU3_lkfaqR37Az&FBrH!);Z08rO;9@jZOO8ihNwW?Ud`^T%H?7NM{`0sH z86CI<=ogma{tR%aiWj)cmCF8mS#hV0BwX4utB4O#mOiUeGdf51Ez4EIYiv^iTtXeY zPtCxa-T@XH%U=b5(D6(zfw)B80_kgV+v9HY=2vLzXgEXaZ6Af~pOa-cdA=wl`SHfp zaYYwqb9gb3+-o?=5=>t~-*_{8~__A?P(viumrsaj0KyVi~iTB;@%T zZd08;QHdV_Rk*DpCN*3P^J_J1GM+>R^{lTVywm|JM_b?d%P-W*BQ@2$43R!(`=68A zy5#Fjyk!w|?2hb6ZHYCgWn+b1>@v)#`k5{EBl{bxJ&Q>@ zg;qEu!bDqJ2Cm#+Kiarari`l2zV|iAgK#;diVDkB>TFAi;dVhe?;}Fhq2#pe>^NHL z(WUiv(@+{RxlHq^TvWbn2 zTNkB>ZW@06Tn(?)YLv=hib~ASC&<_b8&M>g@AGl_$+X*bX|qXYo0BX4ZHC_{Rm?&p zjJ(OhCpRkA+i1IpNi?p?cpKGFM`q)4VkFHDEQ2y_vZIro#_|FD=L0YA%KZobKi1v? zD$2HN8-`)%?gjyo4n+y+5EPW|MnDh*hVJeXL`rD{36YY{p%DZT=^UifQM$i#yzl#Y zp8x&d_y4|ct>3kDsmyiFb)B*IKKDM3V`Cd^ci}PVy(s2R9#P^YZI!V`PaUv*)RutM zPlW{YadY$E3#bqv0qYV2uBEA*w?-%G`~#YX1=4Y_bYTa&)aGI2VExf0-;;B7@$N z>;>I8Ayp9YrDNca!<03jyC}*r{|#sp zLsZ3x#2g@*o?j0r8yeF2hTNc`1;OQ+MFe=^Ct3IJ0hAtk3n29zc7=kUQQ(<}1dvgD z!zXD)Wk(!W{8eNHIZ0;k1o{~%=#voOU0#?gD=RxXmK**Wy+ac|Q)C;~w1>|faw9Cg zv$GS0jz{~TUuZ3e2Vh>HXjcxQnGz=ZH-X_l%a#A*-#Kk+9Or!i5u(-ixLo&R%Y}eR zR|H{S9Brtn=hn^Z4_u)B3#C3w@Qd`isnQb9A+4ai%f`kgW@e~~;yPi2B(U&EFhgg& z95g5FBKyM)9Jwj^`g1^>AD-0*wniIIMSeJ~8eli=5)9|%-VC5ye+$axjf#%`mI}{1 ziX>tjr3wK7$*bzH*icR19oj2U?EH3%i>n0WLA19?4w8>_L3qW_9X^155fz0q)7Zsk z7ijGcDAYBS4E)u>rs|d_pn{=6iz9o6ttFogB>2j7VEm!vet0F zm@1~lQrX~la2GGU)KE{(zy`9@BP;s` zQl{E<^|h&(0k(jZg2E~>@Q;_(i8)1*nGf6Nx}5t%xY3b=5>PNfvKVCUc8`>IUYdaGh!CY>{PGP`6lN7Y0@U0Twu((f-(SKK3{~y-s05+SRC$iJavO|Kv z%%9#j1a`Y=!{a;nw|2ijTz?~E;K_14=KhUf4pDjPc}F@Y-gLZ1Vj6rC^?9K$b_s10 zc|Gm-4>J5UQYO5X@hBZ%cv;UM>Dgb_y}G_oDO7pYTK0oIp=COKz{&sO8j-to)}&MR zBs`bxulseSy4iXuMLLq-ccXQ1cdH%nvZQxm7Q_d?sb-==4(vV&hA~@BN#S@~(jTjp zs}P015k@vWE{jdhSH&fkGwfzJr^r zGpWmc@EPxA%?N(`@xb82K!G4Ui5_?lUw`9@CXp)3$wLJ0u)6cd_+NDip0+dJ&Xn~d zuc+~UiIzK3v{Ye%*JVb%atyogNA~YjoyOhS=CGScsExY1^eK^&h(VUo$t^V6iB$RY z&r9)`S}e!tmU%z$zSij<{s^ATD=B9C*9g+j920!+Yn!Oa!T;AMJ3s&X=l{&?r1Wd}q=^ z?!}88ZgJHN*rr06WCwJEVeYZ$lR(OXJR38WG z;*iX2Ro(@y=rIR6R1ep`=BOET_$vg3%aB~yFMm^F{;2!W5Xe*kk*vW7a_WiaGU6cnZT_7%LN+dJEQ$U8Hy*>32!NVIrbn+>p(qpR9 zg0ubVY#3L$_fhNbODYnve_)7$vS=q z-;|!by}J}g4i00Q`uhE};@N;n)s0VwLO{eq!B`TO_sAH{0%(ZU_1R>5XZw#B9sp@n zTn#**UF?sJkDsq|pg)B(zHw&s1yGVM)w$_}Mr5H9YUCkq4hI_`@$@x2vqU{FHy1Ik zXU5(R2tF&u06PsNBf>)q1^JztRD#mX0t{+365rptbxTv%mmK8=Nh_ zx4UhQ0&G~Kz@PEs!s~=O_WGL5UWaYls~^rXEgJ<3AQ}j zO$b5d#cB&-1*Sf}hY^5mit=kWO0XRV3!+os_kB4su?6))z=L4`$7)z(D;th@YhT`R z>i{pMOY&V}Zee9iD+sq-dQSHn8yoBD2$ZMrsr@!zp_W!6j^OCCp|`86s~Wh!7JPr^ zZN>1Edud2VV8b`VpT7&ef%>krd*Hjq0lk%p9p~Y28W|&F{O>*{-xpx9 z!!{Um^-R6P?Jyb3`{wSU<|u>>J~9YScRSOcBKHfm%k*!HiJc;sxF(OXy_cT;*-5$f@4zl< zcnZArfHxsHK*~`z02Fe=!!Cf|8Sg8BUbCW}wWBLr1(1ZpQb971Wqr3{B}qk7ODphv zS7aD}*xB_9BP(iPaVC2~^b0%O3LukF(blG*n*%h%Oln9dY6A##n*(su)E0o)L`qw9 zLQg=J`8{_BV+)OIIBM+7P3ghv@&?dbNmv)yIj2jJ%s^QM;=+tW?njEzA&z?98-I+< z{l_HeL{hA9xf_Z0?=Eh*GTw?p!)7KmXCdjY)d)yzBqE<9#W+(8MhN_MO@u5?C!yBb ze_qygBMJ?F?}sNkN^&Y7TJkr*KDnA!%A*WVZ=*1ns^i@w!0sN5Rr$T(xBrBnPvp-9 z)aZLtRm)pLw1km2>u8=oFR3~UKC$ByD$s<% z31CB`$Gg0T$KgxyIF$(gHhg5-qC>NF*uCCtFG*`;;3(EL2c>ZTn#857&ua_Z;;r9L zvFhtAfBop$NumEUO*8h(v3!rB4yeARjvIY)3Y;1Bc#&T{{FUPyU;ZYtfZnkoSY%`b zB)C9A9M*gB=TVyh-RYD-4qw>nnW4g6X=Kv;wVccTk}kRV|TZIlI`bCLW&qoQp2 zQ#7eCy5jCxd>~S|uKoiEG%>I}F~VA69K$3^0ht^#XAgJG1cy=Erx1V)vku@}A{WPwq_hEx--dck^ui`h z5G(>1^&%z>0h2_-o=m`+zvby0z0_x4i~>^^E_>ddtS=s3*O?5Hg1HVREst+p{Jre; zla?!?&Btyha`Rx%Y3Pc>Eb!9Zkz()fHFFCnCAr_&o9l?(c>(rVgCXaGzdRfbl+?V# zwMg6zL?dAvWGWz>V(e=_<=8pt) zXW#ZW%(k&Wc%cwdL@w>WFW36|Oy1zpQwxCcVhmidSdYm$ULCD?`s!YMwUt$lPDa{u z{r+-sB+En_^7$z~cC3lvT^?86aCf2MCH=F2(vXT$bGr}uKje68Yr|)jI~!(&f#<9X zxbzwJg1j8gUPpyyg1KlF;wKi*92KwYwSYnx9INg1+c&;G^TEV!LxAS(!0~6UU(zIV z!R?8^Y@X#G7EOJ=QB%ovEh*{Yc-F3$-FW$2#X{rr(B-mwJIVQmeF>O4J(KTy^pDsI z4-}o==op;ZjA(6eLJt#?vxcEGGjXX+avz=R9RS1qJ@CevT(PQju!HA?FGhk~FsY{z z?}eWVKlo*O-~ERPehA^Dh_$hVq|X5&juGTofhSmtTer2eTCN(8BzK)n_;mNX#H;PH z=h{EAmihWrR~;qll$$>q#5M2!S%a_&o3COZ&S!F=CvGNrp(1Q$QGe|t)p(WbB|1yI zc8~6DV!A0bCx9=t?3=oZqGM^(`*{)ecz`70xN`+7it#ffw`45ywM?(;qj%mT1ZC12CM(x7?_i#scd0}SNsJFrscr=(B9XDhY}dhDpS^Z= zkG6shnd?4+g!|i{wJ2>5fI!i3U1J1VOR~5F(sRFJn%Z`U`i_G(%kdEQiyQ1(MtLza zxlmQGHK~81WAIYfIV}#;)^}u2?=#@G{Koz36=V}|!Vb^)!-Ofh3ml5P8Uc@x-sP9AH1?Xz}3Yr{b@UFGU5ORU&VBH3tyHXU|x zAUhMTj)dZBb9>k{V0l9_dpcCQN60?g%;-?94# zcU~>ua0E8z&qhX^S^il<28yc6AN%nW>ec6n+wPyA3QXfP7E0S#jL#xh^=>r1%1HpK zZ5__De0v(cj^d)n0-Mpl+dkxs?hS>%GS%E#^MwmSGhD{#Tdl^j{SFB&Gt)g>@9fp> z_T0Zq0{tHRK}19(O{h(_3XSvr)hA(<@awrl8!e&ObwDxTv_*m_6#@~=^cR5_-iOW`MkMB;WciC~?}|*=;i9$x-1Ad; zPETE1-B@!c3$4rfTGIUNNKkA>#OJdu4`r30p=8eIC-2Jc?0Im3<3x99h>+v*aJslW zq{0}*!8SaE7yV&tDaBs-YxDnVvWjTQaz)N$KrZLyyHN@N@VTNM0sm+1Y~w3VY$vUFo< zMR_ZFeDf|nYBS$NwgjaVX%zt_nZz5xKsBLK(_xB)n%d^O08b)2KR~?#+~C{{0W-6! zmqT}NZ}mEp$by$;t=+ija%Q`BS$8&}L);4~eF{?lCAK$ScnN;uhvm_avT{_m1f{A5 z@h2)&Mm~AX({`ySD+fH8!k{fbHc%qC$`$B)H+O0COZ^|SkH`o1>Wa+Vx>fHu_`nd; z%XGF+PrBrTyRB3kcAU#WGF3y9NC1L-rW7D6kvjC&a}JAemZ|+rbxbCn1VY`L^kv$@ zFW&XTUJ*LpcbG}PZRtgwb-(eNkiX+erU^Zbo#$Q3hHLNVBgj4$~vf zau9#cg70Ak!9NQCCe>xzb~{pz>Fu*i_%pT)H0hQW0e;Fkye(ckwKNdx0jC~`+psb1 z`f?F-N=d7NhRuB+EDV;H_s(tP`*+*19~*90Dg)<1udgcmP;?nf-zf12L876d!K?iJ z{Ze_K@@hnczqc*_ZWscXczO|Q-UCoE`%FvdCD>WmLwj62+wdS(_!!pv*&m+dU6Wf} zoLf9H9P{Jrh1glf(#X8wr&$^2&vrx4L&=i2k1S=D9mW5|6LY9%5{G2IO{tXjrjJ{2 zkW6jMQC3g!5`H&9Qz@1r&<7g0C8%E^-mKN>VP+9_WGSs)S4bKZr+}W{2_UgzLf-pUQrJ=p%iXNwm zc>PZh!_p$`W8385)VS%c{yIB+U6$edQ$YGf82kF3p*V6 zw`@Q#lHV zuM+x1s~kP}>VHKfQ3e48#{Vp?C^TOyuwGEnd~jd|VS+CFzCWJt--DM1$3_17jxDX9 zqvb0jw`>s&Xra}a!&g_xq9)<@LVv#jh?yf)b2=CP-fBJVkj$hM!RH!^S;1%R7up_! zQQfI*Fu5f8a98Ke;@-e%K>c{z6`%N~8|mrd?r_J_joz`c zy&Hp{^%zmtkhU#^ZiMDZ+p>|#--AIJsSbmJ%rW4JE zUap2}hL%O??$2gP;0K%#BRk$NQAwt2Q~HIEK{kFq>OAGUw}z^EEx4F3Yhp8>VE?EY zD&IaXoe32h6yTgl*?2sQ`2KTw(&3S&fwi{jTKT};zgMNK;9rg#X86Kn_Pnj0puXk0 zXBrShhfD&GN<_4nGCk~$(rk``^)D{#2J3dmHJJ26MxZZY%rM8(FWuydJWPOCDbn!W})khsp>^Y7lF`|CP`U+ZrTnfk`j6@Q%jXG%IAeEt(7)nczbzFctwM;uU} zSGA_FUQdCWp53+V|8n=Y>u;5-g>@^(w(*Kjl_tBVyPQlLX2IA2Qo~yUwvpWI?nF;M zyl1(+=Mj#__%m>aS!8lKE;9Gizh}S{Sh0{9D>niPYNA5JP_)rV5Wo;pH8%0Vsdum6ZL$kKBcCw8gMUv!q7R=&^wsv zyzD%Ax+{--ylJUO9cYoNTkY9uZ4Nbs2HKrKfw!hI#R$_YTPMi(BTZ7O|C!s~o5(a` zxrwv(Icr>4e|)nE9#zMV_kWJ6+nGZ3*83;v#~F^U%l|rfXas$JnvhlRKiNO(*I<3o zau5+p?$hSCHKCE(as9}F)Px&eM~OrT^je&x@) zGf8vHe~xRwHnI5(H19M9tLF4G`R?Lp&22rNTu}JOHU^U*1cm=?A^xaqxME~Ug}46m zZ8()m|5&q=;~C^-T4m?dKek5+ovKCVKM$M=L7@pal_WS9z7AeB5UQI0G?OvJET~lxbqqD@mO&Dsj&P{2*sy;nmx#(V-S`g!`>uioS5|Qk!y@FLBP_6= z-ncXUsDvOYYtu_@o|%VG^3unMn@b)<<%mNPJR?~ggj0Ez9nT{KEcrnusq2sVH#bQS z1MepviAo=lnxi5y`ag&HrP@6F*T7eJY@m|9Hy}H&fJ^BbgBXKcUgq_$SwB2rQTpxQWT>dTjcBa8ZivLF*0PjNSti$Ls4oouKSIY?} z$#Rijhqy;|#cDz>p1Q!kt6xpWg%pW9u`jmw&J!`R!Yv@>E=MhLo+e3!9P5&k_Sg)? z2p$X%HSdc_Oo7X)gDkd3eV@$ye%JdOeV6v%N0vFa+l#P#e?W(etirHa(o!ZYX84B5 zT%sFv9Vdywv1Cg{L`Z8YwarBW!s_BcT7g^~2^i|4t zkgq;hod*u`*Z@pL`_|gZYA*2l(y}*h36F*I??uog7#!c^sq)N6Dl^`m?A4D__2)mT zc9_#RTEwqB2e!UkRncpxy;3Q$3v%f`Nfk&+w8QiQ_~r%<`02*@wQSI)Vlam<#i-Ez z`Nw|1&TN2HLQaCQ)7Wt`NnNEEv)2?xNkx&CY+j*D@|*;*dFiR4<6KqyBXphEQQtaR zI`dLZt8`gaF8KB97tm?EOy<(+eaYzinLQ2U#opMtqm)a5 zkTpt{#S93AgA_NsqrBJ(X!-;7@EeOv?g~NArWpD{)NC-7Qbx!aeC1EUnwxC(^|V4~ z%p#fr{Y(VfPVzxk3^~pO>d0cwKNL>JdU;VpxXQj!Mf~Dj$4zxPuANrgb7`+V_}35Q z`1s0ofvz!l?(a?8SjQE{*is+4jXR(NHLMeROqy41{{D9g-tM8*h|FNjvyLdv-)#~4 zXYgot#8`nfIXYVVk<&24kBq3Bjp|!S`N_=Y%j5IYKGZ{P+uon@EOQsg#f+b+m*wBK zeIGKRgxY`Gw>_UAF$5$f8Dn~A>93CiuOs<^d_rJw4#@Doe*HQnj>3=?uO(AiSzAkzpLrxD>ynX`^~4~mTsP>^;F9I_ef0N> z_+j1rCQTHPx3Yi&P-GP!G@Z`A{*IJvv#@FQzL;;}M&}Cr<#(P%ab=NU%AbbJez)r$ zwu4>`n|WEZT{8}6C-Krnhrf0%xfbKI5&po{MGRInR1dOYh>?3K1a(21Fz(AAQ%G2+ zx~IH*;Hyy@sb1aXD`c6yAhKp=BvPR=^ZUVQ=VqX~_h>Z3ctqzYu(3ykla_mL5!j~A z&bNIjxk0*R4QSKBM#2G+6>k{SHD2kK8+!vp6b$6?!nGqkh-ISL(VfFtxxa|MnJ;2_DZ7X%EQBH!X|U zd|Tp@CFXlocvvN_)(3V?C9~@}2fYnc8X0hPx(>hrU|d9Q-C6^X;KJ9aYSV_izEk_9 zFjYe?sBQP}O@HcZ8tQ3i>xBlA*4TGlx#*zFx#0VrwS7dABDCc;XBwSnga>Naae5c) z3ttmsFO?B`D4ZdQGB$Q^4kTfGzo)f&-}Tp1nmshfHtpW|iIJlPrjCnWs-y@|I+5~y z2{Q|_m>T~jZ{v(!VG!_cNf4ItBeP)Jy`2Q<}u0s@@UH z`!Y3|4_OIaC-OVky54Z5Pxmi4VBuA`?+Ec0n;3cL@{w`M+3++36++6r zC=JKb^B-70&nvsfUG&ULF+z>m>wT++F-w6)F)QBo^SfHL+N$CGr6l91aHBHtNcr|} zJbgj@t5#tNA4VPOTd_75(Xn%kN7pX0KA%ZENz7xU*z==|@7^+rEWx%O;QoH^JEkZ@ zj59?k*?eorKuO}!J4b56WyeS&R1LRbc*-@58p5!E+DPZOf1-Moq@YSV1PSHwcl6*= z0fbyF@y(3)QMvlRPga%8`>3WM6}+gzvX`3!iaoL;*3q1nI~e_6JpPd-zkn=$+p-z@ z5zc=G1k9NFNC;}*l3@1lI6igKVv$;oGWE5KZZTj5$bv>fqFO}AjhRHs$`VrGg8u$A3yM}iaJh05 zS5OnBM;`_?1@%lo@mBNzMG{aOeZW$v4OsR91%RiWhlPBVq~@!24&ax07_X#(v~$7z zjpt~-aNQ=1=^R`)I;9H0M`$viohbvRAE)!H7_KQBgyGf+C%mGjy&81@71`;#So3)@ z#e>g=FUYXL9)$LqKdGAAf1E0bWJ`v{?vefT%Pp96X{sM%aY$KGsPGJim;^cd(!gyl zCKLLrOS&Zf$Eagx9?CY5{B>xykY0O#LZ=B^z?dj4Ly-QzKZjF;mayTc0)%>+Jp+r4 z>am&qD*FnEM7HSC2SLsqT>qPex5M*-2hNm-Pwwk`@r#4>YQ9N&>@e7o*uB+E&;CAY zIe>9zfk7ndR6Z@J9e5v9EwnOu?<__7N*;`KDM#e|gz?cmP8ao_JHcI6Oc-^vhW+hw zP-QW&S_rQLa8#&o8E#Js{QlOLvvIhw-0CA0V&j~&gw3>Zgv7|w5O1`}u;Eb6zI%FF zl4WmKf2RSpTyS9NBsl2VQ-!~RfA;*UVT(rd`; zib!BMJOBb)=2}hnzVCLYE86U_fi=cs-r*_d<*?Bi_5ui11M#T@qmCf-P#C#a!#W$i z@I_oKv)}$u{gaG)eG%gy%uvl(+{l9nwvWv9-<~Re`e3ZmhwCy*SmEzmRKo;Vl|C?n_fMGo|O&x1dL$Nw#Q(av5 z@(RVuGqx<}$#FXl>;U~=6PF?%)9235Jvp>Irn{9&2H2Kg|8ZELOx8gQrV`x*T?vS5 z-Syf?%+oM7$?ME;YI~EGHd$-$$?1H4#He1q@(G_f3uNJz!=hgod6XB<5<@(`ro}Eb zFjrD+!q$ji1iJhb7EKKm{~Y{{uQS#+cKD>_&!B>`x&#^GTPU{q1Ecl;jPg&r7umjh z!LF+*Iv~Qu5(N#+=3{iWU!>^f_K`f>Rqd}9(sXB!Vs#-Aa+>p%3@^0PP0xwd#uZGF z2_p5Pr*4S}(zd{L=rtJH8h==O(0|h6-xcu18k6_rB`VF*7Eu7P@;NVkm&0^^kX~i1 zk=F@yfB^)z|0lCpV&>%arAJQNWk0rOjib;;8u2yI3Xd5KF`uR`x%e^9YnxiK}kTcHUGCB^nq zPr<$g&*cf?4#>g10&M1;;0!pbs}fV~>!c*0q%pVss*ear3Lk9TmT+Cx9jP@sR$|^J z41B1ztyJ~cW~`P3WY2Da3Z2FGnQOw}NaXgyWnR_1QGCGJ8`5v@Seh!qhFU(Q4i+(o z-`Myy$XIGGauB=p%}X^W)vI7Ul{1Hc(2YGC<#%@a=5_>Rog}ZU8$lAM16hIQ8z!vC zbg8H0L-q69>E5res@+e_^|cSQF0l|qX!H_RQ$e!wFtsS3>LL%Gz&Fu*7;WIBdFwug#xVLQrzc@xWLmf_Fl-28$P1~byTE`kA%DW5x@_WR~lrUx8PV8-0JK~47 z61=~%KR+MTkzcG_O+%Sv91~P*Dj~QlEjxYRuitOFOZ{7Rbvjc30g(V2Fkk((G zu)yCmZoa0n9-~bkaVhA@n+IL&RyhZM_U&IkoGzT5p!oYaJB-mIbN;q{2z3IWpxrw+ zHwU;0I<*~zAvrk9l*l7;GifO{oe#AMtsZ!nI?2hIP7*sIf+BQpz4))=X6ATNw*o;x z%Q>B<*zLVKM(L{)OQ1p)jDbzWBr%oS5iVeL#)yiyjunoSr|X@}baizN4QHl5*#H4$ zMr!Il%8(oo7JdpOe8lyAy5&1zq_Mi$6cACZWm~QYIOry>5`%$~yHjCFczhNQWQQ57 z!;ABKHvt#Mv442DSCaf2AvTJxunh@{#`NmyL7^6T2H20DqE#L}I5h)6rsCq_>&u^k z#@r{k=eI@9ClizJJo*2~GUCX@l^D#+rjG3>)M97tKAQ8pIAEzs5cemU294>G_SwUG zPC2FBw(9eLbpA+U9m~gC9Saf;%cIiKZr<&lgvF@Dlw=AQb$)4atBub1d#F)9RYv0K zWO;LQ6O})h=m0!ETnH9rz{=J&#^TYQR?M;AqhOTdg`zK6Kr|5IJO$JcNBGTTs>b>^J zyr62+4XSl`X#b61xWtvCwze^R8u>XHO)GC^&VSjFp5Pj<5`n4p{xKIX?_8s+4G3Um z7F&es(JHE{Y`=wPJOuuKipORu7M*GCH_)Mb$%DJI(3-&ml?DL-bb_H~NByNg3ubot z$PB+ixVBvNf&GQ2mbB!&U2@@OWtbN2gK2N}$~37Vnma=A7l2LBek}GT!|4#P-t~M< z#R)Kr_ceTRV1N{an9N=z(ox=Gc&qzF{^^+gyQRg9>bJ%nPnf({UX_)Vq1av5&+% zXQn>8{jU#x-{EDQxr4|W3zW?9o3&K|^eqlkwdWQ*dXTx@PXQ10m<<;_cR58cQE5G4 zn)fYSk*`?vcu!O5IyVb^`=~2S^WqKni)h7%1(3cK7c`jRpmnNYxzks=i^P zcKf@d|9$Gw{}meXh2M*zH=YN^VpDXWzA6QBgz~y@Qxaz81Wcx}W1yQ`4QY$&?|(Qa z3xHm?WojrUo(jMlr~zv&6eEWEg6LLQ85CZjDfK1=M>_UNf5+8^O1B)2u_cU@W zJqS%ph`T$uIBXOgUz06~?VM!E5i0zR1%cOBi3?yJassd3s4!L)WOjhqKGJgU)-zJ` zsjC$;uut$~W88Gr^^;R_qOp$oH+8l{g)guwhZ0*oKeNqb(R>FM?@t=fO#TOwQv_DO z3n=kws;a{Pu(l8Q%_ir4LIHh#kW32`x zM-XUrYT{k^f!H2-sPBkyJXaJWH9y|{xO;tdTsP?+t_aei9e-!nL?s3Y=N!bZj-|J; z(WFlh7?br+7I7E+%>m$XLlgWV=7hmcFZ==#hrk$yqDbY`TJxa)1wexqzC>()V@F{# z%e%ZK-?Q&YB^?h6w8KiomOxenr*b;J*+CGMEe0DvB62b<3OqZ|UtYAPx~ z9qVgDXC`<{AjP8sB!zT+9y(L6j-rtioCj?ir3i@Jsd#hWom(tezjSoC-f>>pApCxD zDj*MRn&KWqnEMhdl*9qg1i4z_?T-E1Q?hpAX+hytZdgUY(0qHKbiD`B)09lm9EdPuik8U!Q$6 zX-*Y^LB`Gi%yWg!qn~A>!mJ&LL=Jm(f^^qVwjQirUP*Ax2jI@ZuT1CeyY!jAc!2am z8IT+s1!%Y&QD+D{!0kU!&W|PIo@?$)N~A|Ola!r zMJ0R4Ih}P*3d`e;6`QvDkeJUvy^8wEdvd=NnYS_BE1NRReOd-2wyqE-PM!vVUcvfR zVjmXZUS|xv(;`*uEro11;E4;MLyA_aLRx-#aj`8>_A3}k-kYr_X6Gyt22#DDp}dgv zZhH2bkN&u|3R`p_J#Nl0wab6WR8D~-%@q)cz=oN_iI(^j-pihW9r|%9*JmO!J2Z6 zwRyYuzmynT@BSp4A$xlIdzBSV4JRYn?|u^-EIb>SZh)k)=~4lYI(yAoFzx_ghM3`A z!;*(%871Px>&u+#VAH2RVg+1WT|pGW>kYOos%-KIZV4|!<23^t78YdfC0HWSPY zi>u*QRvpPbMSRaEn15o%N4hLWk>YwOthUn7qbgZ}CHf_f%T#;U(IHd^S(0CI^U(0I z`an%!yMo2!I#6W!tio{DZotKBYAO(-wlpzWj|^i>Gfq!8J(#jD0?4nk>E5y`*Y8|M zzN+ij{WHG2(w#@G$f%8OX}$@a60Tg+$VBrw?59tk3d#>>H*36{0iX}qLdA&D$1O$! z9~By@*00J=L!xi5Qahl=@uTY`~?e=Q1tIs;O zh~mBq@K#~HiO)FC*^%0%nCL*h9*w0VYP>OC%StX7*Fr5G^8&6>p@7by(mJ3}O#LB& zS-K6Rw0~a`f#_Qh0xEXklpRN6O3%O~tk{;5e}ya^=N!~}>lp}^)4Uje)ePIjxZuWJ zmPfNLzw_M|!WlDOjKp?vk8Z9$^|}29&L`IzJq_O+n_~dL%^{fzG2fz znQq*!r~Ulx1vtURu*6C4y0bGP7#tCO#&)#3$*{*Cs;sZSRgv~=Z;shXbCcnv+S3UydZioOWBa`M)*!Q6j+y>6dPc@^>3bxv zc2sGkZDj2p9PDF;w}0(p3kGy7uP^>T_djt@`F?#E9U$#6wsTNQ%$(pO@=;@RR*g^T z7XOK`J&0^t-mq<83PNsg8wRD6X;Q^6Ei2}=nlzNVW22sW%-FN~((tB@1dB{gl;d>u zDkw2TJ_=F$xv5)WDhV7uYL6t2ig#;6A{TCYG>ngp=EUj7kE^L@rkw3QauC|pK%j{R zboVj-KHP|6KBBrylQeddn{>j>>*>5K)o%+R`{sE5Zs+F(4GA!&fR+K^nJ5H7SkF_? z0KDz_bA$*@($ovlbIvm9jTIn*+)z@2aEyE@Gjwl^8Vp-DIVmY=)C0i+iXW3*N3kOZ zm}a|b@@(OD`SE9RD6U^#17|Qy)3TPbHW1|gbu8K8#emO{v60#pgVTHGiv<0jLMrH>#?Z1r({ORT}{uYoJX-1!uDyt9k4e+2CCJ@odH*em6 zKr@poe1VxzYO7}j*chPBm_>n--j|wJ+;q33&@468c`tl0>I|ACjd1POaaL#+;vJRw zluDAcp2UF!$7i9|0M=!-*(ducJDoQa}-3;p`+ezH*{-yjAnd^IxY%e@0d|IUZa zRM4Anb+!|2TLHM@RR>qsUqBei@Gz=77j8CJON_jMK%`iK%ELkDszURmhK@gH+nf~f zJM@eD-!)yya8nz_nd)Z=VDzjiDmZ1u?3QE_cIlI`|H`RSg^{RBVK_Z3daz>>cQkLt zXP0MOo|xhC7ZFINk+5a6_360E5Z-wT|IO1~jvu00!lYu|^wA(5v zYN;efEWufR)5YX?tg;@|u{k0|i5u%MHR>T=<1PY#ay8hA7Xy z8^2pHY#PPyB-$dZ;NL~oNR0Dem`E~>crB@Kua{DtXi}s9sy2s^Kn9A?$fxVBvESuD zgD{#re`cqqMsOF5rH_IbCJzQ2P~GkTzKa(BKh!2y=GdN2(^Hx9crY@xH^R_Rte?Ng zNti6qFi%8@*=wL^UU;Tsm@v|6XY#KBfJ%SwI-0ZqoEq8Iln?%adgt#ed^pk#Ar2~$ z8CpXB0~zlBlI8h-Mzdr422LxW&nU~PpIA^6=)Rl)T1mOSLdpI@(*w!ecG%IUJEVniwJz-Gle$b7NoeL6nHWM4u5M|~aoAKNn zEi~57lLMU85|aMGZ-7uK2M^v$+S&*s-)aCX8X#Ngx}XSJ)f78xY9?JOzbdtr9&z76 zSErO^pTa$*rKPRG^&6ElYNa%C?LCy$IR}tL^-n+@Kr2Na`A~MKQiPxT0bvi&QGZn^ ziBZZ9gg6R#x=DI^dq;K0?qSx(#l;o4$_JJ}Vfge-{z(bw1-vj196K?fyFZ+kihyMb z@cr75RR#l$L#8`ag&rPc2rFS!7N~_~5puJjD=Hh5vKEIC$+m$mWvWKD_KrP6p>l4Y zHDI-j>f&|?*c%lcG^uXWQ#|gDcevJh4%+!Qe%&foU?gq$q&85x&O#lj-uk}Gu=~3? zBr=j5OnV$8RoyLP;*Mo6L3v%C6`F`YZ;{XO-jiuSxpj1|askM4P^HM%VGL4JaTtIY zxe^oMZR~5=k?(S=4+6EGV)S)^h|(ZoaiEJZ@PkBK&fsuM@}4@6d#zU?FC6zB;m5|3 z?rAA>>*q8k%yQ#~A|KSz$y5zw*jL^SCR2h3Ai#E9C6cBDS_}0qwKb3}cC)O71}-=# zc3Pub2Mccx%cf+bpOf)vFf@pGk9duZZ1RST2w|WV&g6$q(OSNKS2srihZ81Bjhn^rH&U zVtqKWdPL4GY-Cg9G^jdk8?aXMEN(9LuoR0Q#L2B5G(oVY1049k!ll8Pd!MJHY)k3( z6AQ^gxneKPMOZ9&>y~21VH%pRvwDbfCa&N>NwDf9ZYcyzW`}(A<#|c6dsI2PHKq0j zEGJ002C_q97l(aoAZhMtPh7w&dF3+yOBpcZf+Sn<`on2*;W8QHZMVyyNegY@-B#6^ zh3nE3+2M92PYh%Xa)+7%|9ng33tpNn7_^b1v^!cjn$vuzkJ`qjkYCnK! zaA`l3<(COQVzlRW z7ER(mdbB#j)F=v-UER8<;!6?r`@dof|L+ZC=xxy7PvFJL)@=7<8;R8r^GJU9{M~uX zThH*8xv1lU`19xPg1mfKr3^I!#U{oH@!Vkj!!ee{!FNO7d;hC9M67tY*Jenk1_hBe!yt!P6@~dcR?rPfq24-ve~Px?QPw>GJfO+Cg*R`uq|s& zV0|Dx?)A9TtrOwCoespC)NM1oPR22F*w79k&P2DIPsOhI;!RVbH(z(r+m8f5<*B_8 z)iG(g=#TSNNeH6&;kD1Mu_~KPkiB#(Qa5mzWP%G!-B?4>*Tg0k@D zHq@EzBt6Hs7&gTv2e;N%->jKfK;9^m%IcuU4H7FIkY_Z*6!O{?}3w&YTnZ|8GSd`2v9nlxxN%H3$0_u&qHL zj*_5&03bB@4;Cm0hfw|!2LScJb_gQ@fJ5+k92Ph=x|9Hvk;z9^5DVla{GQ`=KrDyf z38XU`paTK!dm|V}VdgBBOj4*DAnvhZbtv^p_@VC7VXr_Ck`U*8^5<5QQ8Z9y+I=L* zRgyp}NN?jv-C0|bM`E7*7ut2mjdb$=%FphKG{=L1Y>o3KgP6+-P|lByiwk?5ojvRh zK1~8>paaQVQ!&CfZ9vsso4o;pFpGo;`XmTQ4}q$sCm_p8?SB!viE1fHo!Z%u{%kn& zb38CPluXgGAX)c~$rJ6rqjn44l_f!xn>7E3X#{fooFAYkpi-HKB5UV$0NS>)fh@_F zkn^X=KIXuk+4=+_YJ5Rwke>w_7l3d4^4yZ2CMyUJ2C!W*F)>n?2U^3kmDJ`x)^h?? zRaMvGZs0tFGu}=Iau09E%M4!c3-4gkAh;OVy;#><5ICN_#)8p!Fgh(MEp9mzH`MFn76yn-LApY-cN&>Z9HY5O*ns6-gTaOVfSTiF_Yd}a_?sW0$ z2=HmCmwmoq3X$GG4*;ZZ0LAO0B@`1Ai>2aY4Vt-ZbbyptAen>hR1 zFn#eCQ-a9=xORx#G_l})RiYd#eg~BJMganCc&ZBM&FrjzH(&8L4!S`3g$1pF3gg@` z**tjgjb?8GbHZ<+>EXH%u`Nl??+BUl`@QlR2Aavf%E(X&UkVmyV@nT3;J`pl39z`8 z+sGzW1X@vzl3;tFSAwx7svkhfoB*UH@H${e3oJEAjQ#DA3BYtR4L_ySGX{cI_qg>0 z?t#O)<7{2L9|HE3VzQu2W*!Q9$w#;d+Kq8qefYzNx3zzYxw;dMX)%%I@&rX6?@3fM zlgTGTg7s?Re>lD(PJuviq$Oq9xsQP6X=8p3o{|y+DbgmiZ+NIBA?8lA>%mUpZf98R0Ku)VW$axX(DWA%0*-0a*e82muo3 zagNs*cR7gBp>$BjznSho7h28yeqmZu&KxyKHtVCi_Vxns+a{YHk=aWk;+;NOK^`n` zTpy58+&VTlX@|z6u^ALj*`p3TQSTTu0+d2<*#UYow^%89kG-n0Bftv!iHsZF5s?2(O_@?ifGQhPDN6MhRDM2h z6I(oI_IeP6$ue?d1AvQA_Xah|Fi`mUp8o)!8+xE6aS)Fi>Fe=}%eILaX!&9rbW8d1 z)vZ~`rMnjylix+xUA$AR&$~jvEOi0$u92?u?vp^HG8T{Y0Zu7U;?>Y4fwF;JSW_z> zNu;y*;04e+1q1PUvC);ImF2LP-bsC_{}o$R-nL|#jg?hrBv{3ucmf(31GO}!;WSO4 zvIj&OfRZDqW(+lyR(^S40VE|sCE+n4uvK@!jO(dLHw0&3tgA%7e5rV9nII_)&_JRG zl~)5i73v>zyAIRwpRvLqs5;zm^6#&KSLA8Kpsx668g!eNO{oB#F9uLT!TIRj>H8qq zrAG-!qy}uA9?d#MCiJ@z$a#QO}# zAFPBxuu9TxgTw|rnlO&l5d$ebeMWkE@%aEomB+5(3(&Nt%bW0k$PuV%hzc0TZQrK; zq7Vd4PrqZ0S5A56_Vf+n4R!n`#|);dAf|Op2e1|?X=(M(zuy{eE&$p6sXTcdDhgKk zlDrINx_-44ZaK|-d$3(}OpF}HVZtRa`$%Jv&LYn}GD)1?EY>T1mh@vT9gL<(vKJS`6W|JEmlq7J;K zY)?u2tiE(%quh0iV)u=c{`5#Fsv|n(EeIU^_F2(MqRr$ zAzgw9(jg!X3X%d+(w)*FQqmyZAgLhTAtl{iiw^1Tke2R-Js0{s`+dLT+x|pY>%M1< zIp&D-vcxpq1%(6H8f}zLai_x#cY42g_Z!*@e}VS>WnpB53s5;#Ykp)Zq6V}vq7%!Z z0omVV0}7~#wH!|f=iHorZ|L536KP3C9$w%fXAZKr&9|m2HlX;VH|-S0Lu3(OcV8vZ<43nxK?8d z4~P7)n)yR)$C&S4CUVxqS>xyde~PB#{OJKhaf8tQTm4JdMY|sBD1B_=TMxoZ7Pp8& zqNuVDJvcvEP@o~D@4Msq_%}e%Wgy`-+BDFO#xw==L;1j2e>1($V73AtqaOTm*iOWd z6EP74nwgb6y$M)yarG&{ZQNg2HzOYg1cGs@-^zkcE8xxEzKmH+xDi?Otp7<*(&)e_Izdm@@Ky{D#LuRm}S7s z=9)c5l-vC`yrZcD3MRF9OLe}Cr+>y*EF=v>iQ94LnBT@xtZtgBiISJm%SRi&eONb( zYLL;`HKj3drhdkyMWL_c9z(mPT#)|>w3ofQlAWoZVz!UD6l z)Uq--JakE9gqqW5c-qLjB!(yZu%Iv^31^9~`Mzg%ihQ_Zl-VFi$0u7T04_9K*z6!S z4OEoH;eyB1-KEbKx8iuCKR0&*I(kTsQpZ}@lmhysNpK^1TrknB=e<#|1a4TZ4B;JN zMEK@0Hu>~4@Mh6i6*Z_M(Gx-K%K?QS+#A=koy6;|NSH-Fx~-xh{GGiZ37L$GTBo~n z(FSjFWW=`fr>vykfar_m>t4bg^+#=l7&0b!(Qo-1`Lq~b-GO?%^}o#Ky65;2+qEZ}{mgZCNqhJZJ#nsL#w0WcK?s>V%c z@4wVx0m;CfOSZ^FR4{KMCJ4{ms&i;t&)}4gmt!nhXaugzB#YH#6_&=yq7J~hV9N`l zN&(=rC~0W2vv*Hf=P>pX;!ZBi=}2CvHxr{@Z@zHG*r9|jz;HAuFqgwaUlypnl&n8L zxG}%w_Xyc+C%pYFaDQV5U>4n=Xqe99J?xXxX0lw~C8re%{DR@7m9~Y!2v6f-3g|>@ zc?p7AXspg;zGsfI)82*~z>UN&0=J8wplN}z=E00Tg2h|Kq<%0z2%Jy@@z*xKhXn|i zC2-@ILhdG!WTh*EgCbs*{~1_@rKNu>DR$RA=jPe@G|K<9ji4}$c!eKL$cf_LFM{Mi zSIwYoIB%4m;s*q1187@-_0y#!AG_6ptK0d08*>%spP|=`$jko!%h?n437|Zb1_X)XP|sgj3F|A^P@s;MM}~h~OQleGcMTePBb76( zJDTFhbTk9USUTPD{0@ZvS#s8FnEk3xQ=e&zuPgj?@46SG4>F2k&?SS=6pScJyM zogO>v-Sf5TLC4i`B4W9F_+Z?-jpkKS$xD?NQIzj#;=O~qag-=R&sXRu)QT9Dz;-@EJEd{K2t#A@zV=ALP2Sl#(v z(P*Yw{_Ony<5slyu&!ov7G$7~jw}>3Ml|(XsL+>|d{jUQ+7i_;XFvI2bZD%3bJlA; z6nb4i@# zoI~PX^-XaNXAois!B|tk)!3ZZebz_-dS|6~&O&^Te-1f3tr88&;{kn{DlH}qS1^<7 zBm2Ax>1Q=_8;TsCrqagL9`~U!yPhSj%%eK#j!cl9uTz{QzN22ozR78HzK*B@Rei%X zv;0c~acJx0t8i|cW5o{49*fnNfqrX=WfK;II|}JcWw0> zYSmDEJz;u&)RUw#9PqOzhrTekuwph($UzIN>UWlw%#WS{kJdN=;{y2iokHGFeV^J~ z@@KvvAP#)dC5&ZIU| zSNq+5(D_bg4P7jcZD9wx`DfYxw4&BrT1)lc9r}w>bDr?W+t-=QHJqTJjcAH!3Sbt_ zuoHaj>H8&*Um^CfIJTEq6(}j4i}lKC9Uhea@Xxrm#Hq~bI83-Luk_bg;g7`ug@6zI zjv*#2&^q(w5z*3_7(J%7ar{=Hf)HUDXDtWIJ2S@P=*oL6Bcob1q^J{ZG+6ur+v%n;pM)ifbAhIz1g+SVL? zA^#$gd(rAmx__a&0IK5__+Co9_PuRr7S|ra#F)+gNWF=wkEs&Vr~mX;9m!ods7b#- zM}@|&=BZI-r~5>BR8+f;_#*t7@{*yUVXb}J7m&)ZWP&5h9xHfX4$l81n@7wjnKksZ z+mE$ycDRK)#9an1HNN8;5D(6s^ToRDuTs6baC)p($$2i<-B*9W215f%TCs-W(3al< zo7{RfWll%`6I4ESRXV(?d@T9KUX}g^Sn;~XICSHd>J4Q!^;KTeU+0}BEaseFGY>^N z$;2H{cMJXi8qF!6;Rgp@%%pHMnOk#^_fN~gt6ki zGinsPu~oViJj6hJF*kO;DsYwZ>2>dsr@L%Z30KB|*9D+noi@s)!Md@hbz%gxTk%xA zbA3-V2Q{a1g05e4;INXHMlgsz_Qkan=E!UV?XmqVr5un@I$rq2$;__eO<^=xt6?ZU zZTAQPImHI>WHzIXU13$1lckhm?k5?wGU?4 z{?qA^m)Or4SA9WG`?nYP;rGpcKG}mFYR3BYKyBF5^@MlV$if=f`;ToT-Z>$W$VA>< z&Se`o_VGXo6m(y#-_K&t5&-VwV0$gbM` zE$_NdEf^Fs0rwKnZrqOZ=-F?xM(<>S=4ejWa+# zYsb{uT@shvlV(*cs;ML7pqQ66W#|4%9pPCneeDqAj4dU#oPQkISC62H1$o(KgM5Z_ z06gGNBeH90_q!2R_wPUeCb*)+y}o+mM5zQRo9cO8cR*!em1NN6vl>wDwP-$O8M^)ehrcl54=eyJB ztrB)#+#m3EFK5g7dGSWE_wK5>0VyY0k;!o4YpQ~5U&;o3VlLNs1f|5 z6q9~P-twYZAPW~dFtRV+<%BuX!U7kL?3fXh4)S{mm%oTh4o73%5&7f1%lW;VaGKxR zdyVB*Ts7H$m%MQzj?1%YKx~%+ay$9eTi4ZtS2Df6fCO*Jod$LU@iZ)mJ*(+pRlg(TP2pWf>OE$a$LX42LZ;SlT@R#S5#koqpEZ zL`X+4=!)r$NqL@6W!%{ont)8&<57aBh)n!E39=u4SR~*bt`vzb#3{ySx@;_%Bv@|C zLvZxymFrLcJ891Zz0~$S2hS)r&u zZ|!g5>*@#I=C@Y`R`scW{s_vS201=9-k2hm*-fTO@oK)Yx~*Mrd!lo_(}$`#enL&q z-(``@SIpG->@2*QiBs8B)xLw zE%u<6_${xCPwh(|m^Kbl&C~YnoRPFlu{ReQqR_MYZ0xvfdn%teLj{|vIlYD!Fr317U zqav>msbE6PWh_I|H$ABc)0a7S<>c34nz{tcjP##Mm%KH|DsnH2cFV~#Tga*`IEqJc z`d>EHeDg~eci)|^r2Nh z0{w^c6$2Fk>-f5$rK1=P#b;lA@s!Dbu{)W=0$^`$NcX>B)WBg2ifCyO$Hv@w5oY4% zp0&+3=+5R3{iGzDjS&86%VY%jG?``TQvqahAQH2f0&JRP4~>ldg8Bc zT%>r$5=Lq0w#OX1I9k20KHt<##y6CQe^55{9ayONH4Ww9AlVx-$30s?W<_5L^*15F`$&8N&$$% zlpTs}DBlY*Z^REH*Ru0W(o0*tagb-irTG+SC{V6DRyA#&?2#yOHW|4;@@RgYXa5z$ zWX`xnIXn-W(UC(O0i~||iPi^SsLizhgA1wy>p9@}l9*ijbr5n`{+Oh)SQSnBH7Y8H zwx@E>>4l$41@)48Tq?)nN0gp58cP2zd2gytmLZQ~J*E~J&fmmCZ#&Os-MaFAj-&z% z4^cTXYe8F=cA?cxRMR%Ra8+B^ynTPD>BFc@nX6BKDw&(sBLL)R!0xZn#|BjeT6Owu zAO%~qRbe{Q{3BwqR#`7<$`12gG%=%R@+}1UT{It21wZ~ivx|-4IkJMSoXmIqV8Wwd_2+@rf ztcte?xhfbJ@33lq)`auxiK_h3d^b~!oKKGn1=hKnX_(J#93h>A`eD4?P(RF;0X;x; zC=^3?)51d@CPLOor|!DOdcEB?hxx;2gRv}wmq~o}*KcQPkH1QJl34WxY*~L9kOnc8 z-#(+^9|~cxE^Y|skQNs1(sx@yjj42C?W4GzTC&vmleTBt>1fAbw=UjZqO`~Hj~86u zuo1ZXDXOINDY9n+a#cB^?+p>o(A^B>uVjoEL81v>ExLab2M#;5^{z_96&J|NHd!6|E(tUx{{cdwE_FPltAC0e{MTn5K@+7-vk9LKrleHUHU;a73lGsp` zS6>e(3DpC>DkKh=eiAm+5|($pdE**J19>Gb*;sOv9a+WN^9IXO@2}~TZBnI%-7$Km zbJTD%%^v-I9xhm++hqYLl9n@>1EB7M9VzZFS)Ab9--WUfT0dQOVK^3boj=a{e2HfN z9q#(~(9MsCRv$_vRdubB^>gn2b;rD03Qv?NvwaO+EOUHRcFEeN!~6rF^WKsV4s8i@jb34j>G4R(=z zy(sb}Wk1}ci$s-CIdzSk3me;MyKHa@k8i!8~l(M=hY|c z&QE_Mqb<aP7Gu&3PXLVp8vXs8ddH+zO9=)utpE57)V1OIQN0Q=#g z`fA`fc>nVQXaGRE8V(a48WY9Vd;ag?plp->8-)GOeP4(cQa6Ya0H7lz6wsFdWYKT` zxKKzBfB%4;#Qsl&<}ZKcVMxJ!HsL>i^*>%+PaXE|Xy^un9h!;mtiD0XJkK9q?0;N? z(P<$9#5!gYgL~{0rH3W$Gw?X;9+6KRue^%B5&$j^GW2ytRUyzWx`(F_Mmgv-%*WES zoqc0y%KxTxO+2J*2L89MuFXV*v&iIv7NExX6?h>;;-o!27>Xy-iYqFu1+8wNI&j%n z@R8GOQ1KaYe@LKgjZwSkNPZ;`{1Y0MUh$sbNlOz-0J#ET3m*xTc2uX%9He~ym*f80 zllv<3G!J(i=%~GWyrhjHz(Gd3NSTmTEn2Y#I^E9Ab|!}LKh&JPl`>d&t}F}!-}ms~ z)Q=hTHuGg|b}d>P7VS0T0m+IVEC_z!9I@Gs5MtGr99XbZ!K%f@?2SgKFnxL}eRb6M z(##G@I(Qgp)YeG84M3?O<+1aXHBXMW6o&p6(j0+Z3o-Z*&Rv$FKjv*MF0dX4Ss{;V zTEZ_1y9Cr8K#|`bmc!$e9qz7-rzNsR+Un=As#{gBp)5#9GDvcGV?T15s?Vvs0y>?i zN@}2O&2ui%?lj}Yl1ONcJeoP{RD|1}-$SR|-x2-&?O9DT+#aiGsjQJ)=7&wmitPCb z&`0+W`~Z=N20BZb4E@8*g75_(6cv42!MQQhCHwPJ`7+8FXm?m!3UZ_VI{IBEZ}h3EfSA%cu08^oMy5P7A-nusf$A-%I5c6E!rzFuV+ZFo?i>ja{KSTEK^ zMal~%2p)S=z>#BKgSN;=0u!VGWtpi$lmuonqE&Wh z9QSRupR?S^d3^JSuVB)boQ>+&a2|_DxL)`J=>3_wPg$S-;9$##@fTkPy0M!do>G@j zlFl{>xZlNgMUEWH!;w(~r#D=Y1-f6J!Ja)Te_{*kVQ7SB`X_ni$%dQrS$l4)|6)o$ zFRKT*$)Az2yzRMFia_ef9JpVf3v;zKF65onyJ7%p&H3Ehn6EC}C;78PzJ<0&E3+}& zqhCsTDg2yC>b`jXngJFQp{fcmq4DGWrwh>gt3>hOXM`X$KUqXvggl4aqIqkZkbBsr!!cXe}l z+cl6ElQ~!SKm)^Ca+`Y6C)TV7awXWMQLdSBbNV$hYV12tgqguDDp_tLO!3vNc7fyg4^<1HN`|w3 zszYWvI#sw(A!LrMilFPx`6Bt{N`%D{HITYOi1dMc805AD7c&r>=6HVlyIOePvr#*@ zfDL2s%^F)Jl97f371b~e_mw5a8bI53ua4?j$D(!r(Wse5zjUI^~%pvTZ-GAk^t zVc+bQN8Q|PjZkRCQtju#QPSWvgJ?Cd?REZqASJpN=$*I^d+vIyIig&7V8LmM$^C)0 z*>nhY~7{_B5FRJ}*AI;B?YBVTKo#B?rR z7Vvgb0q`}HTgJ>nC2;x?^(lv2Ji@cd#ljG>w=T24EycCbN^C;wwR<5pZ2#RhVDFZ^ z@>d4Ofgc&k;#)abosEO!ta8B<-(|05<4@?ly%kKpxT}Lepz|r^Mu7Eu2m*gIqeilYX z+7V3~iQzD@nDiEJY+H-0;s5cj5@700 zckvc=x7#>&?_8UU=DMK=asv`$T-4hFueRBLCt#eiSHB7LHC>^{?GP$gC^+mN&S+#1 z{07J8JPAFN35ViS3gZ1a{|*_Dr8N1^$Kn1DF9P67K7tS1+>>`8s@9Wx9Qw7ui|_Tn zS*5_$6Qp=}<@2W{OLYHyqoMiqRcezKM&RY(y#ctlhm7+PDRArlWlSFYB}ZDQGxvY} zrE0hsz7L|KGn!(}evs8UCo8{J+VV|5a?pL4Jf&p@;Y1-1?`tAOloR{{Npt zM-*xLLKTOA?Ys2+p=^!lu``MWE82f6*T9DX!U5y|tyepE4hr9Dc#-emR(hVBD_ckG zptJ}*6uEJbue+mYSYq6DT&&QskwP=}(7bCn2hp!Eo78y#Rwr-O@p5=TlAd7%m4RFo z7gwh{rsL|wk}gh+5C7g^};_G!2n`@1E2J~LD#1U z$gzRMZ*lE9A&^J_$s{%?>*T+VZ+(Ds^Ox1D2`F@ootBOcC4O{NRQH%|dS&&h zPw(Ho`*LIV4R2-9%>9ko106E4ZQF;C|U-(Z46NlE-YskNh>+8iu&tpLC+gb!}*TK zMeTdh8TtU#a9!yEz`g4gNK`nm-zZ$Y9tN+9?E%OQvsHyaVPZ8BhK$SFt1L~o*#nw7 zt=gR~e=#yOH5L4n!B+dSbJ~dziEL0FpN1##CDmD0&gVa<+Or$SVJlI=A(Bhai?%G6 zS{G7lJPpEhy9WoEXgqFqc#(YN~6z?2izFf9`gW(@${1feoQP)|j3|EB1mGXibSu%v8NyFZ!_x#?L zaN3A1>}QO|_31V4(}o%fCklvWlemwsYal=-qJws*jJ^4Y_!KPTl?0K@ngQhmC&ufp zHI)5t_sL~Ehp%o+k5uOqPWU8gU zHnKsd@?hfelPAZZ?kJSS2$YY}jy=>Of@a7nP(<))0Tc-Z@Y1C|^Q3iyu7&Xb7YWdb z+4Hbx@ytTz{-nRHyz{nb-cR~`N(Rc`qkElZ8*G!Q0ei=GhMwe?@t%v_68o1ii6|Z? z>L;HlOPaAMUg#7&5{Firboz@9d{M!(_&k2yb;rBnIeN7xvLi?rP`N0w)=T@Nf;y1> z&FshutJ^pXKhyVR-6*gtT6W@q!^vH&b%LZvs zQTdOwnz(|T`TL_>?;fpH{ zO;;ynw*x#+AHdq+3R&N0f99O~()|H$ z{c`ahB5Uvo04Ap1*&qS)?S`Z0a;GY{y8yKstjJI&%!FYHs@)!zxNXzg z(Iml$|LFZ&7v_wmju#$E{+JjlZ&8>=43P%%&H|Y)2y!Q4YYv}Z4k`csRN#I%~c z+3ygh?J8PAI?l=|xg;DhBTAy922zZ4y~EFFov;7o7k?w>9svs67f>-870~eVQxeTI zWklA5UgUc(MPsIqI<=u)B|pv1WIb&Q4OQdR)0C$mfys!)2M(hN$!=D) z*^UKok{q=rNBgzwEUqJ<@q1+@7uD@*D`#iAqt8~Dp?>+!J5B7T){DTRB7TA}HTan_fjaJY${ zIPa3PopX@B%keRLV^jtPvCtm|ARsZEaXz|ubqeOX4~14}scnL5G0VrkUt|E;-q}q5 z$Pr}*^#a^Pwu;zx9fG9H_M&{9aH(gHvQEnbY5RVa7V+nPbwQ$CHklaAFH9YI_1z}b zqMdPGh~?Fa&7*-Q3WL{*CigQQ$ioYSfT>ZbX<6&f#xan>$EyeN0&uR@YIR0s@x+_E z`qLGMvy=VB)C6n|mFM@M29g5gR`wQ~$-bRYo8Z#Js%oI6WrUjP8Jn#vsDki^y4 z7T~X43!%!YRL3*K{tIK=igQx}`FcKa-TuI{i36aRrR}^Hom8qKI0bTjq89dWNpW%F z@0nt?Yfi93TWE`=*Y_@>&JRtIOgW5hm#&R$S8=U*VfUkT?BbAgjA3hMVA3V`f@Yvup7=I z^k=nSH|BnSsI$vroTtTn12f7wtP2Bbj6J1TWy^516>5eK!-CdL{@ja%g#UPM!m?N-}kbiTa)NTvkXsy=qpMzQI{1~-?Wa2_}&C!=aMHa4_W z;VLc#SzuUzky{Cs0I$~k*4L2Mo`FMd@BpkZjKmWTV|GsO*p!*ecVnc{e4m$gh}sNv z$q28{el`M*BZK6C5w4!hr+OV`pS`CaYF_AVp&Mte$Fl#750=+fjW4$?91dANd*=w_ zxRC$bGS+8loQ|2VvIcRZ2V|JRkuA4a{q!zSwgE2Nes_+`g2-qF=aJW5EW5pK4;k3^ z?t$OSRw4HN%C!Sp;-APLO(o~(`RnWYAa+Anu5t~ip9Po<63km1Rlx2jAw;xlx}+(? zM7XHmZ}LU7%hviqBnSU{nBccYN>|0PKLz3<;AZ)zMTOT>QF*f~ zim%+Sru39dGt^eaHa0GD4!%f@pN^k?Re#)PWo*UyXTO3I@kVrVB|amU{eWcD6pLC% zwkQ7R_x@3k3CAA>{&-uhcRjHA3z-xwW*{kgm!Rvs9GP3zV;?W+R6?Q??i%0luCiPS)Lu8W$4MX6RG|JI zYxxk-nn$Eyzx7=*Mxkym%4%c@O^KQMdg`cN#|SmvfnyCp)?wtS%;ake3P*f5hx{wl z&k*9QtcK5+3^F@a?n_?qXnb5tax`Smi+eb|FOQZu^|eEPQN|`kD_DT-0QX-DYu zOiC<`Y{|lF-jRH-g#W0tkVvUy6xnDczy!v47+4z?zKersOpGJKQLzkpYlE_$CT5BbfZaUAEi%UplxAG;4sj?3XjzNWS9ew_N_ds(c=y{Qj*V=gmijk z3WIm=GInnTUNpQG51esQ>KsdH4%?BpHs@iYbCu5FD?Yyu!T)|+(_5k6!%aYKXhFZ2 zNpT-ZN6i>tCQJ^Nl=c$t`VH~={e(M0)-RFwer+S|ma|BhTqQ$6@HCn)H*k~H2dwS) zU+?~Ux52}$$G@^gsF#{|Bf%d}oMfwgd?L9Z<|E8AXR3oO(8@D7zt26c9oy zvr}$YV~;k&Ld~&{5zqwZmSt}eV9MUJ&C-oL&TuTr*%qdRj_CP;oqgtZ45=gOzFvm|>Tti@y z@T?(CFG~jpmPeR2ZP%>uDM-AETIx3soXpN2tCGk4?_bi-Or?O88(KbWXEKqK{`XK^ zdW&&W#uyML6Znj_6%>S>prF4i0v2h9=*jgCG4s677kCcD(HuE^3WWdt<u z760E)zz`D(2){zS`{C7i4O;17AI4%k-0O;91SqV45}~eTZ~?GOYkyyA;Exv)O z{IS|!$H_dOJ(@<#^Rn)Y3{qdiShc=l<$06BuL|)^ zwIs$JE0l`e;C1x)=HeSj^ z;pb6{(#(PcL8n|C`^TM&mu&%bQd@Hm*6G;=@^~&?p=UP`N8KCEpZvuVfyLZ#w%NKp zl`EAatDV4^Cx^dKvi!WKGc=iXrRQ3?&;Q8*yKmL!f>H?VL1Vs zKkKBYXzYKA@j)2eeT9Wu0nkk9_HE9<;BYXrZj-EA|Cj zBna%oH}@c7h9YBPqU@3N5|FxA%*zQ_NJ&UI|JlL}6SzES5e`UoZDL@lJ01u&?EJ== z2mkqDrF}rZ5rs9seLRtAY&J@pv;#C!GtiBCqo^CD%nA4na51`G$fO*t<-OL534Hb@ z%Kac}gE0Tka8A(f-?J?>pyUfWDGraG-W0^T^rCKi0YrQNr%pDxZs>81Ok!13neBMx ztS(RoA{Edd0kfO;=RI5QWJ^B-t@TdMYh9Vmbgd7Av>Ww<3sT;4!ZS$%CqxK+UFUXQ zjCAY7w>gn18P2ub#MV9S?|6|szC3zHLMkU?B`CKgAL;f7a?o-jp#(Z8 z92Y%q0p;k5MkW!1ghP?Bey@%~I3SGlrH~3%JP|&l?$(Ft_^lJq0mJHFk~JY|O>13Y zG&W;BO`=8y84oALW;-m_*T837v<84WY%uLG*`LJC#>bZocYEdk=4|Ro(nUw_II-TB z%HpB|KN%g-4`iQy64 z0;;7m2Ju-(FH)Q$;Hnoi_KNE1719hkg3Tv94BxB6pcp82_DW7&yi!XzNu~;1TVp9d zbSNaQS3>30J6kIYA-9p%?0D|VDgDWj!ahqgQT-#CmQknR3rwfhOghTbx^1AN8kLy% zYZ(yknon%WP#f+#6Vu@lZ7D(7@~@55eV$rv>h~y#mCDpzJ&s>D^D%bmu z`Z$j$tFPl$ihl|Zp1UB6eo=)cgVd`Y4t12C3RgvE3pXYih1<{wzh!+vUF%czgF~A< z=fkX27b@PkS65KLh&g1!`AN^IIM$_^(^2PQDLqO2&D;;8YD*SB`-)o+N$0HL(>As5 zG6;QY>Md&?`-X6oMid-di z*E>1}3SW3cowrz+HBzx@8!2eNQAmFu4+`j(8i>};TB3zL6T+XWwNtnUQrTZQ7uu!B zJbsFX;k)|3Igewsw6ND6=FV$KqRp$@s%U$K)Tf5ct%u6ipsZtRUuesRyM#{2=a&9R zKp6Ec+>MO-li7vao$_RjikdSOv7&e_D{&Kb^(pe;rlFOY6Faa`fswURO|&gTjP5~G z{Z`1`8MGCnaWeUIgma{P%8Z+znrW zgm$f++Bv??&Bmf9h^z}+a@j*5vdQ(~+gaA?)jsF?m$8Q)+(BTy97caFcJZ3pw$g`Ydl zop9SN#cBfTjMyfr5jV*)K_w7a+t;D~x4Ub$AM^R9$$eSo!FuZ5-H~(q$b~o@cD(no2IC(d-8ZT z=k|20~B7>G5L(2@7pEzjrBOI9k}4Nmu*mM+E+Yrrbiub6t-MA&2Ed!^kLa zGjgZz`B2{;>P7o}3q$7q+ABb?&~C{aK-8sV?sohVv8qQ~6P<`(@oGuGO{{TZA;Sv$ zPcxYpJSL<4wpUL#5;)}zHcP%gNA&-RSF8W#NJYoVUO_vUUq-=W?mHgIwJC=7$0#8+ z!sNB6fV)wyST=c&UFJ8{hv%sjxi^u$XNgtXJ^gP|?G>Z)w31 z@&%pb<>xs+f-u)*^BbOhq=)M;uU^??N{y?kYuFywRy%Nk9$d_|)J&3Z^CWh3<+pxu zF1`kNr<_c8%+K#^E9k=nfa|d_+GYUTNg54ox2L&Pzzyu=C>*tjiMV0U zD{b=(3J#11(2xbkS8n6et z9<(3{T<@@fP?`0i5-Bc{?a2ol&zGgM1K=8m3|Jmie2q(NZGe`C8yFmngSO*k zXh~e*glrBv{+`k+$8p*G2!zK+fuvH9G#|J&3J&kQ?>U`74vINDP7meP4UHK7f=x3p zVIO#Tamg@J>O73uxx-_#jHNp{Kg#hOOek#ldujaFW8t7}G;;?i8!lWUn-+GH&{ zf4a4utd)&O*$B-nbo`M4J3$S?M3--|itJ1bL9Kul3L%?l|^y)abQ;i=;P0ur;#)Cf?sw(as$pazx<95Xj}txlu4?iAv{ z6E6Ybs~78%krb_ks{3TOR??qWT4^um&2Igr>@iHUgC&JyCew}@jf<8X{p{C=%R6@=iX?Y@YT>M`Euz321qiNcI~ zQd*E-JJHoo7A|?>;lP+m$yP6Z!M692xm~kBRnr_vXiZP- zRCF4uZp+Ls=g8HP8geR5_PZwL%eZG66M&MoS9>BQV6lj~tO1P%lmZxdlYk{4g(Tgo zhLQz=l_~<47c!v31>6&Qt8}oC4PcScleP#g>nq4`c2+m`3o=-tg`L!<&Ss7um8I*227uer$ru2hDZUc$n{AeTxWu>-cL7LOh( zt4sp>vZysg^2d)KfErZ0(CC`jzYg|Akn$4h5r5(0^1I12ds(sHXALLo2slkU=E4hd z&+5nz8y)EQ7HWpbfx2QovGwWT%ngAZ7cVC-i@@L_2`jEOd0X+1zF)tjFyn=Cw9l>KrFd5*&`uTW&tdj9n@!FFynb-J&9}&D?9Cj)PK%_{`R2 z9`^oXB@B>?ZUF1p^!R{{>Ku8hnBsd!mgqPUt%v=X;hDee=x-gMCbsZ)w8S4jQ4`0! zkt31V;{I!fiS{_Zg|>z1c2a{6*)`_HXJXuv{o#AOD+FK|WW~jwaaw8ttP)@aiGau3 zivz_SdPN9{29q^i~=ORFD{%=p&X@7krM$b zD!(J^_eLD3xVYs`Cm_#f1kjFjbO?iiWxP2s#*vl7o{8V)wlOb;(uILOy~$`cMv&h3 zM{tPz+gd_vsoh1Z9~Wj{)Cr16`&w6yV}i<~v|&l zTKft;3v2F;vP$9t)S;7ycuRw?0?nlwO>URq%m-r>sLcCl;$@DnkT6c?dWt6R5@3U1 zhQrfLcGJgb7fZsH8)u=Pq^9+)zM@M;Q~b#M>nVM?1X96C|4pi5o&8qpYikR?LEM96 z$AwzJ1Ffj22*GFJ@j!Jk@}l@Wivo9sS5a}YGq#g7m!ws>wzfeMbZgO*dYsQ<{@la2 z{Y(K(GTmNm_8Gc+51M09Xc!C+1ipv}LD ztFZEh(>W6FUPm5RgvO8!q<;HPl+bsML9`}yrh zP)lu^Ocv|vQcVjbg~9qxS-fC97z{5^xO~x;2p&Go3;q!pRf@t9K;&UF9ti)&<9<~k zUg>)`7z`(#H}WN-8P^6405?v<3ePn`<;c5@A>h!-bln4|0YA6u@kWMw)777R&_n(8 zy{*m2QAkiwP`+v*4%*F=(pgzA%^^e%Fp25+1HElbxRYOmJ??MM1@3!L6R?jbo8N5b$PDCva?DyTR$ZO9QCl0P}hgfR<5+UUn*pJyLL23eQ|-zom#xAKYmFwdQg? zQS>KQ`=v<;5O~kd&i*nW6Q24rpBqAaCMYB+*$c9aOs_79z^svndw%%A&K*_Sc3-xLg8%5*uOXt?lK>)+llIaWs4%=%RGonWCU9Dlz(nhi!{iI{T#) zs}8Z5Qjg>S7;ujJ6qV;9H1oPU zN`c#xNaR}oDLbqK-~3RHQn$+^N&~dQ^JQTCZjQCC^)3gz0Zp>CM~V{`3|kPD^ehaV9#~!23#pb^%Itk2SQo)~Hss0teKm_UkDi)AYg%9`SFujkX8_!Q7)i zW|FIRK?D9r^F5pIL~FscL+wX#_8`6Zje7XYADK*>X4Lx(EOO2*idK+VZry?2kc5Dp zIVpLsA9*@Hy&5^zvRAr{27|jxeU)-uf`0Z=rHMc1`NLN-dqGu{R1uT}&_KCmv*tqg zR!Fr@I{1k4C^l1xPTEqFX9>jNv)r?^dFA)~MH+@CJ2AiT_m@@Wo?vq^ja z32R~CgwP8-e|Ik~GBgRv+BtG9lf@!7mZ2~2BU3%2$5=OVIlGf066jGi1=WzXiEdJV z98u~c^|dX7s9D{r$hcZD{c$&?!Ss0k>HxQPI};r!&dPwH<^e`WSXfNh&nu*xSt#fs zW=>o5RcN9}d)B$G2C|Ly%Hnn>`Y>Ou7;mkRe6Uw1=1xi3 z*>rSGPM=jnJvJOWlb^8ZjUSBIe7e{SS!KNEFt{3BU{BUruKdMLh;NQmzR^BlSxI= z`WJ+F*w=ycpM5~S4aeH+zYh$AMdULM_>V)Za2wvKjaW?hvv@S!?9F`mRz*mSxcyo< z1Bm)MDlDMJR6Tl{e%~uKduX&*6DyxBK9zwVGG)j6tR3oo`_3qe6ihX_^LqB;N0I+_ ztAJL$oOZjnYBq(d5M?%L?@d3?@z?>o-D z?;mFzheL4hZ>%-fTyxF%%sHJP@QBap7#SzXRP4+n3$KnJs(#g$8HlIQ++$dHzi4nz zNZ?^?IUtu_{ToGSwHUuXDSfu44Uw`YDRd-LE}G9@vzCR!!V>xUjWLJlD^!i_Zo5)I zb3ajXPSKa1slPr$1M<9(;B8WI4yN+baQOkUN6Z_yFeZ!Kijd1}#xAzW z1esUw%Rh}_1kY7kM(2qA`Y)(_a+%n6Sa*yZTjqG6_vWgVb(e1YW(~E__#(Cc+0(8J z<5k6&Qyzxw{RQ6y=uNYEolC1z0F1a@aCjB4Zczj{XqPmYU0702O zjzAWtFc|L^(uy@prg#TZqeYw!7tJ*-G!nV$oxd>Le+Si;*|_m+b-8%NP8aAYaxG=v zT86JUM}_TXgDlRcZvbri8Lbe%i{UxncJe}43zwthw96DcSG995P+{Rs-!HfoKlO`W zyNt;9@y_ae0D_b1YHg7FTbi#7SKAhqp&e}Cyrhz}^S;s)q+>~;eSx(x7fWPBVskGv zDt{cTcMxruVb7WbGL~Z|)(*EHSf4P68_36^*fd{) zr%guadAa}W%#46!?DDwi=+{K%C@J>r;uvobuCluJLq=3&ZQGRY8KV+&Q0y&@aJEgr z2I~i}K4BfGQj8BmRhX!#aSsgNXD*JGP1fa5s;qJWD-ue;d0}|aUhxEYF!T}cj23o@SdbO|jmHwa|9m3H3K#{jNtKxNL>h z+YgsiLJb`e#P!S?>dDov#;MaG2WIG>2~5q1@%z|*Ia9;252)e3pCIhjXp*y+8%opP zL^+t&8##>nlTW=|5Jjy1QtdK96>;#DN9~n^avxJd!P5X~p}kiaIahfRFdskzYXn+LisvcD#UujeY`zG#MOlP6oi) z|K7eCt9t8n9@!#DiW=-)&Z8fvtNucjXPl4b-x?TjtO)TlWL=^m9=1qa?3eZ?M1G$3 zxHiZ}^c2!OyN{H~M8T=^`G92gai8ty$70)6si{b~)C(9#UV=|%#O$p#x_f)`2x-`@ z%YY+u8oGD!m4wopVbT3$G!wA zAYR7Phhn?^gmbzhA@!l;7d;ij((C4QjgUcnPQHcLgN;lygg|F%F?u+L+K^=#kN%)| zjycI^dH!ZaE2$x-Qc3WzcCb)LL z2azxuT6ZXw?*ASB|At; zWa$y2`^Z2wX548~fCkW@eRJCLG0^nhOFmVf0vq~eSYT|;wo<<0gJUF}!^Yz_!=;E~ z4E4g)t#^7ZlF{D*w0or)j7U1R*K22Q&;z>Fr$%?U)ISbA-*~4(~RA=>gP^48crqL*)AxcjVQDaA=4? zZ~8qCu6Wr*39t(56L58<#^Sm?aXzP|eKvybSZ|zq}m?q&l!) zDnTUbbACc}t!OaY`hhYYBUJPJY6bZ60t~7Y=7`erJ4)d{fBBpajcs+pVc}5hl1&ud z=7u2o79j%F*WMbRo}NBarev?>7%5I@4q9f>wZhXVbM5|e>N0d9X&lspbqCE0m<-Xr zh^_pfa0=a>s_ZL;&+?$=#d5Vo%}@Fc9fVH8<_`$2#wPX!CU|Y?=YxV2L~-_C`1<;u zemCd&)}#C_0uZlqL}folwG9RDY+(@%udc~+v1JhvuMH{{{AD00lT{RdkE-1ULI0kh z6VlRD$&$mk>Y3=UO*wl7DM9(^^yr6YG0AcGeSwuoF^`IPzaVG5Y)^9YmZvD0j?Wyr z{>J;!BJ0%zTlI_>oaccnFca`$xO~@4sF+XYXDkv${iF4v@k}PNwNJ6k{v)n9W73!! z?3KD`u4bAF{&VD-G}Z8W;J$LeCT)_g8fRSfZ(kV3yX}=G3)KPYChPL*;`L8 zWs85WY%}Qar@~RDtr+P;gai@x;WuLKrk$JW6IURhJ33 zq~nnK87{0Kr5ibN_vBL<{#A4)4K3}NwI(?|Q@ zWtVe9aIZ!$aBrDwbTJ_OWq@THArK65qEz~0cQx+20M7?aaufBRmx{)#cCbC3q5_^( zKc(g=E}3jz<6iuO8aj98Ofi4dUL|Pc0msX#=NqD$aLTGGivYz8RsHUC2>oiK;V(o{ zLXA_0;q9}~*kRpXi6a&8=w3xKIeZpUKRTtoxj>9pzXLk6&7 z=%v#TViaC|bsQiA?V3AN?@(|;=v%K`9-Aw(#%zCeP?f;7BjQDv_cQ!yNls-S?^2kH z@&X~ahGM~6)Z5PS`$72^Dh=WU{84wLGyQ|cWxpGqy-ha`#H0K-f_J_)|5}LI0{AKj zxp?DeZH_bjj}eS&=Q>&vBocRO;1hgyhc+I71LHeL=V@-5{1lG2sU((M!`P&eD(0V% zkZ`Mle8y*ADb<78zY zfnp5&=X#3;Z=X`?239dt0L6qCnpDlP#i3qcJ$lpseSmY@`4p3lk2({YKH`3Wa)K=>`z$R9E%D9SI9Xt z+u&Xzz;1M%7wA)6%r4rTnN3pa9y%!h23#|*`zx4o&d_y_oZ3Di()$2s^#vulAY`}xcJat7RuH=>% zI)NEmMQD~y9yOnVKOllw!v;%jxEg>B`RRLCAMzL(lNbVlFu?&9pq8$7myNf(d{7Et z?fa%95J>X`Kt_p2#Kd&_fXI(MxbO&(oRgDo7|zth1lBLIVufrGre%7$ulAE%bCa^_ zUu%)BEDxqD%3n*7LLg=+zc0sIHeT5=YvdY8MrWMpyWx9uWy&~!8Q%vF@m7PqBF(9J zTjBde8GKeIH*hj@LYiH`$0@!uB|n~3gK?n2{`byS0gwq|hD^>0Mxp=@K5J{n(g+Nw zV`2=0gNTTOIXnmm3tJPBkpYS`Vi6Iz%nt1EuL0pJ%y~oU&yW!jXNylmZ9!%F&+IzA zfd>=K2_viJ-}07@sKB)l3#d?pd-f$?u~UBUJVlQl!XcJdEwtafyoMeP_I;38Q3Y_W zEpufS8Q*1nk~0j~r-vYm_}utxuJXAGLG}&oqW^3Dc+U?3hat96Aaq{V4W0f_!JXTG(y@Qrwy~_d^obOI;}A5EPAP zf8A;R^dmWQFEY!L2tGU<`t1|Ey=a~9;gOLEn0`$FJ(%{CaOFisMaQQ$Ksr%E%T@pZ zewzQX!p=uIxaY1=l_G7;`+MR!q8&vny!K1&AfB3Izk#CcI3k*2#l3bjU2V(3f+{F+ zNVGJO4flJ5KJp(hDc<##E{Kd*S$$VBVtLmaWZWToz6nwl@joqLR}v{H!B0NXo>K1m z_R&q!rckd*ciWWuBjcX=1xS@(DI*-tSLZ0zkX^PWs%F+}bPu#t8T)Oe*pGnzeUdXle2%L%Lriqql4oKG?s?y}xcid5MigYE+T2mFHV~z*n5d}CQ+ikF zT_J=~KrOu;wS%AQpJQR?#vv6Nj7X^BLeIoP$fQuHHCD#$@`rQu$-2rlWXu9=4`W}A zLGfR)=}1N9%n3jL@dgZ_GzEL5JE@hX68y%*Vy)j$*?$g4Roh9Ad%pGV?}tq;GsAK0~bIAzEIWr3Yya zW@19YiJib**bB@T7Im3+?Z2I{+FIw3CMu4IU&Aco*Pn%MCju0$!Gxi_?xp8{UW-(6vD$JIHOw>9;Q|1%)qQKDQeIw+gtJ%WWYawX9sb<|d+r>>E05rv zs+i~>2p1Pcv^$jrh9iy{j{OnTkJ%NKx-l5!5$N}>2 z(cp{Hm(ooW1Z1RX1i#qIGYgS#bHOEETwKhX7sq9kPq_E#9x4&dXM(f{|8bgx%{8{v zn{lcV`XGYlicb};K)|Xr(#qUtm8c{UnpEy|gnJJ9S9B#^raM{a#d9Q@SlQI#j%nw# zw`4^ms99rKLxepD`Qr@n6 zmwul4Jx1>(>J6!LIrRL*=G846KV9Q$E}x7bfs+WpNBt6=Re3(j#FSZ`ZJEcZzXQa-Jdebq=-TAEfMKw z2K#--=IOQm*dO}Xwvj0*_i*#I6H+*t!wu%Yi(9H2yzo0NtIyL?u|C_+a3>+KW{fu8 zw&u3JyMs@ZoyU}Yps32uYJVRQY=Jz@`&A2t(!zNkEb!uAqEOtKvO?ZHa)59l+fvgi zC3>I4jV6{c1F0^<#moCW2c_6Q^^w|rJ9zbKRNurBz) zY`jUI|E%&K&ig;`!`tJ1k>%DSy!<;5GwJp(A0&WclpwxR%*9nI1M%+E7le3QtJp{@ z-(GN+oD$Nks&cLM`@iU&2$0FAE1w_VUi-Ou5e}j)`Ty-!tEs-->K)Gd9r3|xj|@{E z=fte93mJi`EH#az&V%xK5A==8$!uBKB2X74DHujm_Gnvv4RIAb)XjJo+>OpruUzc( zUCDq10*8=zNplAR`I>q#!7W$U0`b|0Ky*LId*Z-Dm=tU2GcmL@7uKduSW$8aohj$5 z-u|e1GN+I&z+Kv#Hlwh5E)-0v>BiyUFl@-&7SZc$LvgBlSZ6UKw&>szg`qiA@iczg z6{qbQ%LeNXsofh0L|yUBGV{!YQv^mJmlp8Jbk0mf#W!PXvWoJ>9q7}ad!Lf5lZjQr z%Qqe&{(7%UqQanN?q6A_#Q&(Zv`u-A3<>fNy)w@nem{rkpXWSwFyY?ah%np|Y_y(7 zoKVWp%MeKYD;NgDZ&DUIHQKr9UIdgVLw1{MlRZl+4S~mA_%=o@2J8iL7B9Ae zE&r17=<(kStFfnf|{ZaN!9jU>es6uuB&&{jc~|ZC0NXMeV9sR1oo_F zAM!+%HF{Vcek{g4h~|EzQsl_0bD8%n6~f8=&#cQLNYSy0u^q&6f|MLk7wA7Sk_gs6R8P-GzL^{c{f&5(0!6 z0o+s)I?VX#6PfaL#xwKfWwTGiV$r#WQa>)b^3V;UAP_d6HUvmb(6DCYD^7?vTr+Hi zm^^T&@%&H)_15+gU$EyUXqP^fFX|#gV`1@Lr4r7&rCAaKVn=X91|i1Mo`6guO&~## z5Qwx9 z6oG|9$Tt)eeTa9HH|%~mICSry-h&7twD-K>AlL9oa_^%#lcu@~B+`JA4 zcq;fojZ#ABL;YsR$FIr4O(6Fn0RzI8GV=VLJKC)b<0OWN(5^qmqfYh%f}-j{#vmxw z-3M||&DfB!DBVHcTU%=@)8#jDL^o1$^5=`Nxc<%*7dQ9kRsolv8G_o_5RSBCR~xKb zw$$eY!~KR*IOvFcD=^W29A1Mq?az}0`Gkd&SpcNJsq~|dyZX71PS-Pt$ z@z`>F8{ks1+TXpqqs5{Gw@1wO)zQ=&#<{3S2!K?f-7pKl!eLM;l~=D{ zpMZ=(nP_%4LXf$0NC!ih3b?Sd(a(9^cmxF@LTV5brD&F4F8p2z?>l~Q3p(o(xPNB5 zubeyqAcQ}%z+Qy!h$6eBo_|Hl4@0$l{&7;_uc?x=)8Rw1q}g&(e6U<7xo{TE)b(5 zc~94PB)Q#g!_FpmT?GZ~sGZ5u^nmG9`9yx<%RPjZkf5N=nc8wGHW^@y0Vqr7*M42f z9w@Ui6Qe5Y5~W=qTXlO^*V^(jK-Ngo)nclb!O&#LU)pyLXT<*@5C<}XP1qf^0u52J z6yHsfCvPjd;f|0iKcSqF-%d~XyxwN^OW+$tkUaqU|1>GS2gDNwcsZI%UVy^&85F)r zkih|TsskVvaJ~aqK}P0V>}Jr-EHor!8I+V5=NolkVi&-~#v1_!=97_6 zewve+{;U{?O9~+a$!%pwoXlglAfh#z{elXI<6Qz(=7)Fh0%|&XfIUdZ!jdQIheXpO z29g>9!jrMr>dGlnR%ka$U6=3e5z`7IXtD>4UOGaJl8qBmZmSy4tGDc83B2|O0K{aV zNnO9&4%)drs}qMq(N7e&rW0#Kyp?>!=u(1I=+Bdd63cQGP}R3SK>|;Xj+{QJ`pEp8 zdS@Ff5Qzsx8Re&0cDxn;@Aq9#=GMD~I37QSwt`%>vz2H)CVt=;*3Db$<7mpr1VxwX z0*IC}1|u=8I)VrQr!u{)+6JWoWQPFw?2I1Cvbw7%c=Udg#jpu8fjjvC6|X+#6?L)wfLNIiqof|5+tD?^8jRht8WSsNsWk zi|@(WAF>MWyUL%lL?uUN0x@9wWXlo@MZd{=O6uQYKw;@bzc~w9*~w3~`-zT$!DO-3 z|05S0f%JxuFyV%KH6=p+X8B;BIaNAYWw9prFvH)mnGI%Bmxj_7n z^~Ie>)0YO^ZaNU>ytg3Q_n9!|;27$o=dZf0NW57~FUEKdgIr-19=bD9bkhw46PeUl{YmGtm(uMI1dlnXca z>zSa~F`2#$DAA$L+cC%Cm0`u;HQwmq_I0G>mAHHbqdm3v6}$y4&H>w>-7gaO!4EE; zqebb9)z-_!6J1~WF%BY}+^Xhqk!pplfbhXvMTWS4?Fo1ZjV_9-s0NnM0fSAB8KP(2gJ?O|9CWGEm#+@gi(=^;(S1enO~9P+g}|| zMC#G)O1Xj^2An8;P-fcCIxadO7zogF99jUtkqg97kP1ZmZxI7dVG-IHp}{A&TlBkOqgB0C_&0W3V33!e>L4G;w*s{fm`3p*Ff&1%;NU;Ht;)YdHv9p1P&e0 zMNjsuysQjJAqFKlJ4}AZzebM?j!{9MujGH0p6uqpRRRD&+uGVVv};77$OT^a?_=oG zzYyHNb5;wU5L6=$D^&n8#D`dvl$4%^5W|ujOCgYKX69t%56D1`Ku59y=lEX2AC*s^ ztm)8I=K5m(2B4xpsE~pFcvGFBWWk6wiXa*SYG{9rXL_KxE4O zt=m^U2pLo!E@s(y(*}Wf;7Ut~s^~QV-_?*7C}lQM=VJRQZvnuS&j2`>8KD%07*)*+ zK!m)(;;;LQX16Q;o&FzzE0xTc3)+PNSf9DQwK|W@46ji)_JJ?Hkn4dFfMZEAKq`E` zKK}QtAdt|MM{@7nf1gj2LN9v6v7S7(7|j7L&p=)Xsj`v~)*q?*N9+06yzdG*CI0`b z;{5OGOna8eJfPtuXz;3?;Hk$&jLT)QEH(K&t4yQqhGys#mZKO}9tfOZDCL_|jd-g2OmecHE7UUTk%hbIA- ziZsJ?JsF^(aX$3iIkX!gdV}-df9(B)MXi9>VY#~j`Fi$nEf__ zr0K*}Yv!m>vwRht6Bk#MBl&tpTqFiI+@pfl#^Wwck_}*6f|_X9vEDeaS=_l#jLu{} zS+eMhh*ot;C}88Nmfm8|CH7qIeS5fK%6@pwezLLk8P_VmE zMLz{>8vuyat!8CDYU^`uHL$?oKk27t+_gk`~vdndMTJ={5tOXpw67DC)7W8HXb3ppG$n8xZcb4$J`On9In@dZ*ZdE zy^w&Cj9qePTi<(qw(b|tMe*E4Q0~uW%a2LBenf|QA5oFwcHKHER+`6PI*N=>GQvMg zB>k(xZm|`P%9zR7w#`oYM+>5EVPL}zVZLxTUGONWK4lf})~w4s!QU@g0KnP#%Qo9E zxi1n3kdtkURymFalAGhAJ^(nl1hIQ>EL^oqmA&NFR4O`$nCr09<)wOCB{Qk!hc${4 z{g0F*xoy6)Z#w$|UA$PMH0|y2CJp8hOl!ZT_cV{kZqv=RT%F+-W{|P%v!$vb{lhPM zBYK{Xt`~7uvl%*s9`bjd-{9dHCMV$vSPV&SF!bzjg#^r&32cITg}AB$aPCP$zn9&I zkN6(IrI#&V^jlmN&rW4g!%5E&(i(J6@B}f{>|0+o&50;~U@i@=%HR&FIf-Y^TlNyp z6h0nRPTO2+))yK4XH9BnpEO*bZ~gdeI!1broYIx!z%kjj?s9aQ*TE#g%vug9emXRm z;gX5N6w?2l@14u~+ z97cx%ZvM#xCRzJ>k9uC708mF z+FD-R$u_4Y0pURj+Yn-OKAxBl$SZ^~0ANK>E9BX$*E7Iv@Kg8w_la_zwUUQqQ4vuJ zDyr0B7@&o;xZelRQB;w*=3jj7#^YgIyYiW>PM(K^l+ml94gNGZc^0R6`5qNHPLp$2 zAb4yL0lt4jv{G>dnvrQIy9~2tk5B0%<#XXy6X@oB)x1$)bEt4Jjd9sg-tKQ)Z$G%S zS8eb0t`pRrWCYAtqb%Ivxx=X$iha0A!Mq&Mfr>A6!@h*-WxZ3A8Lf^%y)CJ_@y;35 z?k>1JBt#FW`d_j^-dF$mrcCHJnn3}Aeux0Qx8Vmuf@Y&-fq7k*j=6jeD32LQ<{)cK zMP=($EZ^>-h6<~)cO*<6!GDI3qhq59YmROuR-F3lmL{7<Lu=fopU1`_$%Mwt3~;ghlfqNJkGTE9!Pu}W<~_=_MuKH?Nk_3l@K z@H>3}h`yiBFF+^PgKDo;)mOuZPzocRv~BdP89jB$IQf>7yN3f9{zz@I<(8izTpoi1OMk zLN{!IhKvO ze@*=h7_mgE2)6=>_Q!OmoU>FPF5ciDF$R0tu+rlN;K8Z zqcE}2U0?bxHD5(RH0`|VkNhE)iiG0u(vpahQZYQIUtAK1S?9#Wqlrajb`M9M~%MW#2A-PaLm?+^j-YRkRXB6 z7Y;x%lyq(${3BEZiS?1tiTB!CW=N?SGV5&dUwHz!@tV1P@iT_w7>Xt%ON(C<9N+GV znxynsipfofEd%)fGnG5&aQ_Io4?G^Rpr0xGwNQ1?(mvyXX~Sx}FgJBo*EZUd`2ebh zb%VP7(olk?i^AdEOH<*sM$pAB4OJiImK1162<;d=Gkbv8VLG4pMVi*L#0}#x15_GTN=8bP~+uV3!cl2{fEEKG7%+kl7Tjsho|314NXTrPL~>*u6u#fKyKi3mLNDo zm>cHy$S;gTLe0p9{ z`tn~=1F``5Gst?8csO0R<@Av43LVoXX6^_2a{u*(Kv*$G84XA62-W#uVn*71&ycBy z;xJYgIWJ*lH%bkBiAji!iBB%yIK{>7)Z`<9`lRRO9ULBJm{i5CyYJ^@G!1GC_AcV` z40YN7fy?kbQ=(GXTOyZE z3RSMlllT?1)$;%Hq7fTV&Ph4>CjJc*``JOS-a`SoutG{)9T)xQ<*6oaPh~CNJr(Am zWp{Y?RLtUBI1sr>xK>Qnw6tns>-%G}{C|C(^HEBqjK zw@}vdQKvQ+@0VBaXooBZN#AuTe1FyzH?NocdQ~iod1ao$bUbeyaise{rj~kekJB&K z%%u7sb~+qK-z#3@*6FXAG6~7GD5>4opH8dp=#F{zw4^Ut zamr;yVn`Rw5@bZrfPrUIzyB16`S+H%v!L&-;EEm7^89f*j4^L@!`g`_x#wMDA66N% zx}Lhb=gP-uHI~6tUu+xxyxf|HdnpIi|{f5GFU&3GOR$9bN0KWBD z{hH{u)P5JYnWhNxTWkFN7DnsYp|G2WwlBUuKi!cS-}4XDF^TEC-&pf&mo@q!`=$9* z)z5WzRZk7+|Cnnp`rglSGR(xDTpp$`xlwk;hH&mj3AAvR7d-`22-~Rod8tDS>v%)z zo_tr87CX8Rru_TVBY4PP(s^+|?-X8AZOmg~*|5E?h(OnfC(C3?obNvNTcX$H##7DG zB6{X}5!QK?HG`xoAAixuFf|zj999XyPcd2?Fb(qzW0$yB*F)nZFVoU;UWt z&j^9y?Kjy9m8z&>^+z+reP{c<=h(bww~|z@47IgJ-x)JkB56Asjz9faXkq6BK;D&N z>6ZBQ*nOi+trCk`2+yH!=2_Ra-p9saDMu@5g(@8ON9ATgy;LnHx=$oL`j$pk-BP7r z^hTRA4}>c9qJP9wHECe?L?%Yl`dkBTF_A|G^Ft+dHa0OqLPj3p4I}|RJbISYeUd9s zEBr~=mYR8kNrZ)6?<6xKG#YIwY?P3BYY-#g({2m8!auFL@VB7Q|5zJ6k_fm-;3ktSAb~MgA zGni`%{Gh_>Z*G(pI}77bpasEC9+K!Q9|)JzC|kbEnIF@o zd~a`L5oPTi9;tnhNZ!?U_VUX+$@MQ=Qjn1q?MxX092dlDH`^#=YFg(nz+$?%h>Ls4 zXaDe8dK45Xcnpy$GlpJ+Of6^RsVnJGk79}9sl-EZlg28y9z8xr%k14hH?mHb{56$0b?)l8DnQG@e zX_nvvwiLylD75nq^KI2qh}CIFlIoG%)h58ntH z>>8j2EzMn$L2#qbF@(q@+Y#1ru~WqWbbP#6kAp^DJlgw~6|DI8&B1{Lj84+;XDBpi z3m#sk`^T1%dYGpCZ>RVr?WH$z=gD8@XbJRh{VzhqXz&foVlh@@t@e}UsUr@z zf}UiZx3)S5b$X10XDnxl#gX$b;{`o8I-cSn%9;;dY$?rYCkhHY+?cKGfm&Y<%ZAdT zCM;#FMCYKer5*W7T1GP7Z;;ryj0ibbRvWuqkFqeN6}}QW)R@t&#o*kHGc$ypN1fu9 za80Izl%LX>99R2sxCjy9b*p*jZNo)W8@qe)*70$v5kc}kpWT9Y0PlzChU#9wgqppJ zJRSACU)B2x#AFxbP{o5g|o4jn`~6CZ5(b zo9xE>y4qrgvpD|WM7%tVA{TgS|*!se>hJ+!JqRxM9o$++6@^aXn*vsK?&Wy7=H zCIl+xI>zU&ag-<4{@>=+?2@f8?$o#Uc+!HF*PyBe&C-L>J2Wa#031~s2?)SbzzYZu zW}5Bd=^|tRO%>t@C+^LB=g}F%cyt_x_rh)HF!T*-{^zOkbHDy4b5c@0GFooSv|x|D z5}v%l_B&j)pnuq;KK?T>(BxLb1ZD)9KG62QKPnRbvTzUut!|Rfn-c!={Tuv;F8I^# z+U=S8e3^?YTWW8t8jqtVO;eVpFoxnC7F$=h5l4UU5~- zJR;RdaNE?FUmXiuU~%%(N{C7I4olv(z3u4d(U5$KY(n>zFkxdAukhTTB6vMo+^9Un zbU(NGbyW<{3*z#pRL^pZLA4O>^P}9|7dHnkOHJi2gZWh%l`5?jpo#a3-H+Els@JcM zi7L~XRdYqHdh;ToY+YilwzMg)3*YR_fR&*KkV0H*!t#D%Xx-5{p zN*wsT&COuptN0PFTic_!b|%swS>3{KAWTh^Iys$6DP&LMt5`A5ANa$ModBfP!b=`{8Pq%A5gT zeD?9aY1TfDi*QM%#6u?aUN~cURc1N4f9Z5ukUE%7_pEAt+3PjMU?IiZxpRLF(A$mu z6smah3pXhi_u1k{kaHydfK(5xgrMp z@&rgysK3rXF&FjJ8Z}M4fS;UlKcUb!A3ju!nT>~*J=8J(*z}XNgHtss+@`8p$r7BQ zh@qJg5fS8rjJMVgq3L-j$XEJu^m)}lK({L~<4`9D@Y4}RQ{wbx-%vta>gL8g(0AyJ z1zrTp4)%J@(0E&7VN)g(5uhxvib3)To#RNOQ<>Q(x=`?Acv_mXHvzjD#~q)(|d20PBlI$(W01zd#F<_gq-goH~08HkpR#XA#lNu zt~xd?{q(SF8y2^Wxq zrWTx5zNO`f>&z~H^(DOFrgJQbD}U3Md+f8W8_jl{$EfdcK8Jh8AW!lAs~;G(s-Ss6 zU@sH%^UVAx#n={I(OB25o|}CS)2}Iyky$Ix61}#f@*MK2)CBMbZbc+;loK`sZ&BX0 zqOn(OHDi{JHP4)it$xVnBX11a=fIa3qUB`Yj2xJlrK3^K>e1a5sAH;|qj!p5Q3+u< zw;@)fEt}mmjOXRQ&YlpEJXtB>KrB&eWEm+0Dy*VpldVxPlBM$4h6bBJQ4p{$DBkeX z@bYT32g3l?C>lApdMLBB<+6I6Gz12_AZOTac%(h55#aZ6LX>f1csF$RXG7*Lugk`% zi`R+@hS2=zff)8f!shIL3ud<$6Z>yLyS9hbB2DI*geXmfYVQV2BD zC0|tO4JEdzt`g+UyRtEzFJuo29hrCKp=l|_90(tDl-o|d6-MEci`Rm-F4IW249iGc zTt~k729Sb3UmhQQK5py5RO^*|W187sS*v6$KQDOm{O^Oi=Rt5dYtH#I%a@L)erV=v zJU{zr^fXNMt^5LFkQOD`XtL%~Vm7Hw#?s=72dW60uXsMDnN@a#Rh$+a`uE*UhsxPhDxe3fi!X-`s98;Ih=aLbO93fn?P-b&bjAg0 zn6@7LlmLLy#kdHK!8hOsJ-3<^BEg9eIa?lUFDMe)u^DXOC^H`G>Bs*fqKsyVhFd^) ziWaA=HS@d46f=$kPUgkwesUC_>vz9Zot!y6!Jg+QlkX-?v)Z3<@qG?$-RbH&VEmY% zIeIkEs@RfUX!(_-HB`!q|ClyYC2>B2$lr>*ZNsnX`TZE0*B&Y<2?mwfh6vyOG5|{4 zW+U^wDoo?$-7Vz#=&W>*-EE ztAyj^`QvFx!IL^GQBi(%c-R3`PgqKrGE!=?RYj+imYGn^6%`E?rRsvlh&uEEe7|fS z1q(~B#x6DB4La8c6as(JxHwgbgyx;Omn)*(AN3~5qU?@a7T3slFNu||L=ib!t?(7~ z2R;y7O6B!nJEo&JCUVL=ki9B!XD=9!$Ej#?J#)RgLBwXdG*&A7HBAwYZL?Ywomh+~ zEGiwI%_bN*?(egn)HmRk5jdun=Jj?2PN>s&C682KpZ8&f{9sZey zv1eaiGX8y*?%JF&G)uj-vb-sMo0#9~*wZv1779jG?jR$hy;6sQG%)Hr!4a{snw*@~ zV)xm5fmc$}S(18;PE8L#dM@YADB4ICVk;}nU>P@JZti;BD!hHHtmVX39}{c@8vJWoIlXyY!d+?ZCO z!pH&D@Y^Perlsr(5)Exb>zDh3*j`K;w;w6&+KjBQEARlqJ%G5={&-o1Xj-`ZMp z98kr|RFsEP#gVDEgTxSAz|CJ0u<}(PZ(G>sy8{64V9J-7aU*0ALUoA z77C?Nc0PyXw@sGY@IAiu+5ke+^uw$6@auCwm~r=vB20Lp$>Fkh-46fnvRr zXG)l0e^Z#%e1<|LZY1lO-V!#mH)xt326mjgmSaMiIqcjx6>1%fBU;di*fYGP97s3a zb7bu|Jk+`EXWVXn`NYxrO+UFv3t$D_ulTCPNty#?ORv67{=QLrEvM`ahHNXXBeZfk zD(jmc--|Es>k;3+$_Fx^TVt}Sg>h%J*t_adKkqDpag!aNBRK1_P}u9FFj2Qrozfe| z!l0%PqZp<5S}gzt0T-_oFwkQa6uUWsa{#K%;;QA_;C) zoX*sHBSjI3a>?Bc)-*Eecy>|7jM0@tl}TLkT&gyfG>SWZeto~`o;Z_@}D#s z^`&XF#{|)@KT6zZqnFO=HQdeBXP5mxTtug+NwOj>pGS9H#>~UtOXSw5Jq^KCESPRS3V{B>8PN%~tnT-sVo zGhjd(Bjj6m!G7q#q)^pykSVC+rR=Q8dOm;dHK-KCg4NFoBRD6*t;ZG43aD>}Dp$0! z+NgUKJGHW1n!ENYsQSi~sNq_X!q)WET?fbR!_vl@@}^RZeKOmHpEg6}o;-uIjRh%^ z(xP8mLcmGN+x zB6!x?_cfku-VYGa%gtflqDQyL2Et~oen{Kp6wJwNavS)9U?RbX z0`X}AJcGc2#cPKGKEFj$0Lm(ZiB~tH!!QSTn_EN{6oAA8-i~)4K8(Pv?JnjwDj*iL zfUjYpbxITOS1kX2HyEGUFR}#@eV|`#O%$({2rfJ_>7Oh&4O>f2zJBx>A7a|3rBm)o;)q zmHeC5Q)@6+`4R5zk~{w%|B)c^x1T)CEDd*V@BY`S7=lHyPK(t+z8!1_@NEK0al=eB zuz{KW0D*Vgm3tQ({30jFRz!^^7>06z@{vM&0nPX zHt3KU{X6LJXP;gOQuRY_E_XNAmzCGpjoeG9uZBt=jQOU60RLk$i_cGo@>+(hyE^)( z?atiAI?pm9g>NpR@^E7~z*x3mW5F(2cjfdZ(T{Qan~@8KZr$)p!(Sj%!@r)-T!wpN}_2f4*qUO8IK&(Xk!;s4DjAgFv#`5#+q34QNuIkH?$r-;O(b6*(p>D@w5E@lQUQvj7CT?^im?qY+e@bVW~v5J%*j>|5_8<0f?82gCy`& z0wSGIIv0_dY(BUUv!S%Eu`Vk+6nGJqZz>CkmiAMabLBNbk+Ia69QWa zI%?|;8eAvP<;%3cH6ytAs#b`YmH2Od1bg2S=`^|B53SlrZP)f?V+0pE&l4`@lX9vN zfR@pht;%}JxE!JAtqgoL~5ptj%zJE^ma*XCCyEItzY-B{9#8{sNTT`i_G7Sz(0fWNCJZi>)g?- z+zL8AvIT`ue-`VVf?^-Q4`p?g9$-BaNtv7=`xI`BrkDJzZ)MFyyX5)_XYPtzlPOim z_tYuKtX9fh{oBn>_yh~dT@YdBSK_X!fvUO82CKCEARWo`VV4mqMv39Ee1Wh;)hJbx zLaAEaU)z0D>BUBC){_+ZGA?uEqYD-*51ZCrt`PjiX^<0lq|>c7Duzr(XIHi&(VDmC z(MOxe^BD;bjpuKA&+@L)^FL^p3+RS`3P$Hy4IB6hS?U<xrr;PrAzFK$sSjKP=!uot0_Kg7Br?bUHqASsep)roknqY5~gGj8hstnAQ|FdntVbQ>gFtbuw$r8&F zN>`+G9@Lsa^oon;i0t3k_45!@=k&o zf`N*>;uUSCI1p}j&8_sKw9c~ONx@XavphD>jNT7ylT6yy(pw+%zYJbk$2mk=#X+HY zIrzyBuViepWFXV`yA@gFt?iq?W>hqkY0Dr6+HFPs6si-gbnQju0>ZY6NhfPtc2W81 zS8ci_N!ZzPdk#WUhh^!WslrDMXq~fAxi#`QeUw2O8NSO}6;y+W{BW{okd~P~bHIsY z^3<|GCBx`&BC)5tH%d#LMPti1q3E%jBzZyWerHaa7R;UfFD?dTVAdqMsCP^S;JT3} zfVSXoBsI0ykbJYYwH)+h5KG5Sv7O*ueX`wz>>81cJb^tLUoBrUrM8aYb@^Cy_5VZK zTSsNJt#P9-EmDG{NJxiB3W9_|gCHd+($dl(-J&2$NQtD!S)NK~~o_GD%C(lm71vj^VGuqjtFFFp69__#ji z-q;dhq8$ACM8!QW{vsGr6oMF;Ni)Y8cnG%)Pu46=O`wR5g^kdn{nW6m0Ka z`<=CH$N28(ID<73&i)2NjxvFxLVpe?3og&=tq;oQLeJCL^rtk=72(q!ujOaA6qs+P zrhJ(ha@aI%oL5;G_HZW)C-*e{B{#ghis0n(~;jl35RGBns|(( zVRiW;1gbmF&k{WJ4x8&|x1-XmIe+uDupaYp_ku^wp(L^9xkKGsX)$fyyLVm$>XnJO z9>iVMEaGbail_bMs8oA#8n0x}RU*D$Wg$WaFOo-LG$41tbL2?X#NPhi{OzVhhH%9o z_3XO0F_GH!uq3K)l04F*UUwa`t|dpHUxSIUu#n}-_#+(7X@=55)roYsC`t`TrEtGu@ifE68uX)ky1aTTFp#1) zcqHUqG})z5bOcr&g~0YW%2Gc+m+^*JqbtDby@R)L+I#wECzkSIT(dPoxusU{30sLY@y70b=%1^8%;K~%DP_WO>|U5 z1fhG|J1+^`J3nj&Cw{*&4f@v@v&iZcSgY*%(Qbim$(ca_@lE51IUh z3;jTSeMr15c`=#=ZAYF&d2F@odKg7mknt=bz>t44)bIS2ojn&5V{H2`mM|8|ukM)y zaMxLgHxJPdsdK7(5Cze>r%RHQfi=NAVd?f6$6xPwH8PZ=xT_|FbeN6PKc5^1SlU!R zXwTmN(KC9yp=PaX{rJJ5ze3WKl=^&azEvqE-b^WySsk9{p{MASIp#Q%BMY;yEe#an z)uqP&sbT=1-HcKg^m220qI(>l|61gZkkeZej$OTYr+AZjQO5lvr-qWf5{dXj!(C@$ z0tqWUPuDn5>c3Rm)cG|K+Kpi!ygIYT_+^vCrS~*xT{OQ?@o1jD5AlR&ibt|f;}JC3 z828_A;VwAaG|9ofkgc(J-Tm;jgnIXn&w~G*FOV<_*NZT`N7dR4$jTt)HLj?)X zwfk>{sJ0^sg8KpSbnm{d1`;zi#&Cxga93j{P%=yzzL|gb4|@T*=MT zPvq)TF{1O1uc4i5u)jhEzq*|U(kir7s{cIszeH18?0=uRxZ?;tneOz*VG*xki2Gy0 zV!*%tWt+lg{C}da#-siFbk?PQXs9Culei3%`AVvPp9((X5ZN$}2of71w8BEcWn()@ zhL8G<^Uq(j`n1vS(K?deXro*_tkDNv30u|1Mfli3%;YvaOR!s2KW~+T=7Kq}qi>79 z`Le2284~5yGXkrY@&)Wx_H_x2yQmLCI7pFvA16{@ z4?)LS(fb%k^L6~(Pfl%hfY0~vfnSD?eOk$iBxANAfA$z735Wl6kYEoAfXVLoEI@DN z>D$Yit@4phx~W=1bP8G{WUDGiFffBRP}MIVy?=ETh2FR0;)2pHdRzjmgUrlKPR+c# z8j2Pc^MGgO1G)#`;$L_?VN*+gc?L}%Pb5LI{v1;8$|M^=58Z_3uAe6Y;4Jb*Jwwo( z3;w_*#(0R5INqI%xuWwP8Tc|2944I{vT|`*fdpUaCQzgptcDdl|Z?vp>JFL2x}*uTJc%v%aO1+eh+_C$q6eS-3D$Q9fy8r@oj_{qmumjBEJd_S(~we$90j%q0yL)oD$>jdlBts4a*0_U?|>$h!)? zaX=eGs6Sfn$$pY^zfkhwO*gl1VrZ9vZAM6D1JZx@?)3mKw=HOEzB{D`sCgAzJ=&zj zvx)B4Xiz`WHq=hw)g>p%B=b_P##EtTkV|uefE9>Te|E(6f-Eubium|D3C9yzb?o~_ zsq7<>H+|W~yxQ6-_N(fv2k*t1=@BlKU;9*T6Dj^-G^s>*kIGy)E=DFiM&xi%>cKZ7 zPyF6{IRu^_&K9bI--Y7RH(eKLvb-je{2O|x)K&80`64tA_ACp=_^;dUr z9E-)@icM>q6fITtD-~z$e85J{8lB@$3WsMG|MiO9F-ja*yU~ShSJsKm+|hT}>gX~a z-+qQ4EAAu}_s{-p;68I68SRH8po9_KzT8de+_%>0#y~ zI}VS~alw7__s#M2J8|M`H3;q-Cm@tgA~8I?%3z0H_|8>sop0(O=?zrcth?|00`;6} z@NA3|feHdlN*iDsUM!oA)}ENe6JciI=U3ZXFt#i%uJ_!^ws(5i4Z8Kh^Os_;mt`zs@nACDsnh5KFO*IOnE<<{=&d3&&M zG77z4Qi}bR*#CV_VeT{UqE0H2+K9HsH@dWna20!_?8BbcD3X4sj``M$chb@*CO`eI zct~NB=yADmM1E;0zy>B5*kia zpG;aVgebk?@qG`}xv}5gKWMEWtpru*3>~#AXr_q$cb#MSK*jEYc&p@Fax^EZb1AsAUpVTk#hRQ46dP~fz6|sj!rKv zsGmtFndA~xC{4*87u@sC9!BRx4A}i4&x>H4IwNtuLST&XnBidd{ z-#~6qU+u{!)h}N2T0!cCr2+DC>}ruOV!~8Z-(_dyv)6t^&7d@yYYS9S3R)8+@%?Ta zi@6^plb1;U1Ul+`{o=BT;ps+-jj?Bqo?3BD{=TyK*qeNm^ibenf)clrsNKXR<~~gX zg-%^@zS4owIfDCg5B438wh{G$>|phQ@3#qS?|bD%crQ&8N(COz@)=KNB#uz7YDx|! zjE62CE^lm14HjeDKmGi>uvc028Y9SVbx-s)e(Aps~Q-2BSPzbAN0C06}*{EQe_O z>%ljt1!)5^?($tj;NzYrA+dLG0E(#f6|+|@v0BmLeBraVrZ>5(f4yszw4|x;Q}b8v znfX4E;Q`1P#+t- zPw1WN82gE#;j9KlgWx4I8(r(;wNY*Y>#qu5WG@Jdt_PA*4te$V(k1QjcGhge8BS6Z zXDK98H{O%}LByG2%+No8Wy)qZXc56jJfnl;d>6#fhnM>)686TwDNyR7@*jM8L9UUb zrBV}o>q1ntiK!`=;1!Z42s^MyO`C?vqUc0j!m6Ln3srV>bf~DPsADwfE1;h$_(nxt}SJbtc8U5ANkuOxo4bfuha;(aPa z`<*#P_RlRgG8d;Jcb?I{H*>R4lAnm0s^Q$*(`2seD*BM>O$#e_ZrAJ@7>*mOG-&eDLQFe z%=`Ed@)~LZvv!a<*dI;5L*2-TRnY(VUZW9CB)6V8H@6KadN2?Q+{k+6BrB&gcHx~$ z1@BgMzD>T_&P`m_(08opZ+;VtL<$s=pgW2Ru2g${Pe`F%oC&-B-={cab-=k}q#Y9Se1zB_>z zay)u+GztpGH`tk8vb#;b5;|UQ;wGKeTLlwj4ha0MC`9W7Bu|nu^+Ar)X})Vbtrc$! zLSH8b0|SHPMKcs;0e45{sO@1`aC%)>&%*B9Jy?(pN1lR<{=VqHs`Y)Kn{nH3pU`=A zMja{)d^}8^ejG-kUo3aLZtn8%t)q3EJWQ_J?P?*< zUJpmNhRyQ|=?T|99~d^)`o)#2>0VxSogy|C>k-UrqHRp^ke9#k{VTzCg?QV_W5=BPRfX>Y$BVq?gb?ZL`qdRHTD z-L)~k(_34!{%R(YAr#w9bfmus^Q8Bo85=_hS`WR6P6WN@sfGyBu~7`oEY)d5cj7ll zpI)Q31J^1LS(iogSnT6%Og3B^_)^*>ht_QK!laARxPpvcdlarFxl`pf8^@sYh zKfXVj`;4iphi)N10Tj}Lqj}iOK=+bC)n*GG|U$T*{Y?l z>m|*4jox(Tezh{pw_Sew?qjRZx@pegDkTZ=N-CBWX^oS~92)mEl;+-OkPYxdyiX2C z_4wU5SJ~Y6(TH97)~z3q@z79E#6(ALcd01{8K3R^^$UV}BN*uroj^RlXp)I79D9+5 zrp)`qi%{EPrEI4hNcxcySFe5n_A>(+CI_>VFR;Ikp3&0!7CmqDEHL2Fx8kU{oWfC| zsB(mlWIpaFDF2>ep=%9vh*6kVcmA^9U0w+~k|RyHK2gwIU{tp1dN}Q$R3VtJWBcr7 zy!?EcUq<4Kvblrd*;)4Lg`;2({=07%d$LlTSGQajyplTiCU>(P1?QC1=W2UcdYk5Y zjvH=%<52#n(SUNbG*&XtKDz`Zv*!&fu3Otx3I!L(n$ei7zs;o=6cm7oDKjT$@C!50 z`8n>()fW1A@7!uT`=wjYZnU8nKU>Jxx5bV+2!nz^acA_j{vkrj@PZpCPAkP(G1e3L z^cKdD<+9D=)#{Bpx+9lEmHm^BKE}vfH11Y_APhhZWP$ZC!zIy z;=_+~&gbz30c6 z&kFr%7gWA7U4_)?CJJK=1OJJaMVHO)u#JzwGv1D7TBMO;@J}BA!?e`_`Vc68Tzq|= zKR+2cW*YWFYFhlXYw6f{^#+L)HABZFUOPfB=iD&xO1? z9B93CWsX)dwx16A|L3)f_+F|f5R(()QU<^3@c$q_ zYV4xU<{>p`+9OI3u#VU_7~_T8*3Nv!csS1n4Y*Qt%mU|*4e06F;{Q2vlqgf%$r^_l z(dVcqJ_kZt7@eksjD=~T@AaSeijUGpi2c!uy}kBO3ut?m?{$Pqg^25 zB^10-WA!d4_MOoW^uC(`f<%5C@=ogpn%&6}bH>P$w$Y=VU_m3Ln#2cvcT4c=kCzrm z&j#sAU7@)_Wa=Z#5fE>Wt)=AD$)IYcN}kR1zKpHAp;p6IX#^Fn?uf0Gg&nouUbUSm z&l@6CN&A^RN;Z0MEcq5|Xb?w&zDlklJNxM-^2W%qr#mIM{?FOm@`V$GToVd30WcNC zZ>k6Hy#y?tAh9&UM@w}gEpi`cOFu^lm*m*{sh_pS88i~(Z7R;-()fNmjOku0YA~n0 z!3fYlH1Aa^-p#hrXZ`6x_MeiSVZZ+ARQ~sd3jhcK6}d|fe`QYlR79oH3$c(!`7@SS zO|n?^#{PbOza_X|Z=KsNlKB#OcR8koprGVHK61f+m0JJ)JB2#Ql9irsWvTz!u&MvY zQ@I-!(w14-AGouKJ{kQ0)E4q8`5H-TOP{}+`MVHCjh@6ci?E$`*EiBWU~3?f(BR zoK+asbNN{Lt*rfvrL$#QZ^y*mw~^v=NZU~6SKW2LD~&vkEF9G3LEW`0cn#+J*+-i@2X_0?QD-!5}-aEOT= zf$%#Xh1WVLpgFq~sqHLtgRx^^k5;)yK!NYLQQaL%k;jtn2C3x??4PDEU^iQ6>wZyv zGx+mP&iViQ^~u|88pcDpdVTWVHYU7MAEC8W?$uaS;;v#sLBY*;7}&4D>5DH}&sKKu z$JOP{aHH8kWaCG;ExP-NVR3xE4oo_#~&Eq9M* zJ&p4aF)i<-=Ds!gHX(zFDLL-XDu7BV3Np$?vyOLZjfbu=p>0+QOrBQHo~tacX)x0I z1Q&!0&+11lJ?M7q{;U~4-E4opnJ+4_^W784nz5VX{tev$1o%V*1P3eLC!A+FwJMIO z2g0GN)sa&7WU;TrG(X0@v~MA*ylRVDpme7k^uESYxEXr|EQTbfu1w{9=OCcsdnq>B zXLnwjFz8Kxk)H)dj#aDC*SOG?jBZ{z-q{h)ZCwpoqNP4%OUb3PUWaC{n=1#y9GEw% zmz|6msL0f<+q9KOq7ozN8}-U867G_1!PZ!S4#Up;mVLo=-YV4nZLUa$7fNv@QQU}JMgu&l-G994_?4KIH-NG{5JrW{HkiD8lX7HQqmA z3CS0?oe}*T9$@z6n{Rdeazo8a?D zu_W5-*64dBRkxoL(Fneme)aafj~|GUcK!r=j3Ue45ViWRTdrw(bs-4~Q5?1Gl41blfb{2{-*h!Dv2vH1mH(HI&f(S#>Gwy;qS-lS zQ%n6nG|Cyci-!&|NtC2IR$iPTDAWwwXuqkJtA4sue!nESlwtROU++;*Z1a1A>&pb# z7YuolGmtOdPCJ%+J|FYoEF96tlcDHZPlu$srR3u}7uIm-+_WBXG=ja3S)88hG#T+% z+g_cheXQqC2`R79KRy?5vXMk>T7LYqcR>*UBD4%WRd0{enEgnkma93pQJ+bg`+$6kOQk1ez$N$2MLW+M*RMcL z#Mn+~JA;z{CuLmw-bN2w;w7@OzyC>)S*TZCs%H}y2HxG4`2;td{1y2EL+v9$RHmvZk)aL|$PmkocpMW^(-Tf@J6HIuF!F&nfqVF6U6RJYPj%pw7g(8t; zevgFfEPn(l%&FcS?!E)QJQ|>SF%VN;lj2-GHgKo~gp!ja$-`atN zdidBkIoWz~BX!nt@y*bY14kYT8Kt)2H~q zM8;bAMhJ_XQa*JNy=r~b3#h&(bqF!MZkYlE!e*U?6M`|l*eYbz; zj~PQx)$o0lC?dR3jgHp5J&RwWdK5{t;D+sK* zJ?eepUmGmp>A6ikHDadHSGkH7oj;`(HbhG`iXx<@73ecKF@_luWU7^OwIRwlQ^h}eB!LKA2}NrLUrA1 zH&A72yPg4?h_Q_VBFx?0F<*a%`o1OHHVQs^218lNB-;Eh-*xVnSX;HE>cXAjve58d zvF`Zp@uG}*yi8OkU)|N+pO8GEPHT>NwxKw*ZV2OBqhc{v=)~96Ic^S=YVjRIjkc$@b)OLd(jB z;6=75&uHA=6jCOc1}QQosmAj@5y0Om4SyvONHaoTc49SL!QqB!wiMMjDibw3V$9B$uoWz zw$WF?#2=aHZIi)z_5oT4p*{SmrJ+zo5bHs^M{=>ab#10CyL{7C<1-dVcR(48xNJM8 zV~n!#aJBwTZ@Ho{RYFuGvG>Fey6i>+1dWLloma5EFfXJ-TAPFMPrG=Zq7pg0Fk>G9 zA92m6twhgtl3zueo-fm7I*V`8ol;9OJQSKt|mN!7))~DYF3@XYG z@kyF_{u@N$c*TcPJ+r`SP_Qv6A;EKPqSgpT$KoElyK9r*omZeUT2w#9@XxpUlh2yB zkJVD?lVtvX0=-jowtl#G_SX*pC_&NA&E9X6`M0*oU9-!y7%z6B!3 zdLY0W-oPcl3Y9!eCq#B!g}~DfunE)@6uqf(=fg2?$R&3$wnK)E@$iCx3!v70TBqW5T_5k|EAT@eiG-wTV@5Pe}B#yCs{Rb)XTMeZN(quq#oFi9NQ)8F7 zSw@m^4yF{eBVibo%%??8C)#EYzAexg0<`%)?0n_YYq;<%F7YktL4T+@rx!;7v$k5w zdHV*oO7^C5q@i1+@^tt@_VYLQ4n1lk*Yhp9PqFEzZ_e$QfCnWB0@=qaC&w#@Zuex! zby|2gPVS086x+oX+Nh`bPhMx>tkzKx9Ct226wlkt|5`+Wl-k>`9 z4Zsg#i=n9Y3#J~>zl6fr9~($nC&Wi;h88nM=Z&a^chWgO%^#&W)3yxRJr!{HY!NPI zNolZrZRwW0^w07Q2?T+EowATqh`X*@)3)S6Q1eREm+uHK1-YY;8VhwUibF%hV|UdU z&)nP`CRJ-|Ymmi~BJ$w&_Uk|}TRS6wlD?gUu60Yk82Tt(1UeuY4IXRGBd)U`su`D% zfDAPNr6TtT{f4`M8|DS{KC~+Y7XVJ_A->lI${#ry-r&sbdBv+ZK*SjJiH0igs;w#5xvoChAm^PVB(4Kn}rESEsmi`AfgI@V|Y;0_6 zp%KQ%m( zS5mSbD(1~l2Ms_Gk+-TB1k5VgQM>17LD1-Dwb5mOxhx{^B3rqu>JagO`)ak(YmN1*Ar0<&~5j!*wyYe0_B|L<5n9R z4xtn+P;iJzii?dXUpd?%6-#pDg zhw1*)PT^4dw2OCV)T26E{~SeUNRBG?*!0Q|K_+)DJ|3)N5qtpX7>kRFio!G;<^(YA zPO6_GKOL6p{4XEz7$-jtt6~mr#j*vhaG<`rO*clx$H#+k z&C+Ov1Scn_goHQDpx{r>$D-Pqq3Zj4Zu{cVj#n_Q(pmSaNIrf?s=#{wc>n)Dem^2y zX1>|$%RFmtHlQd^@%xJGRdEc034!X?ukS>>Rwav?P)nbk(ow6hC)tstrX~HC8w&U$%8<5`c0E0aZl^QRVcw(v@+i;^TE*e%PppRm;Z-xE#=M2HAdhgK)CqkX9@nJ5cq% zo9_ry20}_6`pcJd?&UrpErUY}#+siBtGO@UF%X3=JbAIO-4@V0Ajv6x38zKNS|{KP-729bM$M zWpTm#MeVXGIA>rMgcguj@{g=!Ogc~ifl*|I(5ogWtx>?A*xoG)!p_EpO2;n?sk93Uy9du(D=?l z_a$fFu1PvceCudV?^x;3d3)KlJNk39{2TJCdz*8^9o~oJWflFMgrsSQ>K0=&28~>g zV1?M5mBoL7Lc`DWL79{}682m@$8s^hl0rG;W|`=)aM{eJ{5EI|b2EYp$95l|iI+ z&GC0TzzpW%foihK{I>}ul32{Clm2*gJpWaEiR@K!&ABv2y9%9#2KT*OJdlcRc^(@_ z$SZMe=x{2vFN3uX(O+1KWw%4507d(T3?IGZOIVz1_(&AW11xQ8F_CAV>N2#Aw6?M&fw75ErgU|F5TxU;uo}r3N8`Z>S-?48W2wadI|w;^?)&KkQH6dmnAZJddbU4?5*Q(;0C2^j?3u zfjqR^wrg{O^T=MGQD{Iw0Kvs;VaLy(JtL$Majf#gLPYt6;g(^Z-hlr8U>sUGz(yiQ z4nE2J)*|M9AHcgu5T7;grzN66r-ED>_ZK38a`k^kKAxV*vhA_Vs3^lhhUzYCc zNej&$I%mg#ykMTOzP=%Ty>#YKFz5 zC-}Z6_8_j03|0Y;)dvEqpr)=4%JSf3xN)LwT~b?36+7-0Dv*YXgpYZ5? z6wW0ah|YU@?M+iOCV`oIOpGaCEQ}|OjEwX=x4KUbXC)ZUqXs5_l1j?$l~$5g3OY3b ziKKtEN=B-NoTH4%3Zg*NP_wz-!))@$7J`d!-;%w3;$uGkH49HD<4J<+IH1|fMqdym z#PMu2rOGvNbM?iGOhg0|g$7=dyUeGJtN(uG8Szzy!Cdj8pp;2(w=%UydW5(Cfc%l9 z4eHbhe%VSKKueh{_I3Z78;zgm)-Ed{y;eWJ=bJC=DR`ca)E$@k8J4LkHdBCwTKu!V7&djU2Rl;BxY2d zxXE$!el}0R)kdeW(lrPM8@LvE6Y(qt(v>!`ogu*NfTP0^T<>pNW&m?rRoxbHihsxvW^#tiP|2z}>E`BOw1E{MhweVaP32GtM={|7O^vW0^S(hl`@i& z7r~qbRJR>Nj`n7g04)R;9<|_OKt21k+8vATT=8Z1N$BO02LkGVSMb2=w@&Q#GA7+) z8!cz`Lb^TPl-w-Nx|g|2OCqtY7{ugeNyO7lR?1HMfxc$_bg!gb6Avco_D9p7v zRDfYA9Y{S&OewOrn(3Qju6{QEH8DAPe^_p*(e3s52J??SQ4EFPPXPWA5$IaXgrL$* zo*gzHPE|~=U`RFdIJJ0}%+{VOUdb|?ni1pk2+j2;j!OgG-vkNo3QG4VQx}1&v@YIk z`1RWUYvECait1L>2BK&WA?x+WAxck84O_mWpm4DKeFgk^oFE|?b^LV)ra5Kxm_FNP zFm2;P>a|)Rx1`v)+pn+B<|xhcF{W=gI?PhlYG=&NT;XqC$yW$bdWAd4R|-N;L?`!- zijBJia?dV;KFSQ4*HQgC8q@1KjhU}G`*Kz9{o;ckYzGt zLJ}Dw8j(!^4D9^wm~6ei^7vPJ3pm!pp7C7BeWGWy%*5=HcpGJjhdZ2_%G6o5E>tN> z)I7R7O^S1k(I@8e{aa_lB=$a#R9?n2e&3ZD&sM#yPbGJ?=|T-Up6YM=NU5s}8qse* zkp?Lv^vgJVf9A_(xJsHc;Re%64U*kO#6Qnh;x`PRfYk;8wP5NmdT#E}7w-1At(kcQ z-p7^^Kr06svCTRKlSJuu4wxMBrL1+be&Dvb{b1+qzH!{>>rQIV{hF6F{7;gGQ#sC@ z(y0EPaf{6}pI50Wivy#dgg_dY=x9^nbp#FdoBqwvTyKKiU+S_Jqvutno$PD18*55l z37xxlC$K}3c#W{J*b-9d1NNu|3C}s*zZ8`kf0+Omcbqg1{mqY540gpZU3ob^J&4cWjwoTBOJDpDR^utX%k1qC3sP`!McuQ$g50rNho>+c5y9jEmY2rYfI=%jtz z`@{5JnU?4M1>o5}gw9)d!^5xc|P5U6Oa;^!Gqg~jus1)e$LrKqI=%*K1TwYSk=aWXVEJ`_wf>ur);f3g@D9Ng5>@+v3@u|tyNU=s$=0X0IUfC>HaRSf(Mgf{s1 z3hnyMFZ;vP1s5ONG0@Chvlsx^AcLNWO>p7z{ZC916=N-j0*jlgg^W-|P~Jf`?Qli( zkjZUdl2eQgGV~6{^RY=TRC)TsUws)oT|f$4Gxx!lLF@0iaHSaf;skni=$N^0uZ${d zj*e6mVz((jK4+arY>`JttIb%WIBtMP!k!R)3^4c)n)&3$K19SVi_kXm4E>*$7Si>} zY(udA?Go?P68y2_q1n*{{!Plt60Yl0P+04vNWX$GJ?b*~iniS83pCs(&*)p=evAKo zH{_zQDw8nIiAZGQ{&9(!9tjmy3FPu#8d2-c1)mS{%4uGVpOva%4cnQXO)}^9I@;3Q z!{&C^X4s$z&i+A?`lycJy`EzLl$&(o9@m(!E@fb7?w8W&x%*^2(W=5xnyOQZ>~YL2 z+4k|vf4Au_SNPK7c!s z6ANc!*khw5vDEM(lreo7_&~=3WuU9?K(nFdc&t262W0 z7F)MIn1GXTo|-LO zNaL){i)Ap1fWq!`qy?-y?aCR=`r2voMxK1Is=Pd$`}JukcE(`No%^`PGZD|j$n3jj z-X)VRC2IuUm80LUok)Im09PnCi)qf$)xw2lyzT>;UK+FNhVbVfs7-XG{tRZ{P{eV5^b zxG=kX_iJelua-d0NDbbH@9wlTP*EB)bf}~ zQN9^J|GR-YS2VwBhmDzzU76iSZs6nH$2~c*hdtryq4?7o z3->HEr$6%;DQ~HWNJzGjacEwu7QC8~_W4Gv?3NauzOas>LYiXK_v8UVm_=P1*0sW4 zX$83AqkvYrvs|k%ul7ZS1P4=p6)5TPVOY-I9_z8+se5h3b4GK7l z6gCMiU5dc2!RMnLGu7(_rQV6W>ilPfgoNaF6}9POn8$vsp1wWkyHMeY7=P=0RFKsW zW~rOKHs*I?I%#j#l-tnSKzEJX?zdk}%Ry(Ir<=+&@mHE=rnX-7`-A5;Zdcz?p~x$? z8;$Wy^q3DS3+JWJkzYlHIoq{reIJfM@evgLQ6yfRd4t zDl1{hm9D=^A$70Jd(%>nywszWW%MW(G+8;GH;jAkZ5R_O=z4LMD`|z(VQH+`ysZ$v zz0WbLvL`w-RN}D6)91aa+S=e8(15Fgy)70`OGn4OA)EBMaFs$*XOgCS35eF(a~}khrI9vK$rEI zH=e*Mp6DE&09hrl_;^_#eYt3CExJ39@A(%0O2Oo8p?TpizO!EA4wu-Se^ST=vAdGa z;Ka{(M3tJD?ziFaXB=|Rr3tTPG-Jgw$8D$go(YO3dwmrIUL?d>j#blHS1M70{}W)4 zN&f)}AgRAP2A&<~f9zZ3!+VZv>B_mvZ~mjkQkcSPB4!uJ3rx~g8E{GcWU>@6MX06f zoiZit8$?RaG02lM>`Q6Ki#S;{I8$}+QCb zDhcw|ua^DqTfgRt?|Fi(;7eaKbLV%wsb^fVk)OGkf~)^*)T}l~uOjh%suQkiEu%~b zAoi{==o*4PIn;F&wJ$>yxE<^-!@y2A)5i}fs$6EcY>B0|6T$2jV zNWFI$d9UoZ=P33#Z$d^bb6RTt{X*%-Xsg&rYWo)<2q_meuOX(Z2no;A4+ef5#Fw7p zb%-d|B|W&QIgWdtrJY>l_9@og6fKANhp8645`#al#O>)fbJY@{#D#J6(xyI@qgL}$ z=^P!~q234%XJlmDoBX=Y{F4-Xy#hUCHh~?slyz@U&^-I@-hKUUya?Rb!oF4#-dnLyz`vYqu-{V$lT*9acIaZ&+#ohiQ_0EQyZEzb`wivE7762ibt!K zrX<4iqFu}26aftlj4>kskIuXD(0?};CR0o(nXCbc!KoV7#0AEK6JUPWGBA2PBcsHp z#O~;nngihiQRNp`Rethk4olU~LO$@-W;AisUn=YqGr_~0PXvU7>G2Pru720N!O3ba zm!B=p!lcF|-<^Hf7aG|pzsJG(6?tN|#T809)TmP%29uLyjre=L&kUTLALW(}g++EBdd7H+Tun_4n;uP7P`0MwMRAWe8`ppi(EPI_;kt8Y z^hPk}Az??eh)>8nkWB+D#3_80kpMehau=VIhmz-~zRx)>oz7Eg)a1@2nrg`*Rti*L zrjq$$yBb@KQGCUC#e^@ETdZcG)EM6+K`K$wdmQP$QPs|&vl!Zdk{7D`#z_YsI47?$oHe& zJ|mU>e&Idtayx_J{2zlVpI5%)LSpkl{HX4pglKxVtQs^i#N4+7xoqFUXaPVGkl%&G zU#$2MUNE;Yb)AxCS2PYQ$`*TseeOcmt?bx}(4?d!Z|IL26LbcVYu}r*Eb~KXs_v{H z69PkeI{ku;^XOWBRX%~*<9c8s^4U@DHvFY{T}2&+>VDBqr}mW(>RCUClx#-Jb(nl_ z)P{+#01HUjZKWx>yc)C*;ceC;$jH~P`Mizfqr9XNJzw`gLyYam=B&iLkhWH#+pO{W zlg1Ssj$Dh1jVA9>kGW3fbveUfrIbSxlrYg*&I(@odwPL1CK-=#+8jWX+$) zl4s@6yu-E-W5C+vMuEII!j%7dacA_IDe(ghk466^UyhWpL3bBid`P)DyXjd+wih|n zOQwg)=ecwU4vBB?mv$OAmV=t;VqB9Abi>0_Z)^r{??E2JZK}H@%9RdjG@~+ybGL_< z@2tD6Ru#7&jNBf)d})4_H^mB`Ck@Pv?L0po6+X^Io1rkNH8#<6TJ&11 zvi8q^h4Zmp4C_= z<}(RN6N*{ zcW6kay1PFOAwzc4$7_l#zuKOEg8Nd4>b=*Q7o{{ubQPPOF@wLvhr&pD_IH!W?Xz2H)Sd;T&FBsR?;er{*TTZWUx34TkIg^y{ z3S0FwrgmSyLbOZ2mVps#lOxY^(rfHC#5$VoNgI`W;qiqX^9ktr3EGTYw!j&nN9g?E ziwPcHU?*3A0XRgpzghS}zsn7!WjpQg6FP>glF`VF6~QY2qW;mryp9L zy-{DO&XlN^yvN!s>9t(jzXmG*ari}|XUSA*#=D6^U0sUVS?1<}-68erh62!?eSnJ( zV^YM|`^HB|{G9`%ub9J-d8cpR{EwIMyt(dMn^uXuzyQ*GPo7TP+qe5mbee9Zw7-ns z&h5>an7*{%c4IUdd>0Q_n8(N9G1q0&Ni?q|esd1u zURtk9J#f)UJ{)DMO;k^0!n$9ajak4!9Uw>d$xNMv!$4R*7*b#osv_MO#qkS^-8r|c zaRagf{c|~<=tXL<^Hbn?(DG0szzDengw>DItkMS~_Kr^GJ3#aI5~LlQP4Wz_5XG`m zBd})(K%@!i)h-4~$F|xscn`$9zid`Qax6_%wUO7e@LsaBGG3-cS~@Reoz2t-_r68L zYi4$K_U)p^SkHGn6qj7G_$z18{4XJUp$ox#oEo;HLLOKlM?k_wfx^ITq;Ei1%)PkXIYg&F+hMw#(0pKCi}m zv|N$Wkh6>^rY4a1zd0mW(0~Kf8iUaSUH5;V& zotMV)par*jRhYTi>ff?=Hn1g`>$~2hK`a*#_S;yaS^9-0vp=rqn2XW#YE(-tfYeyB{~X zMV^zw!!dfIXx}@+an1iJUuLxGjb*|pjh;z(Pbf3V(|zOn^T=7iV9t`uGL~35-{u~3 zCak~QsD0&4ljy291cUYgb3wBk=Q>FM=rG#_h(Pj$# zo#xKR*{jf!kJ!oE(Mv|jC@OuG&oTAMU3e!S>3XK5-oz_bHcFfB5ss949Ba%29J4HH zt}{tE@PNpsx_DP~*p_Kd1L)D);pkd5OSfHl$S#`wy8_CSXldkz>Mt~(QOT<`3X|V@ zS_TBsXjgR=eIl>p*fjR%8iswLKFD@}UD8vWs=jhrGtK!!36sCn&?R+N0oxnC;MNAh zC=W&DjhPK}8iUSNR(^+#olFe#Ez1%2xkcD*!GY7xr3|TK$JgJkTITiny=^;(myhlva{2qk(+i5HapLa(l4s;qq+)Rz zQR3)L^qZbwa%OMlE?0e^bui83&pxLAladnpc&PR>aG)GHRHC=pj<6Tu?)f9;H>4fC2=L2PcI8}&>;=c(Ln%w}e2ZtS)yP}sZb z7q5Z+UL8fj+qy^Cz`L*X?FUFexvc6OJ{j?vR;clM% zB)`+K;epToi{(GEo7MJ}Up~9N%Ywn-eu51{LvBGX!vg=j^%dWjeBa0O$~xxATsHC2 zghTnI^RE7F_`1GKvo(I=QMLHur7cVKb>H62TIa#k*z)v3s9SN-xyfFaK4NVJG3hP!=xj;+52NOYc-bwGfbrP<@)q_ zePs<<A+xYFQiXR>Ww){JGTz;^KjfXSgHUA&4XkaP# z|LtA-ENMp(_Jq)^n4NqUQ}q`pxy~}tbULu1ZO#Gr3yMYT;;V`}zvSfj-;SjbjpnsVpIhnK5Qp5Nephu>j#j)Q>{@0(LQ7(bpc z-ExwRMfu8sbv@Do{V~p9DcP$jMwJy67FJeK`TcIO*KKWW=gyh4X8S{bNpl6coo!|} z8xmTqR-I+5u#t<8jrCoAc}rbEK|y`}{;0LnPN&{X-#o+5X#V;9yu6^NU%r^kH<`4=m~HSy>)TXpTE)HkF9*UclU1Z zuhYBw`}_AUo^^EH>%9DYet!PX4_8jS#=xF-r=jAB)U6ceCc#%LUbSso{_}Evck15A zD-8QJkMVZ6uUqJArXaDfZ`I8UVyWAT|Au@>E39CibNzGux$D=j|KxuXqJnJiXa6Pd zwx8*Kw`*P9mKhaDfq*A`_FjK=^4L#pcX4s?pUeL*u49|A{V*gf=l-3sq#{CRnt?ah zSF3w(X20#fH7z{o>%EMtzw7ch&zK^fbalGMbDw3GGv6Mn^L}_X@BDJ0T!Ugz{48}*^{>naR-Vl9bGmnF&9&D%r?2BbZ+#_rL9nWQ#T@hM m *[KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api#kep-1645-multi-cluster-services-api) provides the formal description of the Multi Cluster Service API. KEP-1645 doesn't define a complete implementation - it serves to define how an implementation should behave.At the time of writing the mcs-api version is: `multicluster.k8s.io/v1alpha1`* + +The primary deployment scenarios covered by the mcs-api include: + +- **Different services each deployed to separate clusters:** I have 2 clusters, each running different services managed by different teams, where services from one team depend on services from the other team. I want to ensure that a service from one team can discover a service from the other team (via DNS resolving to VIP), regardless of the cluster that they reside in. In addition, I want to make sure that if the dependent service is migrated to another cluster, the dependee is not impacted. +- **Single service deployed to multiple clusters:** I have deployed my stateless service to multiple clusters for redundancy or scale. Now I want to propagate topologically-aware service endpoints (local, regional, global) to all clusters, so that other services in my clusters can access instances of this service in priority order based on availability and locality. + +The mcs-api is able to support these use cases through the described properties of a `ClusterSet`, which is a group of clusters with a high degree of mutual trust and shared ownership that share services amongst themselves - along with two additional API objects: the `ServiceExport` and the `ServiceImport`. + +Services are not visible to other clusters in the `ClusterSet` by default, they must be explicitly marked for export by the user. Creating a `ServiceExport` object for a given service specifies that the service should be exposed across all clusters in the `ClusterSet`. The mcs-api implementation (typically a controller) will automatically generate a corresponding `ServiceImport` object (which serves as the in-cluster representation of a multi-cluster service) in each importing cluster - for consumer workloads to be able to locate and consume the exported service. + +DNS-based service discovery for `ServiceImport` objects is facilitated by the [Kubernetes DNS-Based Multicluster Service Discovery Specification](https://github.com/kubernetes/enhancements/pull/2577) which extends the standard Kubernetes DNS paradigms by implementing records named by service and namespace for `ServiceImport` objects, but as differentiated from regular in-cluster DNS service names by using the special zone `.clusterset.local`. I.e. When a `ServiceExport` is created, this will cause a FQDN for the multi-cluster service to become available from within the `ClusterSet`. The domain name will be of the format `..svc.clusterset.local`. + +#### AWS Cloud Map MCS Controller for Kubernetes + +The [AWS Cloud Map MCS Controller for Kubernetes](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s) (MCS-Controller) is an open source project that implements the multi-cluster services API specification. + +The MCS-Controller is a controller that syncs services across clusters and makes them available for multi-cluster service discovery and connectivity. The implementation model is decentralized, and utilises AWS Cloud Map as registry for management and distribution of multi-cluster service data. + +> *At the time of writing, the MCS-Controller release version is [v0.3.0](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/releases/tag/v0.3.0) which introduces new features including the [ClusterProperty CRD](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid#-crd), and support for headless services. Milestones are currently in place to bring the project up to v1.0 (GA), which will include full compliance with the mcs-api specification, support for multiple AWS accounts, and Cloud Map client-side traffic shaping.* + +#### AWS Cloud Map + +[AWS Cloud Map](https://aws.amazon.com/cloud-map) is a cloud resource discovery service that allows applications to discover web-based services via the AWS SDK, API calls, or DNS queries. Cloud Map is a fully managed service which eliminates the need to set up, update, and manage your own service discovery tools and software.. + +## Tutorial + +### Overview + +Let's consider a deployment scenario where we provision a Service into a single EKS cluster, then make the service available from within a second EKS cluster using the AWS Cloud Map MCS Controller. + +> *This tutorial will take you through the end-end implementation of the solution as outlined herein, including a functional implementation of the AWS Cloud Map MCS Controller across x2 EKS clusters situated in separate VPCs.* + +#### Solution Baseline + +![alt text](images/solution-baseline.png "Solution Baseline") + + + +In reference to the **Solution Baseline** diagram: + +- We have x2 EKS clusters (Cluster 1 & Cluster 2), each deployed into separate VPCs within a single AWS region. + - Cluster 1 VPC CIDR: 10.10.0.0/16, Kubernetes service IPv4 CIDR: 172.20.0.0/16 + - Cluster 2 VPC CIDR: 10.12.0.0/16, Kubernetes service IPv4 CIDR: 172.20.0.0/16 +- VPC peering is configured to permit network connectivity between workloads within each cluster. +- The CoreDNS multicluster plugin is deployed to each cluster. +- The AWS Cloud Map MCS Controller for Kubernetes is deployed to each cluster. +- Clusters 1 & 2 are each configured as members of the same mcs-api `ClusterSet`. + - Cluster 1 mcs-api `ClusterSet`: clusterset1, `Cluster` Id: cls1. + - Cluster 2 mcs-api `ClusterSet`: clusterset1, `Cluster` Id: cls2. +- Clusters 1 & 2 are both provisioned with the namespace `demo`. +- Cluster 1 has a `ClusterIP` Service `nginx-hello` deployed to the `demo` namespace which frontends a x3 replica Nginx deployment `nginx-demo`. + - Service | nginx-hello: 172.20.150.33:80 + - Endpoints | nginx-hello: 10.10.66.181:80,10.10.78.125:80,10.10.86.76:80 + +#### Service Provisioning + +With the required dependencies in place, the admin user is able to create a `ServiceExport` object in Cluster 1 for the `nginx-hello` Service, such that the MCS-Controller implementation will automatically provision a corresponding `ServiceImport` in Cluster 2 for consumer workloads to be able to locate and consume the exported service. + + +![alt text](images/service-provisioning.png "Service Provisioning") + +In reference to the **Service Provisioning** diagram: + +1. The administrator submits the request to the Cluster 1 Kube API server for a `ServiceExport` object to be created for ClusterIP Service `nginx-hello` in the `demo` Namespace. +2. The MCS-Controller in Cluster 1, watching for `ServiceExport` object creation provisions a corresponding `nginx-hello` service in the Cloud Map `demo` namespace. The Cloud Map service is provisioned with sufficient detail for the Service object and corresponding Endpoint Slice to be provisioned within additional clusters in the `ClusterSet`. +3. The MCS-Controller in Cluster 2 responds to the creation of the `nginx-hello` Cloud Map Service by provisioning the `ServiceImport` object and corresponding `EndpointSlice` objects via the Kube API Server. +4. The CoreDNS multicluster plugin, watching for `ServiceImport` and `EndpointSlice` creation provisions corresponding DNS records within the `.clusterset.local` zone. + +#### Service Consumption + +![alt text](images/service-consumption.png "Service Consumption") + +In reference to the **Service Consumption** diagram: + +1. The `client-hello` pod in Cluster 2 needs to consume the `nginx-hello` service, for which all Endpoints are deployed in Cluster 1. The `client-hello` pod requests the resource http://nginx-hello.demo.svc.clusterset.local:80. DNS based service discovery [1b] responds with the IP address of the local `nginx-hello` `ServiceExport` Service `ClusterSetIP`. +2. Requests to the local `ClusterSetIP` at `nginx-hello.demo.svc.clusterset.local` are proxied to the Endpoints located on Cluster 1. + +> *Note: In accordance with the mcs-api specification, a multi-cluster service will be imported by all clusters in which the service's namespace exists, meaning that each exporting cluster will also import the corresponding multi-cluster service. As such, the `nginx-hello` service will also be accessible via `ServiceExport` Service `ClusterSetIP` on Cluster 1. Identical to Cluster 2, the `ServiceExport` Service is resolvable by name at `nginx-hello.demo.svc.clusterset.local`.* + +### Implementation + +### Solution Baseline + +To prepare your environment to match the Solution Baseline deployment scenario, the following prerequisites should be addressed. + +#### Clone the `aws-cloud-map-mcs-controller-for-k8s` git repository + +Sample configuration files will be used through the course of the tutorial, which have been made available in the `aws-cloud-map-mcs-controller-for-k8s` repository. + +Clone the repository to the host from which you will be bootstrapping the clusters: + +```bash +git clone https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s.git +``` + +> *Note: All commands as provided should be run from the root directory of the cloned git repository.* + +> *Note: Certain values located within the provided configuration files have been configured for substitution with OS environment variables. Work instructions below will identify which environment variables should be set before issuing any commands which will depend on variable substitution.* + +#### Create EKS Clusters + +x2 EKS clusters should be provisioned, each deployed into separate VPCs within a single AWS region. + +- VPCs and clusters should be provisioned with non-overlapping CIDRs. +- For compatibility with the remainder of the tutorial, it is recommended that `eksctl` be used to provision the clusters and associated security configuration. *By default, the `eksctl create cluster` command will create a dedicated VPC.* + +Sample `eksctl` config file `samples/eksctl-cluster.yaml` has been provided: + +- Environment variables AWS_REGION, CLUSTER_NAME, NODEGROUP_NAME, and VPC_CIDR should be configured. Example values have been provided in the below command reference - substitute values to suit your preference. +- Example VPC CIDRs match the values provided in the Baseline Configuration description. + +Run the following commands to create clusters using `eksctl`. + +Cluster 1: + +```bash +export AWS_REGION=ap-southeast-2 +export CLUSTER_NAME=cls1 +export NODEGROUP_NAME=cls1-nodegroup1 +export VPC_CIDR=10.10.0.0/16 +envsubst < samples/eksctl-cluster.yaml | eksctl create cluster -f - +``` + +Cluster 2: + +```bash +export AWS_REGION=ap-southeast-2 +export CLUSTER_NAME=cls2 +export NODEGROUP_NAME=cls2-nodegroup1 +export VPC_CIDR=10.12.0.0/16 +envsubst < samples/eksctl-cluster.yaml | eksctl create cluster -f - +``` + +#### Create VPC Peering Connection + +VPC peering is required to permit network connectivity between workloads provisioned within each cluster. + +- To create the VPC Peering connection, follow the instruction [Create a VPC peering connection with another VPC in your account](https://docs.aws.amazon.com/vpc/latest/peering/create-vpc-peering-connection.html) for guidance. +- VPC route tables in each VPC require updating, follow the instruction [Update your route tables for a VPC peering connection](https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-routing.html) for guidance. For simplicity, it's recommended to configure route destinations as the IPv4 CIDR block of the peer VPC. + +- Security Groups require updating to permit cross-cluster network communication. EKS cluster security groups in each cluster should be updated to permit inbound traffic originating from external clusters. For simplicity, it's recommended the Cluster 1 & Cluster 2 [EKS Cluster Security groups](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) be updated to allow inbound traffic from the IPv4 CIDR block of the peer VPC. + +> *The [VPC Reachability Analyzer](https://docs.aws.amazon.com/vpc/latest/reachability/getting-started.html) can be used to test and diagnose end-end connectivity between worker nodes within each cluster.* + +#### Enable EKS OIDC Provider + +In order to map required Cloud Map AWS IAM permissions to the MCS-Controller Kubernetes service account, we need to enable the OpenID Connect (OIDC) identity provider in our EKS clusters using `eksctl`. + +- Environment variables REGION and CLUSTERNAME should be configured. + +Run the following commands to enable OIDC providers using `eksctl`. + +Cluster 1: + +```bash +export AWS_REGION=ap-southeast-2 +export CLUSTER_NAME=cls1 +eksctl utils associate-iam-oidc-provider \ + --region $AWS_REGION \ + --cluster $CLUSTER_NAME \ + --approve +``` + +Cluster 2: + +```bash +export AWS_REGION=ap-southeast-2 +export CLUSTER_NAME=cls2 +eksctl utils associate-iam-oidc-provider \ + --region $AWS_REGION \ + --cluster $CLUSTER_NAME \ + --approve +``` + +#### Implement CoreDNS multicluster plugin + +The CoreDNS multicluster plugin implements the [Kubernetes DNS-Based Multicluster Service Discovery Specification](https://github.com/kubernetes/enhancements/pull/2577) which enables CoreDNS to lifecycle manage DNS records for `ServiceImport` objects. To enable the CoreDNS multicluster plugin within both EKS clusters, perform the following procedure. + +##### Update CoreDNS RBAC + +Run the following command against both clusters to update the `system:coredns` clusterrole to include access to additional multi-cluster API resources: + +```bash +kubectl apply -f samples/coredns-clusterrole.yaml +``` + +##### Update the CoreDNS configmap + +Run the following command against both clusters to update the default CoreDNS configmap to include the multicluster plugin directive, and `clusterset.local` zone: + +```bash +kubectl apply -f samples/coredns-configmap.yaml +``` + +##### Update the CoreDNS deployment + +Run the following command against both clusters to update the default CoreDNS deployment to use the container image `ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.6` - which includes the multicluster plugin: + +```bash +kubectl apply -f samples/coredns-deployment.yaml +``` + +#### Install the aws-cloud-map-mcs-controller-for-k8s + +##### Configure MCS-Controller RBAC + +Before the Cloud Map MCS-Controller is installed, we will first pre-provision the controller Service Account, granting IAM access rights `AWSCloudMapFullAccess` to ensure that the MCS Controller can lifecycle manage Cloud Map resources. + +- Environment variable CLUSTER_NAME should be configured. + +Run the following commands to create the MCS-Controller namespace and service accounts in each cluster. + +> *Note: Be sure to change the `kubectl` context to the correct cluster before issuing commands.* + +Cluster 1: + +```bash +export CLUSTER_NAME=cls1 +kubectl create namespace cloud-map-mcs-system +eksctl create iamserviceaccount \ +--cluster $CLUSTER_NAME \ +--namespace cloud-map-mcs-system \ +--name cloud-map-mcs-controller-manager \ +--attach-policy-arn arn:aws:iam::aws:policy/AWSCloudMapFullAccess \ +--override-existing-serviceaccounts \ +--approve +``` + +Cluster 2: + +```bash +export CLUSTER_NAME=cls2 +kubectl create namespace cloud-map-mcs-system +eksctl create iamserviceaccount \ +--cluster $CLUSTER_NAME \ +--namespace cloud-map-mcs-system \ +--name cloud-map-mcs-controller-manager \ +--attach-policy-arn arn:aws:iam::aws:policy/AWSCloudMapFullAccess \ +--override-existing-serviceaccounts \ +--approve +``` + +##### Install the MCS-Controller +Now to install the MCS-Controller. + +- Environment variable AWS_REGION should be configured. + +Run the following command against both clusters to install the MCS-Controller latest release: + +```bash +export AWS_REGION=ap-southeast-2 +kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_release" +``` + +##### Assign mcs-api `ClusterSet` membership and `Cluster` identifier +To ensure that `ServiceExport` and `ServiceImport` objects propagate correctly between clusters, each cluster should be configured as a member of a single mcs-api `ClusterSet` (clusterset1 in our example deployment scenario), and should be assigned a unique mcs-api `Cluster` Id within the `ClusterSet` (cls1 & cls2 in our example deployment scenario). + +- Environment variable CLUSTER_ID should be configured. +- Environment variable CLUSTERSET_ID should be configured. + +Run the following commands to configure Cluster Id and ClusterSet membership. + +Cluster 1: + +```bash +export CLUSTER_ID=cls1 +export CLUSTERSET_ID=clusterset1 +envsubst < samples/mcsapi-clusterproperty.yaml | kubectl apply -f - +``` + +Cluster 2: + +```bash +export CLUSTER_ID=cls2 +export CLUSTERSET_ID=clusterset1 +envsubst < samples/mcsapi-clusterproperty.yaml | kubectl apply -f - +``` + +#### Create `nginx-hello` Service + +Now that the clusters, CoreDNS and the MCS-Controller have been configured, we can create the `demo` namespace in both clusters and implement the `nginx-hello` Service and associated Deployment into Cluster 1. + +Run the following commands to prepare the demo environment on both clusters. + +> *Note: be sure to change the `kubectl` context to the correct cluster before issuing commands.* + +Cluster 1: + +```bash +kubectl create namespace demo +kubectl apply -f samples/nginx-deployment.yaml +kubectl apply -f samples/nginx-service.yaml +``` + +Cluster 2: + +```bash +kubectl create namespace demo +``` + +### Service Provisioning + +With the Solution Baseline in place, let's continue by implementing the Service Provisioning scenario. We'll create a `ServiceExport` object in Cluster 1 for the `nginx-hello` Service. This will trigger the Cluster 1 MCS-Controller to complete service provisioning and propagation into Cloud Map, and subsequent import and provisioning by the MCS-Controller in Cluster 2. + +#### Create `nginx-hello` ServiceExport + +Run the following command against Cluster 1 to to create the `ServiceExport` object for the `nginx-hello` Service: + +```bash +kubectl apply -f \config\nginx-serviceexport.yaml +``` + +#### Verify `nginx-hello` ServiceExport + +Let's verify the `ServiceExport` creation has succeeded, and that corresponding objects have been created in Cluster 1, Cloud Map, and Cluster 2. + +##### Cluster 1 + +Inspecting the MCS-Controller logs in Cluster 1, we see that the controller has detected the `ServiceExport` object, and created the corresponding `demo` Namespace and `nginx-hello` Service in Cloud Map: + +```bash +$ kubectl logs cloud-map-mcs-controller-manager-5b9f959fc9-hmz88 -c manager --namespace cloud-map-mcs-system +{"level":"info","ts":1665108812.7046816,"logger":"cloudmap","msg":"namespace created","nsId":"ns-nlnawwa2wa3ajoh3"} +{"level":"info","ts":1665108812.7626762,"logger":"cloudmap","msg":"service created","namespace":"demo","name":"nginx-hello","id":"srv-xqirlhajwua5vkvo"} +{"level":"info","ts":1665108812.7627065,"logger":"cloudmap","msg":"fetching a service","namespace":"demo","name":"nginx-hello"} +{"level":"info","ts":1665108812.8299918,"logger":"cloudmap","msg":"registering endpoints","namespaceName":"demo","serviceName":"nginx-hello","endpoints":[{"Id":"tcp-10_10_86_76-80","IP":"10.10.86.76","EndpointPort":{"Name":"","Port":80,"TargetPort":"","Protocol":"TCP"},"ServicePort":{"Name":"","Port":80,"TargetPort":"80","Protocol":"TCP"},"ClusterId":"cls1","ClusterSetId":"clusterset1","ServiceType":"ClusterSetIP","ServiceExportCreationTimestamp":1665108776000,"Ready":true,"Hostname":"","Nodename":"ip-10-10-77-143.ap-southeast-2.compute.internal","Attributes":{"K8S_CONTROLLER":"aws-cloud-map-mcs-controller-for-k8s d07e680 (d07e680)"}},{"Id":"tcp-10_10_66_181-80","IP":"10.10.66.181","EndpointPort":{"Name":"","Port":80,"TargetPort":"","Protocol":"TCP"},"ServicePort":{"Name":"","Port":80,"TargetPort":"80","Protocol":"TCP"},"ClusterId":"cls1","ClusterSetId":"clusterset1","ServiceType":"ClusterSetIP","ServiceExportCreationTimestamp":1665108776000,"Ready":true,"Hostname":"","Nodename":"ip-10-10-77-143.ap-southeast-2.compute.internal","Attributes":{"K8S_CONTROLLER":"aws-cloud-map-mcs-controller-for-k8s d07e680 (d07e680)"}},{"Id":"tcp-10_10_78_125-80","IP":"10.10.78.125","EndpointPort":{"Name":"","Port":80,"TargetPort":"","Protocol":"TCP"},"ServicePort":{"Name":"","Port":80,"TargetPort":"80","Protocol":"TCP"},"ClusterId":"cls1","ClusterSetId":"clusterset1","ServiceType":"ClusterSetIP","ServiceExportCreationTimestamp":1665108776000,"Ready":true,"Hostname":"","Nodename":"ip-10-10-77-143.ap-southeast-2.compute.internal","Attributes":{"K8S_CONTROLLER":"aws-cloud-map-mcs-controller-for-k8s d07e680 (d07e680)"}}]} +``` + +Using the AWS CLI we can verify Namespace and Service resources provisioned to Cloud Map by the Cluster 1 MCS-Controller: + +```bash +$ aws servicediscovery list-namespaces +{ + "Namespaces": [ + { + "Id": "ns-nlnawwa2wa3ajoh3", + "Arn": "arn:aws:servicediscovery:ap-southeast-2:911483634971:namespace/ns-nlnawwa2wa3ajoh3", + "Name": "demo", + "Type": "HTTP", + "Properties": { + "DnsProperties": { + "SOA": {} + }, + "HttpProperties": { + "HttpName": "demo" + } + }, + "CreateDate": "2022-10-07T02:13:32.310000+00:00" + } + ] +} +$ aws servicediscovery list-services +{ + "Services": [ + { + "Id": "srv-xqirlhajwua5vkvo", + "Arn": "arn:aws:servicediscovery:ap-southeast-2:911483634971:service/srv-xqirlhajwua5vkvo", + "Name": "nginx-hello", + "Type": "HTTP", + "DnsConfig": {}, + "CreateDate": "2022-10-07T02:13:32.744000+00:00" + } + ] +} +$ aws servicediscovery discover-instances --namespace-name demo --service-name nginx-hello +{ + "Instances": [ + { + "InstanceId": "tcp-10_10_78_125-80", + "NamespaceName": "demo", + "ServiceName": "nginx-hello", + "HealthStatus": "UNKNOWN", + "Attributes": { + "AWS_INSTANCE_IPV4": "10.10.78.125", + "AWS_INSTANCE_PORT": "80", + "CLUSTERSET_ID": "clusterset1", + "CLUSTER_ID": "cls1", + "ENDPOINT_PORT_NAME": "", + "ENDPOINT_PROTOCOL": "TCP", + "HOSTNAME": "", + "K8S_CONTROLLER": "aws-cloud-map-mcs-controller-for-k8s d07e680 (d07e680)", + "NODENAME": "ip-10-10-77-143.ap-southeast-2.compute.internal", + "READY": "true", + "SERVICE_EXPORT_CREATION_TIMESTAMP": "1665108776000", + "SERVICE_PORT": "80", + "SERVICE_PORT_NAME": "", + "SERVICE_PROTOCOL": "TCP", + "SERVICE_TARGET_PORT": "80", + "SERVICE_TYPE": "ClusterSetIP" + } + }, + { + "InstanceId": "tcp-10_10_66_181-80", + "NamespaceName": "demo", + "ServiceName": "nginx-hello", + "HealthStatus": "UNKNOWN", + "Attributes": { + "AWS_INSTANCE_IPV4": "10.10.66.181", + "AWS_INSTANCE_PORT": "80", + "CLUSTERSET_ID": "clusterset1", + "CLUSTER_ID": "cls1", + "ENDPOINT_PORT_NAME": "", + "ENDPOINT_PROTOCOL": "TCP", + "HOSTNAME": "", + "K8S_CONTROLLER": "aws-cloud-map-mcs-controller-for-k8s d07e680 (d07e680)", + "NODENAME": "ip-10-10-77-143.ap-southeast-2.compute.internal", + "READY": "true", + "SERVICE_EXPORT_CREATION_TIMESTAMP": "1665108776000", + "SERVICE_PORT": "80", + "SERVICE_PORT_NAME": "", + "SERVICE_PROTOCOL": "TCP", + "SERVICE_TARGET_PORT": "80", + "SERVICE_TYPE": "ClusterSetIP" + } + }, + { + "InstanceId": "tcp-10_10_86_76-80", + "NamespaceName": "demo", + "ServiceName": "nginx-hello", + "HealthStatus": "UNKNOWN", + "Attributes": { + "AWS_INSTANCE_IPV4": "10.10.86.76", + "AWS_INSTANCE_PORT": "80", + "CLUSTERSET_ID": "clusterset1", + "CLUSTER_ID": "cls1", + "ENDPOINT_PORT_NAME": "", + "ENDPOINT_PROTOCOL": "TCP", + "HOSTNAME": "", + "K8S_CONTROLLER": "aws-cloud-map-mcs-controller-for-k8s d07e680 (d07e680)", + "NODENAME": "ip-10-10-77-143.ap-southeast-2.compute.internal", + "READY": "true", + "SERVICE_EXPORT_CREATION_TIMESTAMP": "1665108776000", + "SERVICE_PORT": "80", + "SERVICE_PORT_NAME": "", + "SERVICE_PROTOCOL": "TCP", + "SERVICE_TARGET_PORT": "80", + "SERVICE_TYPE": "ClusterSetIP" + } + } + ] +} +``` + +##### Cluster 2 + +Inspecting the MCS-Controller logs in Cluster 2, we see that the controller has detected the `nginx-hello` Cloud Map Service, and created the corresponding Kubernetes `ServiceImport`: + +```bash +$ kubectl logs cloud-map-mcs-controller-manager-5b9f959fc9-v72s4 -c manager --namespace cloud-map-mcs-system +{"level":"info","ts":1665108822.2781157,"logger":"controllers.Cloudmap","msg":"created ServiceImport","namespace":"demo","name":"nginx-hello"} +{"level":"info","ts":1665108824.2420218,"logger":"controllers.Cloudmap","msg":"created derived Service","namespace":"demo","name":"imported-9cfu7k5mkr"} +{"level":"info","ts":1665108824.2501283,"logger":"controllers.Cloudmap","msg":"ServiceImport IPs need update","ServiceImport IPs":[],"cluster IPs":["172.20.80.119"]} +{"level":"info","ts":1665108824.2618752,"logger":"controllers.Cloudmap","msg":"updated ServiceImport","namespace":"demo","name":"nginx-hello","IP":["172.20.80.119"],"ports":[{"protocol":"TCP","port":80}]} +``` + +Inspecting the Cluster 2 Kubernetes `ServiceImport` object: + +```bash +$ kubectl get serviceimports.multicluster.x-k8s.io nginx-hello -n demo -o yaml +apiVersion: multicluster.x-k8s.io/v1alpha1 +kind: ServiceImport +metadata: + annotations: + multicluster.k8s.aws/derived-service: '[{"cluster":"cls1","derived-service":"imported-9cfu7k5mkr"}]' + creationTimestamp: "2022-10-07T02:13:42Z" + generation: 2 + name: nginx-hello + namespace: demo + resourceVersion: "12787" + uid: a53901af-57a8-49c7-aeb1-f67c4a44c2d2 +spec: + ips: + - 172.20.80.119 + ports: + - port: 80 + protocol: TCP + type: ClusterSetIP +status: + clusters: + - cluster: cls1 +``` + +And the corresponding Cluster 2 Kubernetes Endpoint Slice: + +```bash +$ kubectl get endpointslices.discovery.k8s.io -n demo +NAME ADDRESSTYPE PORTS ENDPOINTS AGE +imported-9cfu7k5mkr-dc7q9 IPv4 80 10.10.78.125,10.10.86.76,10.10.66.181 14m +``` + +Important points to note: + +- the `ServiceImport` Service is assigned an IP address from the local Kubernetes service IPv4 CIDR: 172.22.0.0/16 (172.20.80.119) so as to permit service discovery and access to the remote service endpoints from within the local cluster. +- the `EndpointSlice` IP addresses match those of the `nginx-demo` Endpoints in Cluster 1 (i.e. from the Cluster 1 VPC CIDR: 10.10.0.0/16). + +### Service Consumption + +With the Solution Baseline and Service Provisioning in place, workloads in Cluster 2 are now able to consume the nginx-hello Service Endpoints located in Cluster 1 via the locally provisioned ServiceImport object. To complete the Service Consumption scenario we'll deploy the client-hello Pod into Cluster 2, and observe how it's able to perform cross-cluster service discovery, and access each of the nginx-hello Service Endpoints in Cluster 1. + +#### Create `client-hello` Pod + +Run the following command against Cluster 2 create the `client-hello` Pod: + +```bash +kubectl apply -f samples/client-hello.yaml +``` + +#### Verify multi-cluster service consumption + +Let's exec into the `client-hello` Pod and perform an `nslookup` to cluster-local CoreDNS for the `ServiceImport` Service `nginx-hello.demo.svc.clusterset.local`: + +```bash +$ kubectl exec -it client-hello -n demo /bin/sh +/ # nslookup nginx-hello.demo.svc.clusterset.local +Server: 172.20.0.10 +Address: 172.20.0.10:53 + +Name: nginx-hello.demo.svc.clusterset.local +Address: 172.20.80.119 +``` + +Note that the Pod resolves the address of the `ServiceImport` object on Cluster 2. + +Finally, we generate HTTP requests from the `client-hello` Pod to the local `nginx-hello` `ServiceImport` Service: + +```bash +/ # apk --no-cache add curl +/ # curl nginx-hello.demo.svc.clusterset.local +Server address: 10.10.86.76:80 +Server name: nginx-demo-59c6cb8d7b-m4ktw +Date: 07/Oct/2022:02:31:45 +0000 +URI: / +Request ID: 17d43e6e8801a98d05059dfaf88d0abe +/ # +/ # curl nginx-hello.demo.svc.clusterset.local +Server address: 10.10.78.125:80 +Server name: nginx-demo-59c6cb8d7b-8w6rp +Date: 07/Oct/2022:02:32:26 +0000 +URI: / +Request ID: 0ddc09ffe7fd45c52903ce34c955f555 +/ # +/ # curl nginx-hello.demo.svc.clusterset.local +Server address: 10.10.66.181:80 +Server name: nginx-demo-59c6cb8d7b-mtm8l +Date: 07/Oct/2022:02:32:53 +0000 +URI: / +Request ID: 2fde1c34008a5ec18b8ae23797489c3a +``` + +Note that the responding Server Names and Server addresses are those of the `nginx-demo` Pods on Cluster 1 - confirming that the requests to the local `ClusterSetIP` at `nginx-hello.demo.svc.clusterset.local` originating on Cluster 2 are proxied cross-cluster to the Endpoints located on Cluster 1! + +## Conclusion + +The proliferation of container adoption is presenting new challenges in supporting workloads that have broken through the perimeter of the single cluster construct. + +For teams that are looking to implement a Kubenetes-centric approach to managing multi-cluster workloads, the mcs-api describes an effective approach to extending the scope of the service resource concept beyond the cluster boundary - providing a mechanism to weave multiple clusters together using standard (and familiar) DNS based service discovery. + +The [AWS Cloud Map MCS Controller for Kubernetes](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s) is an open source project that integrates with AWS Cloud Map to offer a decentralised implementation of the multi-cluster services API specification that's particularly suited for teams looking for a lightweight and effective Kubenetes-centric mechanism to deploy multi-cluster workloads to the AWS cloud. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..20aae806 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,12 @@ +site_name: AWS Cloud Map MCS Controller +repo_name: aws/aws-cloud-map-mcs-controller-for-k8s +repo_url: https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s + +plugins: + - search +theme: + name: material + language: en + favicon: images/cloudmap.svg + logo: images/cloudmap.svg + diff --git a/samples/client-hello.yaml b/samples/client-hello.yaml new file mode 100644 index 00000000..ef168a6d --- /dev/null +++ b/samples/client-hello.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: client-hello + namespace: demo +spec: + containers: + - command: + - sleep + - "1d" + image: alpine + name: client-hello \ No newline at end of file diff --git a/samples/eksctl-cluster.yaml b/samples/eksctl-cluster.yaml new file mode 100644 index 00000000..31135473 --- /dev/null +++ b/samples/eksctl-cluster.yaml @@ -0,0 +1,18 @@ +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: $CLUSTER_NAME + region: $AWS_REGION + version: "1.21" +vpc: + cidr: $VPC_CIDR + autoAllocateIPv6: false + clusterEndpoints: + publicAccess: true + privateAccess: true +managedNodeGroups: +- name: $NODEGROUP_NAME + instanceType: t2.small + minSize: 1 + maxSize: 10 + desiredCapacity: 1 \ No newline at end of file diff --git a/samples/mcsapi-clusterproperty.yaml b/samples/mcsapi-clusterproperty.yaml new file mode 100644 index 00000000..ac39ce8e --- /dev/null +++ b/samples/mcsapi-clusterproperty.yaml @@ -0,0 +1,13 @@ +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: cluster.clusterset.k8s.io +spec: + value: ${CLUSTER_ID} +--- +apiVersion: about.k8s.io/v1alpha1 +kind: ClusterProperty +metadata: + name: clusterset.k8s.io +spec: + value: ${CLUSTERSET_ID} \ No newline at end of file diff --git a/samples/nginx-deployment.yaml b/samples/nginx-deployment.yaml new file mode 100644 index 00000000..7218a551 --- /dev/null +++ b/samples/nginx-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: demo + name: nginx-demo + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginxdemos/hello:plain-text + ports: + - containerPort: 80 \ No newline at end of file diff --git a/samples/nginx-service.yaml b/samples/nginx-service.yaml new file mode 100644 index 00000000..0d2c1508 --- /dev/null +++ b/samples/nginx-service.yaml @@ -0,0 +1,10 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: demo + name: nginx-hello +spec: + selector: + app: nginx + ports: + - port: 80 \ No newline at end of file diff --git a/samples/nginx-serviceexport.yaml b/samples/nginx-serviceexport.yaml new file mode 100644 index 00000000..da54514f --- /dev/null +++ b/samples/nginx-serviceexport.yaml @@ -0,0 +1,5 @@ +kind: ServiceExport +apiVersion: multicluster.x-k8s.io/v1alpha1 +metadata: + namespace: demo + name: nginx-hello \ No newline at end of file From dac99e382ffd7f270057d9ed1b5c927f46a6867c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:03:43 -0700 Subject: [PATCH 120/163] Bump github.com/aws/aws-sdk-go-v2 from 1.16.16 to 1.17.1 (#237) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.16.16 to 1.17.1. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.16.16...v1.17.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8227258d..2bd154b0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.16.16 + github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.17.4 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 github.com/go-logr/logr v1.2.3 @@ -38,7 +38,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 // indirect - github.com/aws/smithy-go v1.13.3 // indirect + github.com/aws/smithy-go v1.13.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 48fe9c5b..193ebb6c 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,9 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.16.13/go.mod h1:xSyvSnzh0KLs5H4HJGeIEsNYemUWdNIl0b/rP6SIsLU= -github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= +github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= +github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2/config v1.17.4 h1:9HY1wbShqObySCHP2Z07blfrSWVX+nVxCZmUuLZKcG8= github.com/aws/aws-sdk-go-v2/config v1.17.4/go.mod h1:ul+ru+huVpfduF9XRmGUq82T8T3K+nIFQuF6F+L+548= github.com/aws/aws-sdk-go-v2/credentials v1.12.17 h1:htUjIJOQcvIUR0jC4eLkdis1DfaLL4EUbIKUFqh2WFA= @@ -102,8 +103,9 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2/go.mod h1:5cxfDYtY2mDOlmesy github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 h1:otZvq9r+xjPL7qU/luX2QdBamiN+oSZURRi4sAKymO8= github.com/aws/aws-sdk-go-v2/service/sts v1.16.16/go.mod h1:Y9iBgT1w2vHtYzJEkwD6FqILjDSsvbxcW/+wIYxyse4= github.com/aws/smithy-go v1.13.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= +github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 0685ed739204cb30a0f460dfa54585b2de7585ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:22:51 -0700 Subject: [PATCH 121/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.17.4 to 1.17.10 (#238) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.17.4 to 1.17.10. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.17.4...config/v1.17.10) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 20 ++++++++++---------- go.sum | 42 ++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 2bd154b0..c1395dbc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.1 - github.com/aws/aws-sdk-go-v2/config v1.17.4 + github.com/aws/aws-sdk-go-v2/config v1.17.10 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -29,15 +29,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.17 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.23 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 // indirect github.com/aws/smithy-go v1.13.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 193ebb6c..a76296e9 100644 --- a/go.sum +++ b/go.sum @@ -74,35 +74,33 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.13/go.mod h1:xSyvSnzh0KLs5H4HJGeIEsNYemUWdNIl0b/rP6SIsLU= github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= -github.com/aws/aws-sdk-go-v2/config v1.17.4 h1:9HY1wbShqObySCHP2Z07blfrSWVX+nVxCZmUuLZKcG8= -github.com/aws/aws-sdk-go-v2/config v1.17.4/go.mod h1:ul+ru+huVpfduF9XRmGUq82T8T3K+nIFQuF6F+L+548= -github.com/aws/aws-sdk-go-v2/credentials v1.12.17 h1:htUjIJOQcvIUR0jC4eLkdis1DfaLL4EUbIKUFqh2WFA= -github.com/aws/aws-sdk-go-v2/credentials v1.12.17/go.mod h1:jd1mvJulXY7ccHvcSiJceYhv06yWIIRkJnwWEA4IX+g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14 h1:NZwZFtxXGOEIiCd8jWN55lexakug543CaO68bTpoLwg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14/go.mod h1:5CU57SyF5jZLSIw4OOll0PG83ThXwNdkRFOc0EltD/0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20/go.mod h1:gdZ5gRUaxThXIZyZQ8MTtgYBk2jbHgp05BO3GcD9Cwc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= +github.com/aws/aws-sdk-go-v2/config v1.17.10 h1:zBy5QQ/mkvHElM1rygHPAzuH+sl8nsdSaxSWj0+rpdE= +github.com/aws/aws-sdk-go-v2/config v1.17.10/go.mod h1:/4np+UiJJKpWHN7Q+LZvqXYgyjgeXm5+lLfDI6TPZao= +github.com/aws/aws-sdk-go-v2/credentials v1.12.23 h1:LctvcJMIb8pxvk5hQhChpCu0WlU6oKQmcYb1HA4IZSA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.23/go.mod h1:0awX9iRr/+UO7OwRQFpV1hNtXxOVuehpjVEzrIAYNcA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14/go.mod h1:GEV9jaDPIgayiU+uevxwozcvUOjc+P4aHE2BeSjm2vE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21 h1:lpwSbLKYTuABo6SyUoC25xAmfO3/TehGS2SmD1EtOL0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21/go.mod h1:Q0pktZjvRZk77TBto6yAvUAi7fcse1bdcMctBDVGgBw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14 h1:c5hJNN2DkK1gAytcKp7LkiKNDJeevFSboPezEHAM4Ro= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14/go.mod h1:8qOLjqMzY/S1kh3myDXA1yxK5eD4uN8aOJgKpgvc4OM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 h1:5/CGPA7fMLZwdXJBpx4lkueY92pOqI5B+qc/iWki7xI= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18/go.mod h1:v+R9xFTquOHSVwn36gO9MOwgbINQNh2m0/Y14CsB1bI= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.20 h1:3raP0UC9rvRyY4/cc4o4F3jTrNo94AYiarNUGNnq6dU= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.20/go.mod h1:hPsROgDdgY/NQ1gPt7VJWG0GjSnalDC0DkkMfGEw2gc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2 h1:/SYpdjjAtraymql+/r719OgjxezdanAQiLb/NMxDb04= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2/go.mod h1:5cxfDYtY2mDOlmesy4yycb6lwyy1U/iAUOHKhQLKw/E= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.16 h1:otZvq9r+xjPL7qU/luX2QdBamiN+oSZURRi4sAKymO8= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.16/go.mod h1:Y9iBgT1w2vHtYzJEkwD6FqILjDSsvbxcW/+wIYxyse4= -github.com/aws/smithy-go v1.13.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 h1:KRAix/KHvjGODaHAMXnxRk9t0D+4IJVUuS/uwXxngXk= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.1/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= From 4eec058294ece95d5e9160d75deadfcf385b22f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:38:58 -0700 Subject: [PATCH 122/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#235) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.17.18 to 1.18.2. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.17.18...service/ecs/v1.18.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index c1395dbc..5803ff16 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.17.10 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 diff --git a/go.sum b/go.sum index a76296e9..bfe8effc 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2/config v1.17.10 h1:zBy5QQ/mkvHElM1rygHPAzuH+sl8nsdSaxSWj0+rpdE= @@ -83,25 +82,22 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.23 h1:LctvcJMIb8pxvk5hQhChpCu0WlU github.com/aws/aws-sdk-go-v2/credentials v1.12.23/go.mod h1:0awX9iRr/+UO7OwRQFpV1hNtXxOVuehpjVEzrIAYNcA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18 h1:5/CGPA7fMLZwdXJBpx4lkueY92pOqI5B+qc/iWki7xI= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.17.18/go.mod h1:v+R9xFTquOHSVwn36gO9MOwgbINQNh2m0/Y14CsB1bI= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2 h1:TBmvwADPIuBUEhWPrLsrE0VEchnSgM/wzL33MiuPjhs= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2/go.mod h1:/J9zJYj/yOhbRTj9EawVe32mkrrFcxCDo5vr5OulEMI= github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 h1:KRAix/KHvjGODaHAMXnxRk9t0D+4IJVUuS/uwXxngXk= github.com/aws/aws-sdk-go-v2/service/sts v1.17.1/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= -github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From 3f1bd63e77c2dae56545c218e0bdc7832ae241aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:39:52 -0700 Subject: [PATCH 123/163] Bump github.com/stretchr/testify from 1.8.0 to 1.8.1 (#236) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.0 to 1.8.1. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.0...v1.8.1) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5803ff16..0bbe01ca 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.20.2 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.2 diff --git a/go.sum b/go.sum index bfe8effc..1c38f71f 100644 --- a/go.sum +++ b/go.sum @@ -490,6 +490,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -497,8 +498,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= From c9932b9f650a4f37fef9c0a19bb3d81c1c91d131 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Fri, 4 Nov 2022 12:18:57 -0700 Subject: [PATCH 124/163] Update README.md (#241) Added Graduation Criteria section --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aa516ab4..7bbcf54e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,9 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/aws/aws-cloud-map-mcs-controller-for-k8s)](https://goreportcard.com/report/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) ## Introduction -The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. +The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. The current version `v0.3.0` is in *Alpha* phase, checkout the [Graduation Criteria](#graduation-criteria) for the next steps. + +We also have detailed [step-by-step setup guide](https://aws.github.io/aws-cloud-map-mcs-controller-for-k8s/)! ## Installation @@ -163,6 +165,16 @@ To install from `latest` tag run the following command. kubectl apply -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/config/controller_install_latest" ``` +## Graduation Criteria + +### Alpha -> Beta Graduation +* Implement the [resiliency milestone](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/milestone/3). + +### Beta -> GA Graduation +* Scalability/performance testing. +* Implement the [observability and deployment milestone](https://github.com/aws/aws-cloud-map-mcs-controller-for-k8s/milestone/6). +* [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid) are Beta or GA. + ## Slack community We have an open Slack community where users may get support with integration, discuss controller functionality and provide input on our feature roadmap. https://awsappmesh.slack.com/#k8s-mcs-controller Join the channel with this [invite](https://join.slack.com/t/awsappmesh/shared_invite/zt-dwgbt85c-Sj_md92__quV8YADKfsQSA). From 2db7055e41bc8b3b48425cb530a8ad6ee88277d6 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Mon, 7 Nov 2022 13:45:17 -0800 Subject: [PATCH 125/163] Add rate limiter to the Cloudmap API calls. Increasing the base delay of the service export reconciler's rate limiter. (#243) --- Makefile | 4 +-- go.mod | 2 +- go.sum | 3 +- main.go | 8 ++--- .../v1alpha1/zz_generated.deepcopy.go | 2 +- pkg/cloudmap/api.go | 32 ++++++++++++++++--- pkg/cloudmap/api_test.go | 10 ++++-- .../multicluster/serviceexport_controller.go | 12 ++++++- 8 files changed, 57 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index a9c1c127..2e8d5d65 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)" .PHONY: test -test: manifests generate generate-mocks fmt vet test-setup ## Run tests +test: manifests generate generate-mocks fmt vet test-setup goimports lint ## Run tests KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out -covermode=atomic test-setup: setup-envtest ## Ensure test environment has been downloaded @@ -117,7 +117,7 @@ eks-test: ##@ Build .DEFAULT: build -build: test goimports lint ## Build manager binary. +build: test ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go run: test ## Run a controller from your host. diff --git a/go.mod b/go.mod index 0bbe01ca..900d7e33 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/onsi/gomega v1.20.2 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.1 + golang.org/x/time v0.1.0 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.2 @@ -80,7 +81,6 @@ require ( golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index 1c38f71f..048f458b 100644 --- a/go.sum +++ b/go.sum @@ -774,8 +774,9 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go index 2eebe4d4..0a075630 100644 --- a/main.go +++ b/main.go @@ -92,24 +92,24 @@ func main() { if err = (&multiclustercontrollers.ServiceExportReconciler{ Client: mgr.GetClient(), - Log: common.NewLogger("controllers", "ServiceExport"), + Log: common.NewLogger("controllers", "ServiceExportReconciler"), Scheme: mgr.GetScheme(), CloudMap: serviceDiscoveryClient, ClusterUtils: clusterUtils, }).SetupWithManager(mgr); err != nil { - log.Error(err, "unable to create controller", "controller", "ServiceExport") + log.Error(err, "unable to create controller", "controller", "ServiceExportReconciler") os.Exit(1) } cloudMapReconciler := &multiclustercontrollers.CloudMapReconciler{ Client: mgr.GetClient(), Cloudmap: serviceDiscoveryClient, - Log: common.NewLogger("controllers", "Cloudmap"), + Log: common.NewLogger("controllers", "CloudmapReconciler"), ClusterUtils: clusterUtils, } if err = mgr.Add(cloudMapReconciler); err != nil { - log.Error(err, "unable to create controller", "controller", "CloudMap") + log.Error(err, "unable to create controller", "controller", "CloudmapReconciler") os.Exit(1) } diff --git a/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go index f74564df..1eca515f 100644 --- a/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/multicluster/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index 977aa2bb..04e169af 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "time" + + "golang.org/x/time/rate" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" @@ -52,19 +55,30 @@ type ServiceDiscoveryApi interface { } type serviceDiscoveryApi struct { - log common.Logger - awsFacade AwsFacade + log common.Logger + awsFacade AwsFacade + nsRateLimiter *rate.Limiter + svcRateLimiter *rate.Limiter + opRateLimiter *rate.Limiter } // NewServiceDiscoveryApiFromConfig creates a new AWS Cloud Map API connection manager from an AWS client config. func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { return &serviceDiscoveryApi{ - log: common.NewLogger("cloudmap"), - awsFacade: NewAwsFacadeFromConfig(cfg), + log: common.NewLogger("cloudmap", "api"), + awsFacade: NewAwsFacadeFromConfig(cfg), + nsRateLimiter: rate.NewLimiter(rate.Every(1*time.Second), 5), // 1 per second + svcRateLimiter: rate.NewLimiter(rate.Every(2*time.Second), 10), // 2 per second + opRateLimiter: rate.NewLimiter(rate.Every(100*time.Second), 200), // 100 per second } } func (sdApi *serviceDiscoveryApi) GetNamespaceMap(ctx context.Context) (map[string]*model.Namespace, error) { + err := sdApi.nsRateLimiter.Wait(ctx) + if err != nil { + return nil, err + } + namespaceMap := make(map[string]*model.Namespace) pages := sd.NewListNamespacesPaginator(sdApi.awsFacade, &sd.ListNamespacesInput{}) @@ -91,6 +105,11 @@ func (sdApi *serviceDiscoveryApi) GetNamespaceMap(ctx context.Context) (map[stri } func (sdApi *serviceDiscoveryApi) GetServiceIdMap(ctx context.Context, nsId string) (map[string]string, error) { + err := sdApi.svcRateLimiter.Wait(ctx) + if err != nil { + return nil, err + } + serviceIdMap := make(map[string]string) filter := types.ServiceFilter{ @@ -155,6 +174,11 @@ func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters } func (sdApi *serviceDiscoveryApi) GetOperation(ctx context.Context, opId string) (operation *types.Operation, err error) { + err = sdApi.opRateLimiter.Wait(ctx) + if err != nil { + return nil, err + } + opResp, err := sdApi.awsFacade.GetOperation(ctx, &sd.GetOperationInput{OperationId: &opId}) if err != nil { diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index a898374d..d2b210f5 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -5,6 +5,9 @@ import ( "errors" "fmt" "testing" + "time" + + "golang.org/x/time/rate" aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" @@ -336,7 +339,10 @@ func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmapMock.MockAwsFacade) scheme := runtime.NewScheme() scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) return &serviceDiscoveryApi{ - log: common.NewLoggerWithLogr(testr.New(t)), - awsFacade: awsFacade, + log: common.NewLoggerWithLogr(testr.New(t)), + awsFacade: awsFacade, + nsRateLimiter: rate.NewLimiter(rate.Every(1*time.Second), 2), // 1 per second + svcRateLimiter: rate.NewLimiter(rate.Every(2*time.Second), 4), // 2 per second + opRateLimiter: rate.NewLimiter(rate.Every(10*time.Second), 100), // 10 per second } } diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index 5bea631a..f49a5878 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -3,6 +3,9 @@ package controllers import ( "context" "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" @@ -12,9 +15,11 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" - "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -295,6 +300,11 @@ func (r *ServiceExportReconciler) SetupWithManager(mgr ctrl.Manager) error { &source.Kind{Type: &aboutv1alpha1.ClusterProperty{}}, handler.EnqueueRequestsFromMapFunc(r.clusterPropertyMappingFunction()), ). + WithOptions(controller.Options{ + // rate-limiting is applied to reconcile responses with an error + // We are increasing the base delay to 500ms, defaults baseDelay: 5ms, maxDelay: 1000s + RateLimiter: workqueue.NewItemExponentialFailureRateLimiter(500*time.Millisecond, 1000*time.Second), + }). Complete(r) } From 2ba9e046a3e0f32fb0ca44d87df65df7d0a81fc7 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Mon, 14 Nov 2022 10:23:48 -0800 Subject: [PATCH 126/163] Explicit error handling by introducing NotFound error types, and replacing nil with NotFound with checks in the caller where-ever required. (#244) --- .github/.codecov.yml | 1 + Makefile | 2 +- integration/janitor/janitor.go | 3 +- integration/janitor/janitor_test.go | 2 +- integration/kind-test/scripts/run-tests.sh | 2 +- .../shared/scenarios/export_service.go | 8 +- pkg/cloudmap/api.go | 6 +- pkg/cloudmap/api_test.go | 2 +- pkg/cloudmap/client.go | 88 +++++++++++-------- pkg/cloudmap/client_test.go | 47 +++++++++- pkg/common/errors.go | 32 +++++++ pkg/common/errors_test.go | 88 +++++++++++++++++++ .../multicluster/cloudmap_controller.go | 33 +++---- .../multicluster/cloudmap_controller_test.go | 15 +++- .../multicluster/controllers_common_test.go | 9 ++ .../multicluster/serviceexport_controller.go | 6 +- .../serviceexport_controller_test.go | 3 +- 17 files changed, 275 insertions(+), 72 deletions(-) create mode 100644 pkg/common/errors.go create mode 100644 pkg/common/errors_test.go diff --git a/.github/.codecov.yml b/.github/.codecov.yml index fbd4614c..ef9ddeb5 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -26,3 +26,4 @@ ignore: - "mocks/**/*" - "integration/shared/scenarios/**/*" - "pkg/common/logger.go" + - "test/*" diff --git a/Makefile b/Makefile index 2e8d5d65..1d1a0e44 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ build: test ## Build manager binary. go build -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" -o bin/manager main.go run: test ## Run a controller from your host. - go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go --zap-devel=true $(ARGS) + go run -ldflags="-s -w -X ${PKG}.GitVersion=${GIT_TAG} -X ${PKG}.GitCommit=${GIT_COMMIT}" ./main.go --zap-devel=true --zap-time-encoding=rfc3339 $(ARGS) docker-build: test ## Build docker image with the manager. docker build --no-cache -t ${IMG} . diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index 73c4b937..0a05fcfc 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -6,6 +6,7 @@ import ( "os" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -81,7 +82,7 @@ func (j *cloudMapJanitor) deregisterInstances(ctx context.Context, nsName string model.ClusterSetIdAttr: j.clusterSetId, } - insts, err := j.sdApi.DiscoverInstances(ctx, nsName, svcName, &queryParameters) + insts, err := j.sdApi.DiscoverInstances(ctx, nsName, svcName, queryParameters) j.checkOrFail(err, fmt.Sprintf("service has %d instances to clean", len(insts)), "could not list instances to cleanup") diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index cd43bd6e..6f1d8c99 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -32,7 +32,7 @@ func TestCleanupHappyCase(t *testing.T) { Return(map[string]*model.Namespace{test.HttpNsName: test.GetTestHttpNamespace()}, nil) tj.mockApi.EXPECT().GetServiceIdMap(context.TODO(), test.HttpNsId). Return(map[string]string{test.SvcName: test.SvcId}, nil) - tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + tj.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, map[string]string{ model.ClusterSetIdAttr: test.ClusterSet, }). Return([]types.HttpInstanceSummary{{InstanceId: aws.String(test.EndptId1)}}, nil) diff --git a/integration/kind-test/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh index 99eccd59..1c5d1b13 100755 --- a/integration/kind-test/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -20,7 +20,7 @@ if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT fi mkdir -p "$LOGS" -./bin/manager &> "$LOGS/ctl.log" & +./bin/manager --zap-devel=true --zap-time-encoding=rfc3339 &> "$LOGS/ctl.log" & CTL_PID=$! echo "controller PID:$CTL_PID" diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index c1c60871..df609b69 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -7,6 +7,8 @@ import ( "strings" "time" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" @@ -86,16 +88,16 @@ func (e *exportServiceScenario) Run() error { return wait.Poll(defaultScenarioPollInterval, defaultScenarioPollTimeout, func() (done bool, err error) { fmt.Println("Polling service...") cmSvc, err := e.sdClient.GetService(context.TODO(), e.expectedSvc.Namespace, e.expectedSvc.Name) - if err != nil { + if common.IsUnknown(err) { return true, err } - if cmSvc == nil { + if common.IsNotFound(err) { fmt.Println("Service not found.") return false, nil } - fmt.Printf("Found service: %v\n", cmSvc) + fmt.Printf("Found service: %+v\n", cmSvc) return e.compareEndpoints(cmSvc.Endpoints), nil }) } diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index 04e169af..0342b5ba 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -30,7 +30,7 @@ type ServiceDiscoveryApi interface { GetServiceIdMap(ctx context.Context, namespaceId string) (serviceIdMap map[string]string, err error) // DiscoverInstances returns a list of service instances registered to a given service. - DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters *map[string]string) (insts []types.HttpInstanceSummary, err error) + DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters map[string]string) (insts []types.HttpInstanceSummary, err error) // ListOperations returns a map of operations to their status matching a list of filters. ListOperations(ctx context.Context, opFilters []types.OperationFilter) (operationStatusMap map[string]types.OperationStatus, err error) @@ -132,7 +132,7 @@ func (sdApi *serviceDiscoveryApi) GetServiceIdMap(ctx context.Context, nsId stri return serviceIdMap, nil } -func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters *map[string]string) (insts []types.HttpInstanceSummary, err error) { +func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters map[string]string) (insts []types.HttpInstanceSummary, err error) { input := &sd.DiscoverInstancesInput{ NamespaceName: aws.String(nsName), ServiceName: aws.String(svcName), @@ -140,7 +140,7 @@ func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName MaxResults: aws.Int32(1000), } if queryParameters != nil { - input.QueryParameters = *queryParameters + input.QueryParameters = queryParameters } out, err := sdApi.awsFacade.DiscoverInstances(ctx, input) diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index d2b210f5..70143ceb 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -124,7 +124,7 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { }, }, nil) - insts, err := sdApi.DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{model.ClusterSetIdAttr: test.ClusterSet}) + insts, err := sdApi.DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, map[string]string{model.ClusterSetIdAttr: test.ClusterSet}) assert.Nil(t, err, "No error for happy case") assert.True(t, len(insts) == 2) assert.Equal(t, test.EndptId1, *insts[0].InstanceId) diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index 19ed6ba2..afe1cde8 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -40,7 +40,7 @@ type serviceDiscoveryClient struct { // from a given AWS client config. func NewDefaultServiceDiscoveryClient(cfg *aws.Config, clusterUtils model.ClusterUtils) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: common.NewLogger("cloudmap"), + log: common.NewLogger("cloudmap", "client"), sdApi: NewServiceDiscoveryApiFromConfig(cfg), cache: NewDefaultServiceDiscoveryClientCache(), clusterUtils: clusterUtils, @@ -49,7 +49,7 @@ func NewDefaultServiceDiscoveryClient(cfg *aws.Config, clusterUtils model.Cluste func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCacheConfig, clusterUtils model.ClusterUtils) ServiceDiscoveryClient { return &serviceDiscoveryClient{ - log: common.NewLogger("cloudmap"), + log: common.NewLogger("cloudmap", "client"), sdApi: NewServiceDiscoveryApiFromConfig(cfg), cache: NewServiceDiscoveryClientCache(cacheConfig), clusterUtils: clusterUtils, @@ -59,6 +59,10 @@ func NewServiceDiscoveryClientWithCustomCache(cfg *aws.Config, cacheConfig *SdCa func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName string) (svcs []*model.Service, err error) { svcIdMap, err := sdc.getServiceIds(ctx, nsName) if err != nil { + // Ignore resource not found error, as it will indicate deleted resources in CloudMap + if common.IsNotFound(err) { + return svcs, nil + } return svcs, err } @@ -81,13 +85,13 @@ func (sdc *serviceDiscoveryClient) ListServices(ctx context.Context, nsName stri func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName string, svcName string) error { sdc.log.Info("creating a new service", "namespace", nsName, "name", svcName) - nsMap, err := sdc.getNamespaces(ctx) - if err != nil { + namespace, err := sdc.getNamespace(ctx, nsName) + if common.IsUnknown(err) { return err } - namespace := nsMap[nsName] - if namespace == nil { + if common.IsNotFound(err) { + sdc.log.Info("namespace not found for service", "namespace", nsName, "service", svcName) // Create HttpNamespace if the namespace is not present in CloudMap namespace, err = sdc.createNamespace(ctx, nsName) if err != nil { @@ -107,9 +111,7 @@ func (sdc *serviceDiscoveryClient) CreateService(ctx context.Context, nsName str func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string, svcName string) (svc *model.Service, err error) { sdc.log.Info("fetching a service", "namespace", nsName, "name", svcName) - endpts, cacheHit := sdc.cache.GetEndpoints(nsName, svcName) - - if cacheHit { + if endpts, found := sdc.cache.GetEndpoints(nsName, svcName); found { return &model.Service{ Namespace: nsName, Name: svcName, @@ -117,17 +119,12 @@ func (sdc *serviceDiscoveryClient) GetService(ctx context.Context, nsName string }, nil } - svcIdMap, err := sdc.getServiceIds(ctx, nsName) + _, err = sdc.getServiceId(ctx, nsName, svcName) if err != nil { return nil, err } - _, found := svcIdMap[svcName] - if !found { - return nil, nil - } - - endpts, err = sdc.getEndpoints(ctx, nsName, svcName) + endpts, err := sdc.getEndpoints(ctx, nsName, svcName) if err != nil { return nil, err } @@ -147,14 +144,10 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName sdc.log.Info("registering endpoints", "namespaceName", nsName, "serviceName", svcName, "endpoints", endpts) - svcIdMap, err := sdc.getServiceIds(ctx, nsName) + svcId, err := sdc.getServiceId(ctx, nsName, svcName) if err != nil { return err } - svcId, found := svcIdMap[svcName] - if !found { - return fmt.Errorf("service not found in Cloud Map: %s", svcName) - } opCollector := NewOperationCollector() @@ -190,14 +183,10 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s sdc.log.Info("deleting endpoints", "namespaceName", nsName, "serviceName", svcName, "endpoints", endpts) - svcIdMap, err := sdc.getServiceIds(ctx, nsName) + svcId, err := sdc.getServiceId(ctx, nsName, svcName) if err != nil { return err } - svcId, found := svcIdMap[svcName] - if !found { - return fmt.Errorf("service not found in Cloud Map: %s", svcName) - } opCollector := NewOperationCollector() @@ -225,8 +214,8 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s } func (sdc *serviceDiscoveryClient) getEndpoints(ctx context.Context, nsName string, svcName string) (endpts []*model.Endpoint, err error) { - endpts, cacheHit := sdc.cache.GetEndpoints(nsName, svcName) - if cacheHit { + endpts, found := sdc.cache.GetEndpoints(nsName, svcName) + if found { return endpts, nil } @@ -239,7 +228,7 @@ func (sdc *serviceDiscoveryClient) getEndpoints(ctx context.Context, nsName stri queryParameters := map[string]string{ model.ClusterSetIdAttr: clusterProperties.ClusterSetId(), } - insts, err := sdc.sdApi.DiscoverInstances(ctx, nsName, svcName, &queryParameters) + insts, err := sdc.sdApi.DiscoverInstances(ctx, nsName, svcName, queryParameters) if err != nil { return nil, err } @@ -257,10 +246,23 @@ func (sdc *serviceDiscoveryClient) getEndpoints(ctx context.Context, nsName stri return endpts, nil } -func (sdc *serviceDiscoveryClient) getNamespaces(ctx context.Context) (namespace map[string]*model.Namespace, err error) { +func (sdc *serviceDiscoveryClient) getNamespace(ctx context.Context, nsName string) (namespace *model.Namespace, err error) { + namespaces, err := sdc.getNamespaces(ctx) + if err != nil { + return nil, err + } + + if namespace, ok := namespaces[nsName]; ok { + return namespace, nil + } + + return nil, common.NotFoundError(fmt.Sprintf("namespace: %s", nsName)) +} + +func (sdc *serviceDiscoveryClient) getNamespaces(ctx context.Context) (namespaces map[string]*model.Namespace, err error) { // We are assuming a unique namespace name per account - namespaces, cacheHit := sdc.cache.GetNamespaceMap() - if cacheHit { + namespaces, found := sdc.cache.GetNamespaceMap() + if found { return namespaces, nil } @@ -273,15 +275,27 @@ func (sdc *serviceDiscoveryClient) getNamespaces(ctx context.Context) (namespace return namespaces, nil } +func (sdc *serviceDiscoveryClient) getServiceId(ctx context.Context, nsName string, svcName string) (svcId string, err error) { + svcIdMap, err := sdc.getServiceIds(ctx, nsName) + if err != nil { + return "", err + } + + if svcId, ok := svcIdMap[svcName]; ok { + return svcId, nil + } + + return "", common.NotFoundError(fmt.Sprintf("service: %s", svcName)) +} + func (sdc *serviceDiscoveryClient) getServiceIds(ctx context.Context, nsName string) (map[string]string, error) { - serviceIdMap, cacheHit := sdc.cache.GetServiceIdMap(nsName) - if cacheHit { + serviceIdMap, found := sdc.cache.GetServiceIdMap(nsName) + if found { return serviceIdMap, nil } - nsMap, err := sdc.getNamespaces(ctx) - namespace := nsMap[nsName] - if err != nil || namespace == nil { + namespace, err := sdc.getNamespace(ctx, nsName) + if err != nil { return nil, err } diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 45b831c7..84da49fd 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -48,7 +48,7 @@ func TestServiceDiscoveryClient_ListServices_HappyCase(t *testing.T) { tc.mockCache.EXPECT().CacheServiceIdMap(test.HttpNsName, getServiceIdMapForTest()) tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, map[string]string{ model.ClusterSetIdAttr: test.ClusterSet, }).Return(getHttpInstanceSummaryForTest(), nil) @@ -117,7 +117,7 @@ func TestServiceDiscoveryClient_ListServices_InstanceError(t *testing.T) { endptErr := errors.New("error listing endpoints") tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, map[string]string{ model.ClusterSetIdAttr: test.ClusterSet, }). Return([]types.HttpInstanceSummary{}, endptErr) @@ -180,6 +180,23 @@ func TestServiceDiscoveryClient_CreateService_NamespaceError(t *testing.T) { assert.Equal(t, nsErr, err) } +func TestServiceDiscoveryClient_CreateService_NamespaceNotFound(t *testing.T) { + tc := getTestSdClient(t) + defer tc.close() + + tc.mockCache.EXPECT().GetNamespaceMap().Return(map[string]*model.Namespace{}, true) + tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName).Return(test.OpId1, nil) + tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1).Return(test.HttpNsId, nil) + tc.mockCache.EXPECT().EvictNamespaceMap() + + tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). + Return(test.SvcId, nil) + tc.mockCache.EXPECT().EvictServiceIdMap(test.HttpNsName) + + err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) + assert.Nil(t, err) +} + func TestServiceDiscoveryClient_CreateService_CreateServiceError(t *testing.T) { tc := getTestSdClient(t) defer tc.close() @@ -264,7 +281,7 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { tc.mockCache.EXPECT().CacheServiceIdMap(test.HttpNsName, getServiceIdMapForTest()) tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return([]*model.Endpoint{}, false) - tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, &map[string]string{ + tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, map[string]string{ model.ClusterSetIdAttr: test.ClusterSet, }). Return(getHttpInstanceSummaryForTest(), nil) @@ -288,6 +305,30 @@ func TestServiceDiscoveryClient_GetService_CachedValues(t *testing.T) { assert.Equal(t, test.GetTestService(), svc) } +func TestServiceDiscoveryClient_GetService_ServiceNotFound(t *testing.T) { + tc := getTestSdClient(t) + defer tc.close() + + tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return(nil, false) + + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(nil, false) + + tc.mockCache.EXPECT().GetNamespaceMap().Return(nil, false) + tc.mockApi.EXPECT().GetNamespaceMap(context.TODO()). + Return(getNamespaceMapForTest(), nil) + tc.mockCache.EXPECT().CacheNamespaceMap(getNamespaceMapForTest()) + + // return empty list from CloudMap's api + tc.mockApi.EXPECT().GetServiceIdMap(context.TODO(), test.HttpNsId). + Return(map[string]string{}, nil) + tc.mockCache.EXPECT().CacheServiceIdMap(test.HttpNsName, map[string]string{}) + + svc, err := tc.client.GetService(context.TODO(), test.HttpNsName, test.SvcName) + assert.NotNil(t, err) + assert.True(t, common.IsNotFound(err), svc) + assert.Contains(t, err.Error(), test.SvcName) +} + func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc := getTestSdClient(t) defer tc.close() diff --git a/pkg/common/errors.go b/pkg/common/errors.go new file mode 100644 index 00000000..97f6537a --- /dev/null +++ b/pkg/common/errors.go @@ -0,0 +1,32 @@ +package common + +import ( + "github.com/pkg/errors" +) + +var notFound = errors.New("resource was not found") + +func IsNotFound(err error) bool { + return errors.Is(err, notFound) +} + +func IsUnknown(err error) bool { + return err != nil && !errors.Is(err, notFound) +} + +func NotFoundError(message string) error { + return errors.Wrap(notFound, message) +} + +func Wrap(err1 error, err2 error) error { + switch { + case err1 != nil && err2 != nil: + return errors.Wrap(err1, err2.Error()) + case err1 != nil: + return err1 + case err2 != nil: + return err2 + default: + return nil + } +} diff --git a/pkg/common/errors_test.go b/pkg/common/errors_test.go new file mode 100644 index 00000000..c84b35b4 --- /dev/null +++ b/pkg/common/errors_test.go @@ -0,0 +1,88 @@ +package common + +import ( + "errors" + "testing" +) + +func TestIsNotFound(t *testing.T) { + type args struct { + err error + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "trueCase", + args: struct{ err error }{err: NotFoundError("1")}, + want: true, + }, + { + name: "falseCase", + args: struct{ err error }{err: errors.New("test")}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsNotFound(tt.args.err); got != tt.want { + t.Errorf("IsNotFound() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsUnknown(t *testing.T) { + type args struct { + err error + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "trueCase", + args: struct{ err error }{err: errors.New("test")}, + want: true, + }, + { + name: "falseCase", + args: struct{ err error }{err: NotFoundError("1")}, + want: false, + }, + { + name: "nilCase", + args: struct{ err error }{err: nil}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsUnknown(tt.args.err); got != tt.want { + t.Errorf("IsUnknown() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNotFoundError(t *testing.T) { + tests := []struct { + name string + arg string + }{ + { + name: "happyCase", + arg: "arg", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := NotFoundError(tt.arg); !IsNotFound(err) { + t.Errorf("NotFoundError() error = %v, containsErr = %v", err, notFound) + } + }) + } +} diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index a235b2fa..85184a00 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -54,7 +54,7 @@ func (r *CloudMapReconciler) Start(ctx context.Context) error { } // Reconcile triggers a single reconciliation round -func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { +func (r *CloudMapReconciler) Reconcile(ctx context.Context) (err error) { clusterProperties, err := r.ClusterUtils.GetClusterProperties(ctx) if err != nil { r.Log.Error(err, "unable to retrieve ClusterId and ClusterSetId") @@ -63,21 +63,22 @@ func (r *CloudMapReconciler) Reconcile(ctx context.Context) error { r.Log.Debug("clusterProperties found", "ClusterId", clusterProperties.ClusterId(), "ClusterSetId", clusterProperties.ClusterSetId()) namespaces := v1.NamespaceList{} - if err := r.Client.List(ctx, &namespaces); err != nil { + if err = r.Client.List(ctx, &namespaces); err != nil { r.Log.Error(err, "unable to list cluster namespaces") return err } for _, ns := range namespaces.Items { - if err := r.reconcileNamespace(ctx, ns.Name); err != nil { - return err + reconErr := r.reconcileNamespace(ctx, ns.Name) + if reconErr != nil { + err = common.Wrap(err, reconErr) } } - return nil + return err } -func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceName string) error { +func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceName string) (err error) { r.Log.Debug("syncing namespace", "namespace", namespaceName) desiredServices, err := r.Cloudmap.ListServices(ctx, namespaceName) @@ -89,12 +90,12 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa serviceImports := multiclusterv1alpha1.ServiceImportList{} if err = r.Client.List(ctx, &serviceImports, client.InNamespace(namespaceName)); err != nil { r.Log.Error(err, "failed to reconcile namespace", "namespace", namespaceName) - return nil + return err } existingImportsMap := make(map[string]multiclusterv1alpha1.ServiceImport) for _, svc := range serviceImports.Items { - existingImportsMap[svc.Namespace+"/"+svc.Name] = svc + existingImportsMap[svc.Name] = svc } for _, svc := range desiredServices { @@ -103,22 +104,24 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa continue } - if err = r.reconcileService(ctx, svc); err != nil { - r.Log.Error(err, "error when syncing service", "namespace", svc.Namespace, "name", svc.Name) + if reconErr := r.reconcileService(ctx, svc); reconErr != nil { + r.Log.Error(reconErr, "error when syncing service", "namespace", svc.Namespace, "name", svc.Name) + err = common.Wrap(err, reconErr) } - delete(existingImportsMap, svc.Namespace+"/"+svc.Name) + delete(existingImportsMap, svc.Name) } // delete remaining imports that have not been matched for _, i := range existingImportsMap { - if err = r.Client.Delete(ctx, &i); err != nil { - r.Log.Error(err, "error deleting ServiceImport", "namespace", i.Namespace, "name", i.Name) + r.Log.Info("delete ServiceImport", "namespace", i.Namespace, "name", i.Name) + if deleteErr := r.Client.Delete(ctx, &i); deleteErr != nil { + r.Log.Error(deleteErr, "error deleting ServiceImport", "namespace", i.Namespace, "name", i.Name) + err = common.Wrap(err, deleteErr) continue } - r.Log.Info("delete ServiceImport", "namespace", i.Namespace, "name", i.Name) } - return nil + return err } func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Service) error { diff --git a/pkg/controllers/multicluster/cloudmap_controller_test.go b/pkg/controllers/multicluster/cloudmap_controller_test.go index b21a29c8..4c8b27ee 100644 --- a/pkg/controllers/multicluster/cloudmap_controller_test.go +++ b/pkg/controllers/multicluster/cloudmap_controller_test.go @@ -26,7 +26,10 @@ import ( func TestCloudMapReconciler_Reconcile(t *testing.T) { // create a fake controller client and add some objects - fakeClient := fake.NewClientBuilder().WithScheme(getCloudMapReconcilerScheme()).WithObjects(k8sNamespaceForTest(), test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() + svcImportToBeDeleted := serviceImportForTest("svc1") + fakeClient := fake.NewClientBuilder().WithScheme(getCloudMapReconcilerScheme()). + WithObjects(k8sNamespaceForTest(), serviceImportForTest(test.SvcName), svcImportToBeDeleted, + test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() // create a mock cloudmap service discovery client mockController := gomock.NewController(t) @@ -63,11 +66,19 @@ func TestCloudMapReconciler_Reconcile(t *testing.T) { assert.NoError(t, err) endpointSlice := endpointSliceList.Items[0] assertEndpointSlice(t, &endpointSlice, test.Port1, test.EndptIp1, test.ClusterId1) + + // assert svcImportToBeDeleted is not found in list + serviceImports := &multiclusterv1alpha1.ServiceImportList{} + err = fakeClient.List(context.TODO(), serviceImports, client.InNamespace(test.HttpNsName)) + assert.NoError(t, err) + assert.True(t, len(serviceImports.Items) == 1) + assert.Equal(t, serviceImports.Items[0].Name, test.SvcName) } func TestCloudMapReconciler_Reconcile_MulticlusterService(t *testing.T) { // create a fake controller client and add some objects - fakeClient := fake.NewClientBuilder().WithScheme(getCloudMapReconcilerScheme()).WithObjects(k8sNamespaceForTest(), test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() + fakeClient := fake.NewClientBuilder().WithScheme(getCloudMapReconcilerScheme()). + WithObjects(k8sNamespaceForTest(), test.ClusterIdForTest(), test.ClusterSetIdForTest()).Build() // create a mock cloudmap service discovery client mockController := gomock.NewController(t) diff --git a/pkg/controllers/multicluster/controllers_common_test.go b/pkg/controllers/multicluster/controllers_common_test.go index 5775246d..12cfb690 100644 --- a/pkg/controllers/multicluster/controllers_common_test.go +++ b/pkg/controllers/multicluster/controllers_common_test.go @@ -56,6 +56,15 @@ func serviceExportForTest() *multiclusterv1alpha1.ServiceExport { } } +func serviceImportForTest(svcName string) *multiclusterv1alpha1.ServiceImport { + return &multiclusterv1alpha1.ServiceImport{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: test.HttpNsName, + }, + } +} + func endpointSliceForTest() *discovery.EndpointSlice { port := int32(test.Port1) protocol := v1.ProtocolTCP diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index f49a5878..f9fd1700 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -168,11 +168,11 @@ func (r *ServiceExportReconciler) addFinalizerAndOwnerRef(ctx context.Context, s func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context, service *v1.Service) (*model.Service, error) { cmService, err := r.CloudMap.GetService(ctx, service.Namespace, service.Name) - if err != nil { + if common.IsUnknown(err) { return nil, err } - if cmService == nil { + if common.IsNotFound(err) { err = r.CloudMap.CreateService(ctx, service.Namespace, service.Name) if err != nil { r.Log.Error(err, "error creating a new Service in Cloud Map", "namespace", service.Namespace, "name", service.Name) @@ -191,7 +191,7 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor r.Log.Info("removing service export", "namespace", serviceExport.Namespace, "name", serviceExport.Name) cmService, err := r.CloudMap.GetService(ctx, serviceExport.Namespace, serviceExport.Name) - if err != nil { + if common.IsUnknown(err) { r.Log.Error(err, "error fetching Service from Cloud Map", "namespace", serviceExport.Namespace, "name", serviceExport.Name) return ctrl.Result{}, err } diff --git a/pkg/controllers/multicluster/serviceexport_controller_test.go b/pkg/controllers/multicluster/serviceexport_controller_test.go index 1386dcc2..b24022bc 100644 --- a/pkg/controllers/multicluster/serviceexport_controller_test.go +++ b/pkg/controllers/multicluster/serviceexport_controller_test.go @@ -43,7 +43,8 @@ func TestServiceExportReconciler_Reconcile_NewServiceExport(t *testing.T) { // expected interactions with the Cloud Map client // The first get call is expected to return nil, then second call after the creation of service is // supposed to return the value - first := mock.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName).Return(nil, nil) + first := mock.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName). + Return(nil, common.NotFoundError("")) second := mock.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName). Return(&model.Service{Namespace: test.HttpNsName, Name: test.SvcName}, nil) gomock.InOrder(first, second) From 4dc8b4d7e3815a7bae64f010b2d8eda71e2d2f6d Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Mon, 14 Nov 2022 18:02:57 -0800 Subject: [PATCH 127/163] Rework operation poller (#245) * Replace ListOperations with GetOperation because it has higher limits. Rework the poller code to keep the OperationPolling simple with Submit - submitting operation provider, Poll - polling operations till complete or timeout, and Await - waiting for the all the submitted operation's result. Moving the rate-limiter to the separate file for the clarity and separation of concerns. Adding the limiter for Register/DeregisterInstance api calls. * Add found check in the rate limiters map. --- Makefile | 1 - integration/janitor/janitor.go | 10 +- integration/janitor/janitor_test.go | 9 +- pkg/cloudmap/api.go | 94 ++------ pkg/cloudmap/api_test.go | 50 +---- pkg/cloudmap/client.go | 38 ++-- pkg/cloudmap/client_test.go | 174 ++++++++++----- pkg/cloudmap/operation_collector.go | 84 -------- pkg/cloudmap/operation_collector_test.go | 57 ----- pkg/cloudmap/operation_poller.go | 192 +++++++---------- pkg/cloudmap/operation_poller_test.go | 259 +++++++++-------------- pkg/common/ratelimiter.go | 43 ++++ pkg/common/ratelimiter_test.go | 64 ++++++ 13 files changed, 449 insertions(+), 626 deletions(-) delete mode 100644 pkg/cloudmap/operation_collector.go delete mode 100644 pkg/cloudmap/operation_collector_test.go create mode 100644 pkg/common/ratelimiter.go create mode 100644 pkg/common/ratelimiter_test.go diff --git a/Makefile b/Makefile index 1d1a0e44..b7eb6900 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,6 @@ ifneq ($(shell test -d $(MOCKS_DESTINATION); echo $$?), 0) $(MOCKGEN) --source pkg/cloudmap/client.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/client_mock.go --package cloudmap_mock $(MOCKGEN) --source pkg/cloudmap/cache.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/cache_mock.go --package cloudmap_mock $(MOCKGEN) --source pkg/cloudmap/operation_poller.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_poller_mock.go --package cloudmap_mock - $(MOCKGEN) --source pkg/cloudmap/operation_collector.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_collector_mock.go --package cloudmap_mock $(MOCKGEN) --source pkg/cloudmap/api.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/api_mock.go --package cloudmap_mock $(MOCKGEN) --source pkg/cloudmap/aws_facade.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/aws_facade_mock.go --package cloudmap_mock $(MOCKGEN) --source integration/janitor/api.go --destination $(MOCKS_DESTINATION)/integration/janitor/api_mock.go --package janitor_mock diff --git a/integration/janitor/janitor.go b/integration/janitor/janitor.go index 0a05fcfc..260fc1e0 100644 --- a/integration/janitor/janitor.go +++ b/integration/janitor/janitor.go @@ -72,7 +72,7 @@ func (j *cloudMapJanitor) Cleanup(ctx context.Context, nsName string) { opId, err := j.sdApi.DeleteNamespace(ctx, ns.Id) if err == nil { fmt.Println("namespace delete in progress") - _, err = j.sdApi.PollNamespaceOperation(ctx, opId) + _, err = cloudmap.NewOperationPoller(j.sdApi).Poll(ctx, opId) } j.checkOrFail(err, "clean up successful", "could not cleanup namespace") } @@ -87,17 +87,17 @@ func (j *cloudMapJanitor) deregisterInstances(ctx context.Context, nsName string fmt.Sprintf("service has %d instances to clean", len(insts)), "could not list instances to cleanup") - opColl := cloudmap.NewOperationCollector() + opPoller := cloudmap.NewOperationPoller(j.sdApi) for _, inst := range insts { instId := aws.ToString(inst.InstanceId) fmt.Printf("found instance to clean: %s\n", instId) - opColl.Add(func() (opId string, err error) { + opPoller.Submit(ctx, func() (opId string, err error) { return j.sdApi.DeregisterInstance(ctx, svcId, instId) }) } - opErr := cloudmap.NewDeregisterInstancePoller(j.sdApi, svcId, opColl.Collect(), opColl.GetStartTime()).Poll(ctx) - j.checkOrFail(opErr, "instances de-registered", "could not cleanup instances") + err = opPoller.Await() + j.checkOrFail(err, "instances de-registered", "could not cleanup instances") } func (j *cloudMapJanitor) checkOrFail(err error, successMsg string, failMsg string) { diff --git a/integration/janitor/janitor_test.go b/integration/janitor/janitor_test.go index 6f1d8c99..95e25c71 100644 --- a/integration/janitor/janitor_test.go +++ b/integration/janitor/janitor_test.go @@ -39,14 +39,15 @@ func TestCleanupHappyCase(t *testing.T) { tj.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1). Return(test.OpId1, nil) - tj.mockApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). - Return(map[string]types.OperationStatus{test.OpId1: types.OperationStatusSuccess}, nil) + tj.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(&types.Operation{Status: types.OperationStatusSuccess}, nil) tj.mockApi.EXPECT().DeleteService(context.TODO(), test.SvcId). Return(nil) tj.mockApi.EXPECT().DeleteNamespace(context.TODO(), test.HttpNsId). Return(test.OpId2, nil) - tj.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId2). - Return(test.HttpNsId, nil) + tj.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId2). + Return(&types.Operation{Status: types.OperationStatusSuccess, + Targets: map[string]string{string(types.OperationTargetTypeNamespace): test.HttpNsId}}, nil) tj.janitor.Cleanup(context.TODO(), test.HttpNsName) assert.False(t, *tj.failed) diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index 0342b5ba..cb91f456 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -2,18 +2,12 @@ package cloudmap import ( "context" - "errors" - "fmt" - "time" - - "golang.org/x/time/rate" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -32,9 +26,6 @@ type ServiceDiscoveryApi interface { // DiscoverInstances returns a list of service instances registered to a given service. DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters map[string]string) (insts []types.HttpInstanceSummary, err error) - // ListOperations returns a map of operations to their status matching a list of filters. - ListOperations(ctx context.Context, opFilters []types.OperationFilter) (operationStatusMap map[string]types.OperationStatus, err error) - // GetOperation returns an operation. GetOperation(ctx context.Context, operationId string) (operation *types.Operation, err error) @@ -49,32 +40,25 @@ type ServiceDiscoveryApi interface { // DeregisterInstance de-registers a service instance in Cloud Map. DeregisterInstance(ctx context.Context, serviceId string, instanceId string) (operationId string, err error) - - // PollNamespaceOperation polls a namespace operation, and returns the namespace ID. - PollNamespaceOperation(ctx context.Context, operationId string) (namespaceId string, err error) } type serviceDiscoveryApi struct { - log common.Logger - awsFacade AwsFacade - nsRateLimiter *rate.Limiter - svcRateLimiter *rate.Limiter - opRateLimiter *rate.Limiter + log common.Logger + awsFacade AwsFacade + rateLimiter common.RateLimiter } // NewServiceDiscoveryApiFromConfig creates a new AWS Cloud Map API connection manager from an AWS client config. func NewServiceDiscoveryApiFromConfig(cfg *aws.Config) ServiceDiscoveryApi { return &serviceDiscoveryApi{ - log: common.NewLogger("cloudmap", "api"), - awsFacade: NewAwsFacadeFromConfig(cfg), - nsRateLimiter: rate.NewLimiter(rate.Every(1*time.Second), 5), // 1 per second - svcRateLimiter: rate.NewLimiter(rate.Every(2*time.Second), 10), // 2 per second - opRateLimiter: rate.NewLimiter(rate.Every(100*time.Second), 200), // 100 per second + log: common.NewLogger("cloudmap", "api"), + awsFacade: NewAwsFacadeFromConfig(cfg), + rateLimiter: common.NewDefaultRateLimiter(), } } func (sdApi *serviceDiscoveryApi) GetNamespaceMap(ctx context.Context) (map[string]*model.Namespace, error) { - err := sdApi.nsRateLimiter.Wait(ctx) + err := sdApi.rateLimiter.Wait(ctx, common.ListNamespaces) if err != nil { return nil, err } @@ -105,7 +89,7 @@ func (sdApi *serviceDiscoveryApi) GetNamespaceMap(ctx context.Context) (map[stri } func (sdApi *serviceDiscoveryApi) GetServiceIdMap(ctx context.Context, nsId string) (map[string]string, error) { - err := sdApi.svcRateLimiter.Wait(ctx) + err := sdApi.rateLimiter.Wait(ctx, common.ListServices) if err != nil { return nil, err } @@ -151,30 +135,8 @@ func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName return out.Instances, nil } -func (sdApi *serviceDiscoveryApi) ListOperations(ctx context.Context, opFilters []types.OperationFilter) (map[string]types.OperationStatus, error) { - opStatusMap := make(map[string]types.OperationStatus) - - pages := sd.NewListOperationsPaginator(sdApi.awsFacade, &sd.ListOperationsInput{ - Filters: opFilters, - }) - - for pages.HasMorePages() { - output, err := pages.NextPage(ctx) - - if err != nil { - return opStatusMap, err - } - - for _, sdOp := range output.Operations { - opStatusMap[aws.ToString(sdOp.Id)] = sdOp.Status - } - } - - return opStatusMap, nil -} - func (sdApi *serviceDiscoveryApi) GetOperation(ctx context.Context, opId string) (operation *types.Operation, err error) { - err = sdApi.opRateLimiter.Wait(ctx) + err = sdApi.rateLimiter.Wait(ctx, common.GetOperation) if err != nil { return nil, err } @@ -236,6 +198,11 @@ func (sdApi *serviceDiscoveryApi) getDnsConfig() types.DnsConfig { } func (sdApi *serviceDiscoveryApi) RegisterInstance(ctx context.Context, svcId string, instId string, instAttrs map[string]string) (opId string, err error) { + err = sdApi.rateLimiter.Wait(ctx, common.RegisterInstance) + if err != nil { + return "", err + } + regResp, err := sdApi.awsFacade.RegisterInstance(ctx, &sd.RegisterInstanceInput{ Attributes: instAttrs, InstanceId: &instId, @@ -250,6 +217,11 @@ func (sdApi *serviceDiscoveryApi) RegisterInstance(ctx context.Context, svcId st } func (sdApi *serviceDiscoveryApi) DeregisterInstance(ctx context.Context, svcId string, instId string) (opId string, err error) { + err = sdApi.rateLimiter.Wait(ctx, common.DeregisterInstance) + if err != nil { + return "", err + } + deregResp, err := sdApi.awsFacade.DeregisterInstance(ctx, &sd.DeregisterInstanceInput{ InstanceId: &instId, ServiceId: &svcId, @@ -261,31 +233,3 @@ func (sdApi *serviceDiscoveryApi) DeregisterInstance(ctx context.Context, svcId return aws.ToString(deregResp.OperationId), err } - -func (sdApi *serviceDiscoveryApi) PollNamespaceOperation(ctx context.Context, opId string) (nsId string, err error) { - err = wait.Poll(defaultOperationPollInterval, defaultOperationPollTimeout, func() (done bool, err error) { - sdApi.log.Info("polling operation", "opId", opId) - op, err := sdApi.GetOperation(ctx, opId) - - if err != nil { - return true, err - } - - if op.Status == types.OperationStatusFail { - return true, fmt.Errorf("failed to create namespace: %s", aws.ToString(op.ErrorMessage)) - } - - if op.Status == types.OperationStatusSuccess { - nsId = op.Targets[string(types.OperationTargetTypeNamespace)] - return true, nil - } - - return false, nil - }) - - if err == wait.ErrWaitTimeout { - err = errors.New(operationPollTimoutErrorMessage) - } - - return nsId, err -} diff --git a/pkg/cloudmap/api_test.go b/pkg/cloudmap/api_test.go index 70143ceb..8990d51b 100644 --- a/pkg/cloudmap/api_test.go +++ b/pkg/cloudmap/api_test.go @@ -5,9 +5,6 @@ import ( "errors" "fmt" "testing" - "time" - - "golang.org/x/time/rate" aboutv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/about/v1alpha1" @@ -131,26 +128,6 @@ func TestServiceDiscoveryApi_DiscoverInstances_HappyCase(t *testing.T) { assert.Equal(t, test.EndptId2, *insts[1].InstanceId) } -func TestServiceDiscoveryApi_ListOperations_HappyCase(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() - - awsFacade := cloudmapMock.NewMockAwsFacade(mockController) - sdApi := getServiceDiscoveryApi(t, awsFacade) - - filters := make([]types.OperationFilter, 0) - awsFacade.EXPECT().ListOperations(context.TODO(), &sd.ListOperationsInput{Filters: filters}). - Return(&sd.ListOperationsOutput{ - Operations: []types.OperationSummary{ - {Id: aws.String(test.OpId1), Status: types.OperationStatusSuccess}, - }}, nil) - - ops, err := sdApi.ListOperations(context.TODO(), filters) - assert.Nil(t, err, "No error for happy case") - assert.True(t, len(ops) == 1) - assert.Equal(t, ops[test.OpId1], types.OperationStatusSuccess) -} - func TestServiceDiscoveryApi_GetOperation_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() @@ -316,33 +293,12 @@ func TestServiceDiscoveryApi_DeregisterInstance_Error(t *testing.T) { assert.Equal(t, sdkErr, err) } -func TestServiceDiscoveryApi_PollNamespaceOperation_HappyCase(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() - - awsFacade := cloudmapMock.NewMockAwsFacade(mockController) - awsFacade.EXPECT().GetOperation(context.TODO(), &sd.GetOperationInput{OperationId: aws.String(test.OpId1)}). - Return(&sd.GetOperationOutput{Operation: &types.Operation{Status: types.OperationStatusPending}}, nil) - - awsFacade.EXPECT().GetOperation(context.TODO(), &sd.GetOperationInput{OperationId: aws.String(test.OpId1)}). - Return(&sd.GetOperationOutput{Operation: &types.Operation{Status: types.OperationStatusSuccess, - Targets: map[string]string{string(types.OperationTargetTypeNamespace): test.HttpNsId}}}, nil) - - sdApi := getServiceDiscoveryApi(t, awsFacade) - - nsId, err := sdApi.PollNamespaceOperation(context.TODO(), test.OpId1) - assert.Nil(t, err) - assert.Equal(t, test.HttpNsId, nsId) -} - func getServiceDiscoveryApi(t *testing.T, awsFacade *cloudmapMock.MockAwsFacade) ServiceDiscoveryApi { scheme := runtime.NewScheme() scheme.AddKnownTypes(aboutv1alpha1.GroupVersion, &aboutv1alpha1.ClusterProperty{}) return &serviceDiscoveryApi{ - log: common.NewLoggerWithLogr(testr.New(t)), - awsFacade: awsFacade, - nsRateLimiter: rate.NewLimiter(rate.Every(1*time.Second), 2), // 1 per second - svcRateLimiter: rate.NewLimiter(rate.Every(2*time.Second), 4), // 2 per second - opRateLimiter: rate.NewLimiter(rate.Every(10*time.Second), 100), // 10 per second + log: common.NewLoggerWithLogr(testr.New(t)), + awsFacade: awsFacade, + rateLimiter: common.NewDefaultRateLimiter(), } } diff --git a/pkg/cloudmap/client.go b/pkg/cloudmap/client.go index afe1cde8..6cd4ddca 100644 --- a/pkg/cloudmap/client.go +++ b/pkg/cloudmap/client.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" ) // ServiceDiscoveryClient provides the service endpoint management functionality required by the AWS Cloud Map @@ -149,27 +150,21 @@ func (sdc *serviceDiscoveryClient) RegisterEndpoints(ctx context.Context, nsName return err } - opCollector := NewOperationCollector() - + operationPoller := NewOperationPoller(sdc.sdApi) for _, endpt := range endpts { endptId := endpt.Id endptAttrs := endpt.GetCloudMapAttributes() - opCollector.Add(func() (opId string, err error) { + operationPoller.Submit(ctx, func() (opId string, err error) { return sdc.sdApi.RegisterInstance(ctx, svcId, endptId, endptAttrs) }) } - err = NewRegisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) - // Evict cache entry so next list call reflects changes sdc.cache.EvictEndpoints(nsName, svcName) + err = operationPoller.Await() if err != nil { - return err - } - - if !opCollector.IsAllOperationsCreated() { - return errors.New("failure while registering endpoints") + return common.Wrap(err, errors.New("failure while registering endpoints")) } return nil @@ -188,29 +183,23 @@ func (sdc *serviceDiscoveryClient) DeleteEndpoints(ctx context.Context, nsName s return err } - opCollector := NewOperationCollector() - + operationPoller := NewOperationPoller(sdc.sdApi) for _, endpt := range endpts { endptId := endpt.Id - // add operation to delete endpoint - opCollector.Add(func() (opId string, err error) { + operationPoller.Submit(ctx, func() (opId string, err error) { return sdc.sdApi.DeregisterInstance(ctx, svcId, endptId) }) } - err = NewDeregisterInstancePoller(sdc.sdApi, svcId, opCollector.Collect(), opCollector.GetStartTime()).Poll(ctx) - // Evict cache entry so next list call reflects changes sdc.cache.EvictEndpoints(nsName, svcName) - if err != nil { - return err - } - if !opCollector.IsAllOperationsCreated() { - return errors.New("failure while de-registering endpoints") + err = operationPoller.Await() + if err != nil { + return common.Wrap(err, errors.New("failure while de-registering endpoints")) } - return nil + return err } func (sdc *serviceDiscoveryClient) getEndpoints(ctx context.Context, nsName string, svcName string) (endpts []*model.Endpoint, err error) { @@ -315,12 +304,13 @@ func (sdc *serviceDiscoveryClient) createNamespace(ctx context.Context, nsName s return nil, err } - nsId, err := sdc.sdApi.PollNamespaceOperation(ctx, opId) + op, err := NewOperationPoller(sdc.sdApi).Poll(ctx, opId) if err != nil { return nil, err } + nsId := op.Targets[string(types.OperationTargetTypeNamespace)] - sdc.log.Info("namespace created", "nsId", nsId) + sdc.log.Info("namespace created", "nsId", nsId, "namespace", nsName) // Default namespace type HTTP namespace = &model.Namespace{ diff --git a/pkg/cloudmap/client_test.go b/pkg/cloudmap/client_test.go index 84da49fd..3de7f567 100644 --- a/pkg/cloudmap/client_test.go +++ b/pkg/cloudmap/client_test.go @@ -2,6 +2,7 @@ package cloudmap import ( "context" + "errors" "strconv" "testing" @@ -17,7 +18,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) @@ -186,7 +186,9 @@ func TestServiceDiscoveryClient_CreateService_NamespaceNotFound(t *testing.T) { tc.mockCache.EXPECT().GetNamespaceMap().Return(map[string]*model.Namespace{}, true) tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName).Return(test.OpId1, nil) - tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1).Return(test.HttpNsId, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(&types.Operation{Status: types.OperationStatusSuccess, + Targets: map[string]string{string(types.OperationTargetTypeNamespace): test.HttpNsId}}, nil) tc.mockCache.EXPECT().EvictNamespaceMap() tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). @@ -221,8 +223,9 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_HappyCase(t *test tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). Return(test.OpId1, nil) - tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). - Return(test.HttpNsId, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(&types.Operation{Status: types.OperationStatusSuccess, + Targets: map[string]string{string(types.OperationTargetTypeNamespace): test.HttpNsId}}, nil) tc.mockCache.EXPECT().EvictNamespaceMap() tc.mockApi.EXPECT().CreateService(context.TODO(), *test.GetTestHttpNamespace(), test.SvcName). @@ -242,8 +245,8 @@ func TestServiceDiscoveryClient_CreateService_CreatesNamespace_PollError(t *test pollErr := errors.New("polling error") tc.mockApi.EXPECT().CreateHttpNamespace(context.TODO(), test.HttpNsName). Return(test.OpId1, nil) - tc.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId1). - Return("", pollErr) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(nil, pollErr) err := tc.client.CreateService(context.TODO(), test.HttpNsName, test.SvcName) assert.Equal(t, pollErr, err) @@ -283,8 +286,7 @@ func TestServiceDiscoveryClient_GetService_HappyCase(t *testing.T) { tc.mockCache.EXPECT().GetEndpoints(test.HttpNsName, test.SvcName).Return([]*model.Endpoint{}, false) tc.mockApi.EXPECT().DiscoverInstances(context.TODO(), test.HttpNsName, test.SvcName, map[string]string{ model.ClusterSetIdAttr: test.ClusterSet, - }). - Return(getHttpInstanceSummaryForTest(), nil) + }).Return(getHttpInstanceSummaryForTest(), nil) tc.mockCache.EXPECT().CacheEndpoints(test.HttpNsName, test.SvcName, []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) @@ -335,51 +337,14 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) - attrs1 := map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSet, - model.EndpointIpv4Attr: test.EndptIp1, - model.EndpointPortAttr: test.PortStr1, - model.EndpointPortNameAttr: test.PortName1, - model.EndpointProtocolAttr: test.Protocol1, - model.EndpointReadyAttr: test.EndptReadyTrue, - model.ServicePortNameAttr: test.PortName1, - model.ServicePortAttr: test.ServicePortStr1, - model.ServiceProtocolAttr: test.Protocol1, - model.ServiceTargetPortAttr: test.PortStr1, - model.ServiceTypeAttr: test.SvcType, - model.EndpointHostnameAttr: test.Hostname, - model.EndpointNodeNameAttr: test.Nodename, - model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), - model.K8sVersionAttr: test.PackageVersion, - } - attrs2 := map[string]string{ - model.ClusterIdAttr: test.ClusterId1, - model.ClusterSetIdAttr: test.ClusterSet, - model.EndpointIpv4Attr: test.EndptIp2, - model.EndpointPortAttr: test.PortStr2, - model.EndpointPortNameAttr: test.PortName2, - model.EndpointProtocolAttr: test.Protocol2, - model.EndpointReadyAttr: test.EndptReadyTrue, - model.ServicePortNameAttr: test.PortName2, - model.ServicePortAttr: test.ServicePortStr2, - model.ServiceProtocolAttr: test.Protocol2, - model.ServiceTargetPortAttr: test.PortStr2, - model.ServiceTypeAttr: test.SvcType, - model.EndpointHostnameAttr: test.Hostname, - model.EndpointNodeNameAttr: test.Nodename, - model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), - model.K8sVersionAttr: test.PackageVersion, - } - - tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, attrs1). + tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, getAttrs1()). Return(test.OpId1, nil) - tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId2, attrs2). + tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId2, getAttrs2()). Return(test.OpId2, nil) - tc.mockApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). - Return(map[string]types.OperationStatus{ - test.OpId1: types.OperationStatusSuccess, - test.OpId2: types.OperationStatusSuccess}, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(&types.Operation{Status: types.OperationStatusSuccess}, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId2). + Return(&types.Operation{Status: types.OperationStatusSuccess}, nil) tc.mockCache.EXPECT().EvictEndpoints(test.HttpNsName, test.SvcName) @@ -389,18 +354,44 @@ func TestServiceDiscoveryClient_RegisterEndpoints(t *testing.T) { assert.Nil(t, err) } +func TestServiceDiscoveryClient_RegisterEndpoints_PollFailure(t *testing.T) { + tc := getTestSdClient(t) + defer tc.close() + + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) + + tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId1, getAttrs1()). + Return(test.OpId1, nil) + tc.mockApi.EXPECT().RegisterInstance(context.TODO(), test.SvcId, test.EndptId2, getAttrs2()). + Return(test.OpId2, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(&types.Operation{Status: types.OperationStatusFail}, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId2). + Return(&types.Operation{Status: types.OperationStatusSuccess}, nil) + + tc.mockCache.EXPECT().EvictEndpoints(test.HttpNsName, test.SvcName) + + err := tc.client.RegisterEndpoints(context.TODO(), test.HttpNsName, test.SvcName, + []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}) + + assert.NotNil(t, err) + assert.Contains(t, err.Error(), test.OpId1) +} + func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { tc := getTestSdClient(t) defer tc.close() tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) - tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).Return(test.OpId1, nil) - tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2).Return(test.OpId2, nil) - tc.mockApi.EXPECT().ListOperations(context.TODO(), gomock.Any()). - Return(map[string]types.OperationStatus{ - test.OpId1: types.OperationStatusSuccess, - test.OpId2: types.OperationStatusSuccess}, nil) + tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1). + Return(test.OpId1, nil) + tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2). + Return(test.OpId2, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(&types.Operation{Status: types.OperationStatusSuccess}, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId2). + Return(&types.Operation{Status: types.OperationStatusSuccess}, nil) tc.mockCache.EXPECT().EvictEndpoints(test.HttpNsName, test.SvcName) @@ -412,6 +403,33 @@ func TestServiceDiscoveryClient_DeleteEndpoints(t *testing.T) { assert.Nil(t, err) } +func TestServiceDiscoveryClient_DeleteEndpoints_PollFailure(t *testing.T) { + tc := getTestSdClient(t) + defer tc.close() + + tc.mockCache.EXPECT().GetServiceIdMap(test.HttpNsName).Return(getServiceIdMapForTest(), true) + + tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1). + Return(test.OpId1, nil) + tc.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId2). + Return(test.OpId2, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId1). + Return(&types.Operation{Status: types.OperationStatusFail}, nil) + tc.mockApi.EXPECT().GetOperation(context.TODO(), test.OpId2). + Return(&types.Operation{Status: types.OperationStatusSuccess}, nil) + + tc.mockCache.EXPECT().EvictEndpoints(test.HttpNsName, test.SvcName) + + err := tc.client.DeleteEndpoints(context.TODO(), test.HttpNsName, test.SvcName, + []*model.Endpoint{ + {Id: test.EndptId1, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSet}, + {Id: test.EndptId2, ClusterId: test.ClusterId1, ClusterSetId: test.ClusterSet}, + }) + + assert.NotNil(t, err) + assert.Contains(t, err.Error(), test.OpId1) +} + func getTestSdClient(t *testing.T) *testSdClient { test.SetTestVersion() mockController := gomock.NewController(t) @@ -490,3 +508,45 @@ func getNamespaceMapForTest() map[string]*model.Namespace { func getServiceIdMapForTest() map[string]string { return map[string]string{test.SvcName: test.SvcId} } + +func getAttrs2() map[string]string { + return map[string]string{ + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSet, + model.EndpointIpv4Attr: test.EndptIp2, + model.EndpointPortAttr: test.PortStr2, + model.EndpointPortNameAttr: test.PortName2, + model.EndpointProtocolAttr: test.Protocol2, + model.EndpointReadyAttr: test.EndptReadyTrue, + model.ServicePortNameAttr: test.PortName2, + model.ServicePortAttr: test.ServicePortStr2, + model.ServiceProtocolAttr: test.Protocol2, + model.ServiceTargetPortAttr: test.PortStr2, + model.ServiceTypeAttr: test.SvcType, + model.EndpointHostnameAttr: test.Hostname, + model.EndpointNodeNameAttr: test.Nodename, + model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.K8sVersionAttr: test.PackageVersion, + } +} + +func getAttrs1() map[string]string { + return map[string]string{ + model.ClusterIdAttr: test.ClusterId1, + model.ClusterSetIdAttr: test.ClusterSet, + model.EndpointIpv4Attr: test.EndptIp1, + model.EndpointPortAttr: test.PortStr1, + model.EndpointPortNameAttr: test.PortName1, + model.EndpointProtocolAttr: test.Protocol1, + model.EndpointReadyAttr: test.EndptReadyTrue, + model.ServicePortNameAttr: test.PortName1, + model.ServicePortAttr: test.ServicePortStr1, + model.ServiceProtocolAttr: test.Protocol1, + model.ServiceTargetPortAttr: test.PortStr1, + model.ServiceTypeAttr: test.SvcType, + model.EndpointHostnameAttr: test.Hostname, + model.EndpointNodeNameAttr: test.Nodename, + model.ServiceExportCreationAttr: strconv.FormatInt(test.SvcExportCreationTimestamp, 10), + model.K8sVersionAttr: test.PackageVersion, + } +} diff --git a/pkg/cloudmap/operation_collector.go b/pkg/cloudmap/operation_collector.go deleted file mode 100644 index 93293be6..00000000 --- a/pkg/cloudmap/operation_collector.go +++ /dev/null @@ -1,84 +0,0 @@ -package cloudmap - -import ( - "sync" - - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" -) - -// OperationCollector collects a list of operation IDs asynchronously with thread safety. -type OperationCollector interface { - // Add calls an operation provider function to asynchronously collect operations to poll. - Add(operationProvider func() (operationId string, err error)) - - // Collect waits for all create operation results to be provided and returns a list of the successfully created operation IDs. - Collect() []string - - // GetStartTime returns the start time range to poll the collected operations. - GetStartTime() int64 - - // IsAllOperationsCreated returns true if all operations were created successfully. - IsAllOperationsCreated() bool -} - -type opCollector struct { - log common.Logger - opChan chan opResult - wg sync.WaitGroup - startTime int64 - createOpsSuccess bool -} - -type opResult struct { - opId string - err error -} - -func NewOperationCollector() OperationCollector { - return &opCollector{ - log: common.NewLogger("cloudmap"), - opChan: make(chan opResult), - startTime: Now(), - createOpsSuccess: true, - } -} - -func (opColl *opCollector) Add(opProvider func() (opId string, err error)) { - opColl.wg.Add(1) - go func() { - defer opColl.wg.Done() - - opId, opErr := opProvider() - opColl.opChan <- opResult{opId, opErr} - }() -} - -func (opColl *opCollector) Collect() []string { - opIds := make([]string, 0) - - // Run wait in separate go routine to unblock reading from the channel. - go func() { - opColl.wg.Wait() - close(opColl.opChan) - }() - - for op := range opColl.opChan { - if op.err != nil { - opColl.log.Info("could not create operation", "error", op.err) - opColl.createOpsSuccess = false - continue - } - - opIds = append(opIds, op.opId) - } - - return opIds -} - -func (opColl *opCollector) GetStartTime() int64 { - return opColl.startTime -} - -func (opColl *opCollector) IsAllOperationsCreated() bool { - return opColl.createOpsSuccess -} diff --git a/pkg/cloudmap/operation_collector_test.go b/pkg/cloudmap/operation_collector_test.go deleted file mode 100644 index cb6e6f5f..00000000 --- a/pkg/cloudmap/operation_collector_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package cloudmap - -import ( - "errors" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -const ( - op1 = "one" - op2 = "two" -) - -func TestOpCollector_HappyCase(t *testing.T) { - oc := NewOperationCollector() - oc.Add(func() (opId string, err error) { return op1, nil }) - oc.Add(func() (opId string, err error) { return op2, nil }) - - result := oc.Collect() - assert.True(t, oc.IsAllOperationsCreated()) - assert.Equal(t, 2, len(result)) - assert.Contains(t, result, op1) - assert.Contains(t, result, op2) -} - -func TestOpCollector_AllFail(t *testing.T) { - oc := NewOperationCollector() - oc.Add(func() (opId string, err error) { return op1, errors.New("fail one") }) - oc.Add(func() (opId string, err error) { return op2, errors.New("fail two") }) - - result := oc.Collect() - assert.False(t, oc.IsAllOperationsCreated()) - assert.Equal(t, 0, len(result)) -} - -func TestOpCollector_MixedSuccess(t *testing.T) { - oc := NewOperationCollector() - oc.Add(func() (opId string, err error) { return op1, errors.New("fail one") }) - oc.Add(func() (opId string, err error) { return op2, nil }) - - result := oc.Collect() - assert.False(t, oc.IsAllOperationsCreated()) - assert.Equal(t, []string{op2}, result) -} - -func TestOpCollector_GetStartTime(t *testing.T) { - oc1 := NewOperationCollector() - time.Sleep(time.Second) - oc2 := NewOperationCollector() - - assert.Equal(t, oc1.GetStartTime(), oc1.GetStartTime(), "Start time should not change") - assert.NotEqual(t, oc1.GetStartTime(), oc2.GetStartTime(), "Start time should reflect instantiation") - assert.Less(t, oc1.GetStartTime(), oc2.GetStartTime(), - "Start time should increase for later instantiations") -} diff --git a/pkg/cloudmap/operation_poller.go b/pkg/cloudmap/operation_poller.go index aacf4a3c..d62278d5 100644 --- a/pkg/cloudmap/operation_poller.go +++ b/pkg/cloudmap/operation_poller.go @@ -2,8 +2,8 @@ package cloudmap import ( "context" - "errors" - "strconv" + "fmt" + "sync" "time" "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" @@ -14,150 +14,116 @@ import ( const ( // Interval between each getOperation call. - defaultOperationPollInterval = 3 * time.Second + defaultOperationPollInterval = 2 * time.Second // Time until we stop polling the operation - defaultOperationPollTimeout = 5 * time.Minute + defaultOperationPollTimeout = 1 * time.Minute operationPollTimoutErrorMessage = "timed out while polling operations" ) -// OperationPoller polls a list operations for a terminal status. +// OperationPoller polls a list operations for a terminal status type OperationPoller interface { - // Poll monitors operations until they reach terminal state. - Poll(ctx context.Context) error -} + // Submit operations to async poll + Submit(ctx context.Context, opProvider func() (opId string, err error)) -type operationPoller struct { - log common.Logger - sdApi ServiceDiscoveryApi - timeout time.Duration - - opIds []string - svcId string - opType types.OperationType - start int64 -} + // Poll operations for a terminal state + Poll(ctx context.Context, opId string) (*types.Operation, error) -func newOperationPoller(sdApi ServiceDiscoveryApi, svcId string, opIds []string, startTime int64) operationPoller { - return operationPoller{ - log: common.NewLogger("cloudmap"), - sdApi: sdApi, - timeout: defaultOperationPollTimeout, + // Await waits for all operation results from async poll + Await() (err error) +} - opIds: opIds, - svcId: svcId, - start: startTime, - } +type operationPoller struct { + log common.Logger + sdApi ServiceDiscoveryApi + opChan chan opResult + waitGroup sync.WaitGroup + pollInterval time.Duration + pollTimeout time.Duration } -// NewRegisterInstancePoller creates a new operation poller for register instance operations. -func NewRegisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int64) OperationPoller { - poller := newOperationPoller(sdApi, serviceId, opIds, startTime) - poller.opType = types.OperationTypeRegisterInstance - return &poller +type opResult struct { + opId string + err error } -// NewDeregisterInstancePoller creates a new operation poller for de-register instance operations. -func NewDeregisterInstancePoller(sdApi ServiceDiscoveryApi, serviceId string, opIds []string, startTime int64) OperationPoller { - poller := newOperationPoller(sdApi, serviceId, opIds, startTime) - poller.opType = types.OperationTypeDeregisterInstance - return &poller +// NewOperationPoller creates a new operation poller +func NewOperationPoller(sdApi ServiceDiscoveryApi) OperationPoller { + return NewOperationPollerWithConfig(defaultOperationPollInterval, defaultOperationPollTimeout, sdApi) } -func (opPoller *operationPoller) Poll(ctx context.Context) (err error) { - if len(opPoller.opIds) == 0 { - opPoller.log.Info("no operations to poll") - return nil +// NewOperationPollerWithConfig creates a new operation poller +func NewOperationPollerWithConfig(pollInterval, pollTimeout time.Duration, sdApi ServiceDiscoveryApi) OperationPoller { + return &operationPoller{ + log: common.NewLogger("cloudmap", "OperationPoller"), + sdApi: sdApi, + opChan: make(chan opResult), + pollInterval: pollInterval, + pollTimeout: pollTimeout, } +} - err = wait.Poll(defaultOperationPollInterval, opPoller.timeout, func() (done bool, err error) { - opPoller.log.Info("polling operations", "operations", opPoller.opIds) +func (p *operationPoller) Submit(ctx context.Context, opProvider func() (opId string, err error)) { + p.waitGroup.Add(1) - sdOps, err := opPoller.sdApi.ListOperations(ctx, opPoller.buildFilters()) + // Poll for the operation in a separate go routine + go func() { + // Indicate the polling done i.e. decrement the WaitGroup counter when the goroutine returns + defer p.waitGroup.Done() - if err != nil { - return true, err + opId, err := opProvider() + // Poll for the operationId if the provider doesn't throw error + if err == nil { + _, err = p.Poll(ctx, opId) } - failedOps := make([]string, 0) + p.opChan <- opResult{opId: opId, err: err} + }() +} - for _, pollOp := range opPoller.opIds { - status, hasVal := sdOps[pollOp] - if !hasVal { - // polled operation not terminal - return false, nil - } +func (p *operationPoller) Poll(ctx context.Context, opId string) (op *types.Operation, err error) { + // poll tries a condition func until it returns true, an error, or the timeout is reached. + err = wait.Poll(p.pollInterval, p.pollTimeout, func() (done bool, err error) { + p.log.Info("polling operation", "opId", opId) - if status == types.OperationStatusFail { - failedOps = append(failedOps, pollOp) - } + op, err = p.sdApi.GetOperation(ctx, opId) + if err != nil { + return true, err } - if len(failedOps) != 0 { - for _, failedOp := range failedOps { - opPoller.log.Info("operation failed", "failedOp", failedOp, "reason", opPoller.getFailedOpReason(ctx, failedOp)) - } - return true, errors.New("operation failure") + switch op.Status { + case types.OperationStatusSuccess: + return true, nil + case types.OperationStatusFail: + return true, fmt.Errorf("operation failed, opId: %s, reason: %s", opId, aws.ToString(op.ErrorMessage)) + default: + return false, nil } - - opPoller.log.Info("operations completed successfully") - return true, nil }) - if err == wait.ErrWaitTimeout { - return errors.New(operationPollTimoutErrorMessage) - } - - return err -} - -func (opPoller *operationPoller) buildFilters() []types.OperationFilter { - svcFilter := types.OperationFilter{ - Name: types.OperationFilterNameServiceId, - Values: []string{opPoller.svcId}, + err = fmt.Errorf("%s, opId: %s", operationPollTimoutErrorMessage, opId) } - statusFilter := types.OperationFilter{ - Name: types.OperationFilterNameStatus, - Condition: types.FilterConditionIn, - Values: []string{ - string(types.OperationStatusFail), - string(types.OperationStatusSuccess)}, - } - typeFilter := types.OperationFilter{ - Name: types.OperationFilterNameType, - Values: []string{string(opPoller.opType)}, - } - - timeFilter := types.OperationFilter{ - Name: types.OperationFilterNameUpdateDate, - Condition: types.FilterConditionBetween, - Values: []string{ - Itoa(opPoller.start), - // Add one minute to end range in case op updates while list request is in flight - Itoa(Now() + 60000), - }, - } - - return []types.OperationFilter{svcFilter, statusFilter, typeFilter, timeFilter} + return op, err } -// getFailedOpReason returns operation error message, which is not available in ListOperations response -func (opPoller *operationPoller) getFailedOpReason(ctx context.Context, opId string) string { - op, err := opPoller.sdApi.GetOperation(ctx, opId) - - if err != nil { - return "failed to retrieve operation failure reason" +func (p *operationPoller) Await() (err error) { + // Run wait in separate go routine to unblock reading from the channel. + go func() { + // Block till the polling done i.e. WaitGroup counter is zero, and then close the channel + p.waitGroup.Wait() + close(p.opChan) + }() + + for res := range p.opChan { + if res.err != nil { + p.log.Error(res.err, "operation failed", "opId", res.opId) + err = common.Wrap(err, res.err) + } else { + p.log.Info("operations completed successfully", "opId", res.opId) + } } - return aws.ToString(op.ErrorMessage) -} -func Itoa(i int64) string { - return strconv.FormatInt(i, 10) -} - -// Now returns current time with milliseconds, as used by operation filter UPDATE_DATE field -func Now() int64 { - return time.Now().UnixNano() / 1000000 + return err } diff --git a/pkg/cloudmap/operation_poller_test.go b/pkg/cloudmap/operation_poller_test.go index 08bcc7c6..87ec236d 100644 --- a/pkg/cloudmap/operation_poller_test.go +++ b/pkg/cloudmap/operation_poller_test.go @@ -2,224 +2,165 @@ package cloudmap import ( "context" - "errors" - "strconv" + "fmt" "testing" "time" cloudmapMock "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/pkg/cloudmap" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/common" - "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" ) -func TestOperationPoller_HappyCases(t *testing.T) { +const ( + op1 = "one" + op2 = "two" + op3 = "three" + interval = 100 * time.Millisecond + timeout = 500 * time.Millisecond +) + +func TestOperationPoller_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) - pollerTypes := []struct { - constructor func() OperationPoller - expectedOpType types.OperationType - }{ - { - constructor: func() OperationPoller { - return NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) - }, - expectedOpType: types.OperationTypeRegisterInstance, - }, - { - constructor: func() OperationPoller { - return NewDeregisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) - }, - expectedOpType: types.OperationTypeDeregisterInstance, - }, - } + op1First := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opSubmitted(), nil) + op1Second := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opPending(), nil) + op1Third := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opSuccess(), nil) + gomock.InOrder(op1First, op1Second, op1Third) - for _, pollerType := range pollerTypes { - p := pollerType.constructor() - - var firstEnd int - - sdApi.EXPECT(). - ListOperations(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, filters []types.OperationFilter) (map[string]types.OperationStatus, error) { - assert.Contains(t, filters, - types.OperationFilter{ - Name: types.OperationFilterNameServiceId, - Values: []string{test.SvcId}, - }) - assert.Contains(t, filters, - types.OperationFilter{ - Name: types.OperationFilterNameStatus, - Condition: types.FilterConditionIn, - - Values: []string{ - string(types.OperationStatusFail), - string(types.OperationStatusSuccess)}, - }) - assert.Contains(t, filters, - types.OperationFilter{ - Name: types.OperationFilterNameType, - Values: []string{string(pollerType.expectedOpType)}, - }) - - timeFilter := findUpdateDateFilter(t, filters) - assert.NotNil(t, timeFilter) - assert.Equal(t, types.FilterConditionBetween, timeFilter.Condition) - assert.Equal(t, 2, len(timeFilter.Values)) - - filterStart, _ := strconv.Atoi(timeFilter.Values[0]) - assert.Equal(t, test.OpStart, filterStart) - - firstEnd, _ = strconv.Atoi(timeFilter.Values[1]) - - return map[string]types.OperationStatus{}, nil - }) - - sdApi.EXPECT(). - ListOperations(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, filters []types.OperationFilter) (map[string]types.OperationStatus, error) { - timeFilter := findUpdateDateFilter(t, filters) - secondEnd, _ := strconv.Atoi(timeFilter.Values[1]) - assert.Greater(t, secondEnd, firstEnd, - "Filter time frame for operations must increase between invocations of ListOperations") - - return map[string]types.OperationStatus{ - test.OpId1: types.OperationStatusSuccess, - test.OpId2: types.OperationStatusSuccess, - }, nil - }) - - err := p.Poll(context.TODO()) - - assert.Nil(t, err) - } -} + op2First := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opPending(), nil) + op2Second := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opSuccess(), nil) + gomock.InOrder(op2First, op2Second) -func TestOperationPoller_PollEmpty(t *testing.T) { - mockController := gomock.NewController(t) - defer mockController.Finish() + sdApi.EXPECT().GetOperation(gomock.Any(), op3).Return(opSuccess(), nil) - sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) + op := NewOperationPollerWithConfig(interval, timeout, sdApi) + op.Submit(context.TODO(), func() (opId string, err error) { return op1, nil }) + op.Submit(context.TODO(), func() (opId string, err error) { return op2, nil }) + op.Submit(context.TODO(), func() (opId string, err error) { return op3, nil }) - p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{}, test.OpStart) - err := p.Poll(context.TODO()) - assert.Nil(t, err) + result := op.Await() + assert.Nil(t, result) } -func TestOperationPoller_PollFailure(t *testing.T) { +func TestOperationPoller_AllFail(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) - p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) - - pollErr := errors.New("error polling operations") - - sdApi.EXPECT(). - ListOperations(gomock.Any(), gomock.Any()). - Return(map[string]types.OperationStatus{}, pollErr) - - err := p.Poll(context.TODO()) - assert.Equal(t, pollErr, err) + op1First := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opSubmitted(), nil) + op1Second := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opPending(), nil) + op1Third := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opFailed(), nil) + gomock.InOrder(op1First, op1Second, op1Third) + + op2First := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opSubmitted(), nil) + op2Second := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opFailed(), nil) + gomock.InOrder(op2First, op2Second) + + op := NewOperationPollerWithConfig(interval, timeout, sdApi) + op.Submit(context.TODO(), func() (opId string, err error) { return op1, nil }) + op.Submit(context.TODO(), func() (opId string, err error) { return op2, nil }) + unknown := "failed to reg error" + op.Submit(context.TODO(), func() (opId string, err error) { + return "", fmt.Errorf(unknown) + }) + + err := op.Await() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), op1) + assert.Contains(t, err.Error(), op2) + assert.Contains(t, err.Error(), unknown) } -func TestOperationPoller_PollOpFailure(t *testing.T) { +func TestOperationPoller_Mixed(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) - p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) - - sdApi.EXPECT(). - ListOperations(gomock.Any(), gomock.Any()). - Return( - map[string]types.OperationStatus{ - test.OpId1: types.OperationStatusSuccess, - test.OpId2: types.OperationStatusFail, - }, nil) + op1First := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opSubmitted(), nil) + op1Second := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opPending(), nil) + op1Third := sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opFailed(), nil) + gomock.InOrder(op1First, op1Second, op1Third) - opErr := "operation failure message" + op2First := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opSubmitted(), nil) + op2Second := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opPending(), nil) + op2Third := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opSuccess(), nil) + gomock.InOrder(op2First, op2Second, op2Third) - sdApi.EXPECT(). - GetOperation(gomock.Any(), test.OpId2). - Return(&types.Operation{ErrorMessage: &opErr}, nil) + op := NewOperationPollerWithConfig(interval, timeout, sdApi) + op.Submit(context.TODO(), func() (opId string, err error) { return op1, nil }) + op.Submit(context.TODO(), func() (opId string, err error) { return op2, nil }) - err := p.Poll(context.TODO()) - assert.Equal(t, "operation failure", err.Error()) + err := op.Await() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), op1) + assert.NotContains(t, err.Error(), op2) } -func TestOperationPoller_PollOpFailureAndMessageFailure(t *testing.T) { +func TestOperationPoller_Timeout(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) - p := NewRegisterInstancePoller(sdApi, test.SvcId, []string{test.OpId1, test.OpId2}, test.OpStart) + sdApi.EXPECT().GetOperation(gomock.Any(), op1).Return(opPending(), nil).AnyTimes() - sdApi.EXPECT(). - ListOperations(gomock.Any(), gomock.Any()). - Return( - map[string]types.OperationStatus{ - test.OpId1: types.OperationStatusFail, - test.OpId2: types.OperationStatusSuccess, - }, nil) + op2First := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opPending(), nil) + op2Second := sdApi.EXPECT().GetOperation(gomock.Any(), op2).Return(opSuccess(), nil) + gomock.InOrder(op2First, op2Second) - sdApi.EXPECT(). - GetOperation(gomock.Any(), test.OpId1). - Return(nil, errors.New("failed to retrieve operation failure reason")) + op := NewOperationPollerWithConfig(interval, timeout, sdApi) + op.Submit(context.TODO(), func() (opId string, err error) { return op1, nil }) + op.Submit(context.TODO(), func() (opId string, err error) { return op2, nil }) - err := p.Poll(context.TODO()) - assert.Equal(t, "operation failure", err.Error()) + err := op.Await() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), op1) + assert.Contains(t, err.Error(), operationPollTimoutErrorMessage) + assert.NotContains(t, err.Error(), op2) } -func TestOperationPoller_PollTimeout(t *testing.T) { +func TestOperationPoller_Poll_HappyCase(t *testing.T) { mockController := gomock.NewController(t) defer mockController.Finish() sdApi := cloudmapMock.NewMockServiceDiscoveryApi(mockController) - p := operationPoller{ - log: common.NewLoggerWithLogr(testr.New(t)), - sdApi: sdApi, - timeout: 2 * time.Millisecond, - opIds: []string{test.OpId1, test.OpId2}, - } - - sdApi.EXPECT(). - ListOperations(gomock.Any(), gomock.Any()). - Return( - map[string]types.OperationStatus{}, nil) + sdApi.EXPECT().GetOperation(context.TODO(), op1).Return(opPending(), nil) + sdApi.EXPECT().GetOperation(context.TODO(), op1).Return(opSuccess(), nil) - err := p.Poll(context.TODO()) - assert.Equal(t, operationPollTimoutErrorMessage, err.Error()) + op := NewOperationPollerWithConfig(interval, timeout, sdApi) + _, err := op.Poll(context.TODO(), op1) + assert.Nil(t, err) } -func TestItoa(t *testing.T) { - assert.Equal(t, "7", Itoa(7)) +func opPending() *types.Operation { + return &types.Operation{ + Status: types.OperationStatusPending, + } } -func TestNow(t *testing.T) { - now1 := Now() - time.Sleep(time.Millisecond * 5) - now2 := Now() - assert.Greater(t, now2, now1) +func opFailed() *types.Operation { + return &types.Operation{ + Status: types.OperationStatusFail, + ErrorMessage: aws.String("fail"), + } } -func findUpdateDateFilter(t *testing.T, filters []types.OperationFilter) *types.OperationFilter { - for _, filter := range filters { - if filter.Name == types.OperationFilterNameUpdateDate { - return &filter - } +func opSubmitted() *types.Operation { + return &types.Operation{ + Status: types.OperationStatusSubmitted, } +} - t.Errorf("Missing update date filter") - return nil +func opSuccess() *types.Operation { + return &types.Operation{ + Status: types.OperationStatusSuccess, + } } diff --git a/pkg/common/ratelimiter.go b/pkg/common/ratelimiter.go new file mode 100644 index 00000000..bae83468 --- /dev/null +++ b/pkg/common/ratelimiter.go @@ -0,0 +1,43 @@ +package common + +import ( + "context" + "fmt" + + "golang.org/x/time/rate" +) + +const ( + ListNamespaces Event = "ListNamespaces" + ListServices Event = "ListServices" + GetOperation Event = "GetOperation" + RegisterInstance Event = "RegisterInstance" + DeregisterInstance Event = "DeregisterInstance" +) + +type Event string + +type RateLimiter struct { + rateLimiters map[Event]*rate.Limiter +} + +// NewDefaultRateLimiter returns the rate limiters with the default limits for the AWS CloudMap's API calls +func NewDefaultRateLimiter() RateLimiter { + return RateLimiter{rateLimiters: map[Event]*rate.Limiter{ + // Below are the default limits for the AWS CloudMap's APIs + // TODO: make it customizable in the future + ListNamespaces: rate.NewLimiter(rate.Limit(1), 5), // 1 ListNamespaces API calls per second + ListServices: rate.NewLimiter(rate.Limit(2), 10), // 2 ListServices API calls per second + GetOperation: rate.NewLimiter(rate.Limit(100), 200), // 100 GetOperation API calls per second + RegisterInstance: rate.NewLimiter(rate.Limit(50), 100), // 50 RegisterInstance API calls per second + DeregisterInstance: rate.NewLimiter(rate.Limit(50), 100), // 50 DeregisterInstance API calls per second + }} +} + +// Wait blocks until limit permits an event to happen. It returns an error if the Context is canceled, or the expected wait time exceeds the Context's Deadline. +func (r RateLimiter) Wait(ctx context.Context, event Event) error { + if limiter, ok := r.rateLimiters[event]; ok { + return limiter.Wait(ctx) + } + return fmt.Errorf("event %s not found in the list of limiters", event) +} diff --git a/pkg/common/ratelimiter_test.go b/pkg/common/ratelimiter_test.go new file mode 100644 index 00000000..f1b6afb1 --- /dev/null +++ b/pkg/common/ratelimiter_test.go @@ -0,0 +1,64 @@ +package common + +import ( + "context" + "testing" +) + +func TestRateLimiter_Wait(t *testing.T) { + type fields struct { + RateLimiter RateLimiter + } + type args struct { + ctx context.Context + event Event + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "happy", + fields: fields{RateLimiter: NewDefaultRateLimiter()}, + args: args{ + ctx: context.TODO(), + event: ListServices, + }, + wantErr: false, + }, + { + name: "not_found", + fields: fields{RateLimiter: NewDefaultRateLimiter()}, + args: args{ + ctx: context.TODO(), + event: "test", + }, + wantErr: true, + }, + { + name: "error_ctx_canceled", + fields: fields{RateLimiter: NewDefaultRateLimiter()}, + args: args{ + ctx: ctxCanceled(context.TODO()), + event: ListNamespaces, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := tt.fields.RateLimiter + if err := r.Wait(tt.args.ctx, tt.args.event); (err != nil) != tt.wantErr { + t.Errorf("Wait() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func ctxCanceled(ctx context.Context) context.Context { + ret, cancel := context.WithCancel(ctx) + defer cancel() // cancel after function call + return ret +} From 682157b85b462a3d88ab1b1ffd6fea73e0cb83d4 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Tue, 15 Nov 2022 10:57:27 -0800 Subject: [PATCH 128/163] Remove non-implemented Ginkgo BDD test (#247) * Remove suite_test.go with Ginkgo BDD skeleton code. This file was auto-generated by kubebuilder, but no tests were added as we are relying on integration tests over BDD. Also, Upgrade golang.org/x/time to the latest version. * Remove unnecessary declaration * Remove envtest assests and config required for Ginkgo tests --- Makefile | 26 ++------ go.mod | 7 +- go.sum | 5 +- pkg/controllers/multicluster/suite_test.go | 75 ---------------------- 4 files changed, 8 insertions(+), 105 deletions(-) delete mode 100644 pkg/controllers/multicluster/suite_test.go diff --git a/Makefile b/Makefile index b7eb6900..53a1b767 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,3 @@ -# Silence update recommendations for Ginkgo -export ACK_GINKGO_DEPRECATIONS:=1.16.5 - GIT_COMMIT:=$(shell git describe --dirty --always) GIT_TAG:=$(shell git describe --dirty --always --tags) PKG:=github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version @@ -68,20 +65,10 @@ lint: golangci-lint ## Run linter goimports: goimports-bin ## run goimports updating files in place $(GOIMPORTS) -w . -# Run tests -ENVTEST_ASSETS_DIR=$(shell pwd)/testbin -KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)" - .PHONY: test -test: manifests generate generate-mocks fmt vet test-setup goimports lint ## Run tests - KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out -covermode=atomic - -test-setup: setup-envtest ## Ensure test environment has been downloaded -ifneq ($(shell test -d $(ENVTEST_ASSETS_DIR); echo $$?), 0) - @echo Setting up K8s test environment... - mkdir -p ${ENVTEST_ASSETS_DIR} - $(ENVTEST) use 1.24.x --bin-dir $(ENVTEST_ASSETS_DIR) -endif +test: manifests generate generate-mocks fmt vet goimports lint ## Run tests + @echo Testing... + go test ./... -coverprofile=cover.out -covermode=atomic kind-integration-suite: ## Provision and run integration tests with cleanup make kind-integration-setup && \ @@ -133,8 +120,7 @@ docker-push: ## Push docker image with the manager. clean: @echo Cleaning... go clean - if test -d $(ENVTEST_ASSETS_DIR) ; then chmod -R +w $(ENVTEST_ASSETS_DIR) ; fi - rm -rf $(MOCKS_DESTINATION)/ bin/ $(ENVTEST_ASSETS_DIR)/ cover.out + rm -rf $(MOCKS_DESTINATION)/ bin/ cover.out ##@ Deployment @@ -172,10 +158,6 @@ KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.5) -ENVTEST = $(shell pwd)/bin/setup-envtest -setup-envtest: ## Download setup-envtest - $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) - MOCKGEN = $(shell pwd)/bin/mockgen mockgen: ## Download mockgen $(call go-get-tool,$(MOCKGEN),github.com/golang/mock/mockgen@v1.6.0) diff --git a/go.mod b/go.mod index 900d7e33..574ee7b3 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,9 @@ require ( github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 - github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.20.2 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.1 - golang.org/x/time v0.1.0 + golang.org/x/time v0.2.0 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.2 @@ -65,7 +63,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/gomega v1.20.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -85,7 +83,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.24.2 // indirect diff --git a/go.sum b/go.sum index 048f458b..47ed6694 100644 --- a/go.sum +++ b/go.sum @@ -406,7 +406,6 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -775,8 +774,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= +golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/controllers/multicluster/suite_test.go b/pkg/controllers/multicluster/suite_test.go deleted file mode 100644 index 6ea09944..00000000 --- a/pkg/controllers/multicluster/suite_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package controllers - -import ( - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - multiclusterv1alpha1 "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/apis/multicluster/v1alpha1" - // +kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var _ *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecsWithDefaultAndCustomReporters(t, - "Controller Suite", - []Reporter{printer.NewlineReporter{}}) -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - - // TODO: Add CRDs for about.k8s.io_clusterproperties & add patch - - crds := []string{ - filepath.Join("..", "..", "..", "config", "crd", "bases", "multicluster.x-k8s.io_serviceexports.yaml"), - filepath.Join("..", "..", "..", "config", "crd", "bases", "multicluster.x-k8s.io_serviceimports.yaml"), - } - - testEnv = &envtest.Environment{ - CRDDirectoryPaths: crds, - ErrorIfCRDPathMissing: true, - } - - cfg, err := testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = multiclusterv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - err = multiclusterv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - -}, 60) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) From acd2c061d0c2e01d5d985b37105137a5a82370cd Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Tue, 15 Nov 2022 10:57:42 -0800 Subject: [PATCH 129/163] Split the endpoints cache and default cache (namespace and service) so that relation operations are independent and unblocking. Increase the size of the cache to 2048 accommodate large namespaces. (#249) --- pkg/cloudmap/cache.go | 55 +++++++++++++++++++------------------- pkg/cloudmap/cache_test.go | 11 ++++---- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/pkg/cloudmap/cache.go b/pkg/cloudmap/cache.go index a808938a..8c75b83b 100644 --- a/pkg/cloudmap/cache.go +++ b/pkg/cloudmap/cache.go @@ -11,11 +11,10 @@ import ( ) const ( - nsKey = "ns-map" - svcKeyPrefix = "svc-map" - endptsKeyPrefix = "endpts" + nsKey = "ns-map" + svcKeyPrefix = "svc-map" - defaultCacheSize = 1024 + defaultCacheSize = 2048 defaultNsTTL = 10 * time.Second defaultSvcTTL = 10 * time.Second defaultEndptTTL = 5 * time.Second @@ -34,9 +33,10 @@ type ServiceDiscoveryClientCache interface { } type sdCache struct { - log common.Logger - cache *cache.LRUExpireCache - config *SdCacheConfig + log common.Logger + defaultCache *cache.LRUExpireCache + endpointsCache *cache.LRUExpireCache + config *SdCacheConfig } type SdCacheConfig struct { @@ -47,9 +47,10 @@ type SdCacheConfig struct { func NewServiceDiscoveryClientCache(cacheConfig *SdCacheConfig) ServiceDiscoveryClientCache { return &sdCache{ - log: common.NewLogger("cloudmap"), - cache: cache.NewLRUExpireCache(defaultCacheSize), - config: cacheConfig, + log: common.NewLogger("cloudmap"), + defaultCache: cache.NewLRUExpireCache(defaultCacheSize), + endpointsCache: cache.NewLRUExpireCache(defaultCacheSize), + config: cacheConfig, } } @@ -63,7 +64,7 @@ func NewDefaultServiceDiscoveryClientCache() ServiceDiscoveryClientCache { } func (sdCache *sdCache) GetNamespaceMap() (namespaceMap map[string]*model.Namespace, found bool) { - entry, exists := sdCache.cache.Get(nsKey) + entry, exists := sdCache.defaultCache.Get(nsKey) if !exists { return nil, false } @@ -71,7 +72,7 @@ func (sdCache *sdCache) GetNamespaceMap() (namespaceMap map[string]*model.Namesp namespaceMap, ok := entry.(map[string]*model.Namespace) if !ok { sdCache.log.Error(errors.New("failed to retrieve namespaceMap from cache"), "") - sdCache.cache.Remove(nsKey) + sdCache.defaultCache.Remove(nsKey) return nil, false } @@ -79,25 +80,25 @@ func (sdCache *sdCache) GetNamespaceMap() (namespaceMap map[string]*model.Namesp } func (sdCache *sdCache) CacheNamespaceMap(namespaces map[string]*model.Namespace) { - sdCache.cache.Add(nsKey, namespaces, sdCache.config.NsTTL) + sdCache.defaultCache.Add(nsKey, namespaces, sdCache.config.NsTTL) } func (sdCache *sdCache) EvictNamespaceMap() { - sdCache.cache.Remove(nsKey) + sdCache.defaultCache.Remove(nsKey) } func (sdCache *sdCache) GetServiceIdMap(nsName string) (serviceIdMap map[string]string, found bool) { key := sdCache.buildSvcKey(nsName) - entry, exists := sdCache.cache.Get(key) + entry, exists := sdCache.defaultCache.Get(key) if !exists { return nil, false } serviceIdMap, ok := entry.(map[string]string) if !ok { - sdCache.log.Error(errors.New("failed to retrieve service IDs from cache"), "", - "nsName", nsName) - sdCache.cache.Remove(key) + err := fmt.Errorf("failed to retrieve service IDs from cache") + sdCache.log.Error(err, err.Error(), "namespace", nsName) + sdCache.defaultCache.Remove(key) return nil, false } @@ -106,26 +107,26 @@ func (sdCache *sdCache) GetServiceIdMap(nsName string) (serviceIdMap map[string] func (sdCache *sdCache) CacheServiceIdMap(nsName string, serviceIdMap map[string]string) { key := sdCache.buildSvcKey(nsName) - sdCache.cache.Add(key, serviceIdMap, sdCache.config.SvcTTL) + sdCache.defaultCache.Add(key, serviceIdMap, sdCache.config.SvcTTL) } func (sdCache *sdCache) EvictServiceIdMap(nsName string) { key := sdCache.buildSvcKey(nsName) - sdCache.cache.Remove(key) + sdCache.defaultCache.Remove(key) } func (sdCache *sdCache) GetEndpoints(nsName string, svcName string) (endpts []*model.Endpoint, found bool) { key := sdCache.buildEndptsKey(nsName, svcName) - entry, exists := sdCache.cache.Get(key) + entry, exists := sdCache.endpointsCache.Get(key) if !exists { return nil, false } endpts, ok := entry.([]*model.Endpoint) if !ok { - sdCache.log.Error(errors.New("failed to retrieve endpoints from cache"), "", - "ns", "nsName", "svc", svcName) - sdCache.cache.Remove(key) + err := fmt.Errorf("failed to retrieve endpoints from cache") + sdCache.log.Error(err, err.Error(), "namespace", nsName, "service", svcName) + sdCache.endpointsCache.Remove(key) return nil, false } @@ -134,12 +135,12 @@ func (sdCache *sdCache) GetEndpoints(nsName string, svcName string) (endpts []*m func (sdCache *sdCache) CacheEndpoints(nsName string, svcName string, endpts []*model.Endpoint) { key := sdCache.buildEndptsKey(nsName, svcName) - sdCache.cache.Add(key, endpts, sdCache.config.EndptTTL) + sdCache.endpointsCache.Add(key, endpts, sdCache.config.EndptTTL) } func (sdCache *sdCache) EvictEndpoints(nsName string, svcName string) { key := sdCache.buildEndptsKey(nsName, svcName) - sdCache.cache.Remove(key) + sdCache.endpointsCache.Remove(key) } func (sdCache *sdCache) buildSvcKey(nsName string) (cacheKey string) { @@ -147,5 +148,5 @@ func (sdCache *sdCache) buildSvcKey(nsName string) (cacheKey string) { } func (sdCache *sdCache) buildEndptsKey(nsName string, svcName string) string { - return fmt.Sprintf("%s:%s:%s", endptsKeyPrefix, nsName, svcName) + return fmt.Sprintf("%s:%s", nsName, svcName) } diff --git a/pkg/cloudmap/cache_test.go b/pkg/cloudmap/cache_test.go index 89968f42..a9231bd7 100644 --- a/pkg/cloudmap/cache_test.go +++ b/pkg/cloudmap/cache_test.go @@ -59,7 +59,7 @@ func TestServiceDiscoveryClientCacheGetNamespaceMap_NotFound(t *testing.T) { func TestServiceDiscoveryClientCacheGetNamespaceMap_Corrupt(t *testing.T) { sdc := getCacheImpl(t) - sdc.cache.Add(nsKey, &model.Plan{}, time.Minute) + sdc.defaultCache.Add(nsKey, &model.Plan{}, time.Minute) nsMap, found := sdc.GetNamespaceMap() assert.False(t, found) @@ -99,7 +99,7 @@ func TestServiceDiscoveryClientCacheGetServiceIdMap_NotFound(t *testing.T) { func TestServiceDiscoveryClientCacheGetServiceIdMap_Corrupt(t *testing.T) { sdc := getCacheImpl(t) - sdc.cache.Add(sdc.buildSvcKey(test.HttpNsName), &model.Plan{}, time.Minute) + sdc.defaultCache.Add(sdc.buildSvcKey(test.HttpNsName), &model.Plan{}, time.Minute) svcIdMap, found := sdc.GetServiceIdMap(test.HttpNsName) assert.False(t, found) @@ -137,7 +137,7 @@ func TestServiceDiscoveryClientCacheGetEndpoints_NotFound(t *testing.T) { func TestServiceDiscoveryClientCacheGetEndpoints_Corrupt(t *testing.T) { sdc := getCacheImpl(t) - sdc.cache.Add(sdc.buildEndptsKey(test.HttpNsName, test.SvcName), &model.Plan{}, time.Minute) + sdc.defaultCache.Add(sdc.buildEndptsKey(test.HttpNsName, test.SvcName), &model.Plan{}, time.Minute) endpts, found := sdc.GetEndpoints(test.HttpNsName, test.SvcName) assert.False(t, found) @@ -156,7 +156,8 @@ func TestServiceDiscoveryClientEvictEndpoints(t *testing.T) { func getCacheImpl(t *testing.T) sdCache { return sdCache{ - log: common.NewLoggerWithLogr(testr.New(t)), - cache: cache.NewLRUExpireCache(defaultCacheSize), + log: common.NewLoggerWithLogr(testr.New(t)), + defaultCache: cache.NewLRUExpireCache(defaultCacheSize), + endpointsCache: cache.NewLRUExpireCache(defaultCacheSize), } } From c09ae3e9f5b6686eb8afe2dbccf2860f8a248df0 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Tue, 15 Nov 2022 10:57:55 -0800 Subject: [PATCH 130/163] Add deletecollection verb. To fix the error encountered with k8s DeleteAllOf operation on endpointslices.discovery.k8s.io. (#248) --- config/rbac/role.yaml | 1 + pkg/controllers/multicluster/cloudmap_controller.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8c546e77..5754c2e7 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -42,6 +42,7 @@ rules: verbs: - create - delete + - deletecollection - get - list - update diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index 85184a00..9a945ba8 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -32,7 +32,7 @@ type CloudMapReconciler struct { // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch // +kubebuilder:rbac:groups="",resources=services,verbs=create;get;list;watch;update;delete // +kubebuilder:rbac:groups=about.k8s.io,resources=clusterproperties,verbs=create;get;list;watch;update;patch;delete -// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;create;watch;update;delete +// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;create;watch;update;delete;deletecollection // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceimports,verbs=create;get;list;watch;update;patch;delete // Start implements manager.Runnable From 05c0a9711f99aa0bae1a117be2814931b4459d07 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Thu, 17 Nov 2022 00:34:45 -0800 Subject: [PATCH 131/163] Configure Concurrency for Intergration Tests (#250) * Update integration-test.yml * Update README.md (#72) --- .github/workflows/integration-test.yml | 3 +++ README.md | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 80054e88..ffd15990 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -3,6 +3,9 @@ on: push: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: integration-test: name: Run Integration Test diff --git a/README.md b/README.md index 7bbcf54e..3b1298ac 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/aws/aws-cloud-map-mcs-controller-for-k8s)](https://goreportcard.com/report/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) ## Introduction -The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. The current version `v0.3.0` is in *Alpha* phase, checkout the [Graduation Criteria](#graduation-criteria) for the next steps. +The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. We have detailed [step-by-step setup guide](https://aws.github.io/aws-cloud-map-mcs-controller-for-k8s/)! -We also have detailed [step-by-step setup guide](https://aws.github.io/aws-cloud-map-mcs-controller-for-k8s/)! +**NOTE: The current version `v0.3.0` is in *Alpha* phase, checkout the [Graduation Criteria](#graduation-criteria) for the next steps.** ## Installation From 48d0a5c78dfc7f83469d9fcfb8725bc5ab39bb07 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Mon, 28 Nov 2022 15:32:47 -0800 Subject: [PATCH 132/163] Add rate limiters for all the CloudMap's API calls. And optimize the rate limits of ListNamespaces. (#251) --- pkg/cloudmap/api.go | 15 +++++++++++++++ pkg/common/ratelimiter.go | 26 ++++++++++++++++---------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/pkg/cloudmap/api.go b/pkg/cloudmap/api.go index cb91f456..3ed865a7 100644 --- a/pkg/cloudmap/api.go +++ b/pkg/cloudmap/api.go @@ -117,6 +117,11 @@ func (sdApi *serviceDiscoveryApi) GetServiceIdMap(ctx context.Context, nsId stri } func (sdApi *serviceDiscoveryApi) DiscoverInstances(ctx context.Context, nsName string, svcName string, queryParameters map[string]string) (insts []types.HttpInstanceSummary, err error) { + err = sdApi.rateLimiter.Wait(ctx, common.DiscoverInstances) + if err != nil { + return nil, err + } + input := &sd.DiscoverInstancesInput{ NamespaceName: aws.String(nsName), ServiceName: aws.String(svcName), @@ -151,6 +156,11 @@ func (sdApi *serviceDiscoveryApi) GetOperation(ctx context.Context, opId string) } func (sdApi *serviceDiscoveryApi) CreateHttpNamespace(ctx context.Context, nsName string) (opId string, err error) { + err = sdApi.rateLimiter.Wait(ctx, common.CreateHttpNamespace) + if err != nil { + return "", err + } + output, err := sdApi.awsFacade.CreateHttpNamespace(ctx, &sd.CreateHttpNamespaceInput{ Name: &nsName, }) @@ -163,6 +173,11 @@ func (sdApi *serviceDiscoveryApi) CreateHttpNamespace(ctx context.Context, nsNam } func (sdApi *serviceDiscoveryApi) CreateService(ctx context.Context, namespace model.Namespace, svcName string) (svcId string, err error) { + err = sdApi.rateLimiter.Wait(ctx, common.CreateService) + if err != nil { + return "", err + } + var output *sd.CreateServiceOutput if namespace.Type == model.DnsPrivateNamespaceType { dnsConfig := sdApi.getDnsConfig() diff --git a/pkg/common/ratelimiter.go b/pkg/common/ratelimiter.go index bae83468..6d60f7c6 100644 --- a/pkg/common/ratelimiter.go +++ b/pkg/common/ratelimiter.go @@ -8,11 +8,14 @@ import ( ) const ( - ListNamespaces Event = "ListNamespaces" - ListServices Event = "ListServices" - GetOperation Event = "GetOperation" - RegisterInstance Event = "RegisterInstance" - DeregisterInstance Event = "DeregisterInstance" + ListNamespaces Event = "ListNamespaces" + ListServices Event = "ListServices" + GetOperation Event = "GetOperation" + DiscoverInstances Event = "DiscoverInstances" + CreateHttpNamespace Event = "CreateHttpNamespace" + CreateService Event = "CreateService" + RegisterInstance Event = "RegisterInstance" + DeregisterInstance Event = "DeregisterInstance" ) type Event string @@ -26,11 +29,14 @@ func NewDefaultRateLimiter() RateLimiter { return RateLimiter{rateLimiters: map[Event]*rate.Limiter{ // Below are the default limits for the AWS CloudMap's APIs // TODO: make it customizable in the future - ListNamespaces: rate.NewLimiter(rate.Limit(1), 5), // 1 ListNamespaces API calls per second - ListServices: rate.NewLimiter(rate.Limit(2), 10), // 2 ListServices API calls per second - GetOperation: rate.NewLimiter(rate.Limit(100), 200), // 100 GetOperation API calls per second - RegisterInstance: rate.NewLimiter(rate.Limit(50), 100), // 50 RegisterInstance API calls per second - DeregisterInstance: rate.NewLimiter(rate.Limit(50), 100), // 50 DeregisterInstance API calls per second + ListNamespaces: rate.NewLimiter(rate.Limit(0.5), 5), // 1 ListNamespaces API calls per second + ListServices: rate.NewLimiter(rate.Limit(2), 10), // 2 ListServices API calls per second + GetOperation: rate.NewLimiter(rate.Limit(100), 200), // 100 GetOperation API calls per second + DiscoverInstances: rate.NewLimiter(rate.Limit(500), 1000), // 500 DiscoverInstances API calls per second + CreateHttpNamespace: rate.NewLimiter(rate.Limit(0.5), 5), // 1 CreateHttpNamespace API calls per second + CreateService: rate.NewLimiter(rate.Limit(5), 50), // 5 CreateService API calls per second + RegisterInstance: rate.NewLimiter(rate.Limit(50), 100), // 50 RegisterInstance API calls per second + DeregisterInstance: rate.NewLimiter(rate.Limit(50), 100), // 50 DeregisterInstance API calls per second }} } From ba90126f7ee71d9990d0ce767cff4487b4762af5 Mon Sep 17 00:00:00 2001 From: Shardul Bansal <33334493+bansal19@users.noreply.github.com> Date: Mon, 28 Nov 2022 15:40:02 -0800 Subject: [PATCH 133/163] Update golangci-lint version (#252) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 53a1b767..7e3f0915 100644 --- a/Makefile +++ b/Makefile @@ -167,7 +167,7 @@ golangci-lint: ## Download golangci-lint ifneq ($(shell test -f $(GOLANGCI_LINT); echo $$?), 0) @echo Getting golangci-lint... @mkdir -p $(shell pwd)/bin - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.46.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell pwd)/bin v1.50.1 endif GOIMPORTS = $(shell pwd)/bin/goimports From ca0b562ddd6813d151118deb2877548cdc94e20f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 11:00:30 -0800 Subject: [PATCH 134/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.17.10 to 1.18.3 (#253) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.17.10 to 1.18.3. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.17.10...config/v1.18.3) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 574ee7b3..13f22685 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.1 - github.com/aws/aws-sdk-go-v2/config v1.17.10 + github.com/aws/aws-sdk-go-v2/config v1.18.3 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,7 +28,7 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.23 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect @@ -36,7 +36,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect github.com/aws/smithy-go v1.13.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 47ed6694..b1588741 100644 --- a/go.sum +++ b/go.sum @@ -76,10 +76,10 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= -github.com/aws/aws-sdk-go-v2/config v1.17.10 h1:zBy5QQ/mkvHElM1rygHPAzuH+sl8nsdSaxSWj0+rpdE= -github.com/aws/aws-sdk-go-v2/config v1.17.10/go.mod h1:/4np+UiJJKpWHN7Q+LZvqXYgyjgeXm5+lLfDI6TPZao= -github.com/aws/aws-sdk-go-v2/credentials v1.12.23 h1:LctvcJMIb8pxvk5hQhChpCu0WlU6oKQmcYb1HA4IZSA= -github.com/aws/aws-sdk-go-v2/credentials v1.12.23/go.mod h1:0awX9iRr/+UO7OwRQFpV1hNtXxOVuehpjVEzrIAYNcA= +github.com/aws/aws-sdk-go-v2/config v1.18.3 h1:3kfBKcX3votFX84dm00U8RGA1sCCh3eRMOGzg5dCWfU= +github.com/aws/aws-sdk-go-v2/config v1.18.3/go.mod h1:BYdrbeCse3ZnOD5+2/VE/nATOK8fEUpBtmPMdKSyhMU= +github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57HOo03ELodttmagk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= @@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWq github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 h1:KRAix/KHvjGODaHAMXnxRk9t0D+4IJVUuS/uwXxngXk= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.1/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 h1:60SJ4lhvn///8ygCzYy2l53bFW/Q15bVfyjyAWo6zuw= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From d68c76895e076783b1cbbf618ca1e61a62f4a144 Mon Sep 17 00:00:00 2001 From: Shardul Bansal <33334493+bansal19@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:23:19 -0800 Subject: [PATCH 135/163] Check cluster ID during service export (#255) Check cluster ID during service export --- .../multicluster/serviceexport_controller.go | 18 +++-- .../serviceexport_controller_test.go | 4 +- pkg/model/types.go | 9 +++ pkg/model/types_test.go | 71 +++++++++++++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index f9fd1700..dcd58978 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -85,15 +85,21 @@ func (r *ServiceExportReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } + clusterProperties, err := r.ClusterUtils.GetClusterProperties(ctx) + if err != nil { + r.Log.Error(err, "unable to retrieve ClusterId and ClusterSetId") + return ctrl.Result{}, err + } + // Check if the service export is marked to be deleted if isServiceExportMarkedForDelete { - return r.handleDelete(ctx, &serviceExport) + return r.handleDelete(ctx, clusterProperties.ClusterId(), &serviceExport) } - return r.handleUpdate(ctx, &serviceExport, &service) + return r.handleUpdate(ctx, clusterProperties.ClusterId(), &serviceExport, &service) } -func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExport *multiclusterv1alpha1.ServiceExport, service *v1.Service) (ctrl.Result, error) { +func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, clusterId string, serviceExport *multiclusterv1alpha1.ServiceExport, service *v1.Service) (ctrl.Result, error) { err := r.addFinalizerAndOwnerRef(ctx, serviceExport, service) if err != nil { return ctrl.Result{}, err @@ -114,7 +120,7 @@ func (r *ServiceExportReconciler) handleUpdate(ctx context.Context, serviceExpor // Compute diff between Cloud Map and K8s endpoints, and apply changes plan := model.Plan{ - Current: cmService.Endpoints, + Current: cmService.GetEndpoints(clusterId), Desired: endpoints, } changes := plan.CalculateChanges() @@ -186,7 +192,7 @@ func (r *ServiceExportReconciler) createOrGetCloudMapService(ctx context.Context return cmService, nil } -func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExport *multiclusterv1alpha1.ServiceExport) (ctrl.Result, error) { +func (r *ServiceExportReconciler) handleDelete(ctx context.Context, clusterId string, serviceExport *multiclusterv1alpha1.ServiceExport) (ctrl.Result, error) { if controllerutil.ContainsFinalizer(serviceExport, ServiceExportFinalizer) { r.Log.Info("removing service export", "namespace", serviceExport.Namespace, "name", serviceExport.Name) @@ -196,7 +202,7 @@ func (r *ServiceExportReconciler) handleDelete(ctx context.Context, serviceExpor return ctrl.Result{}, err } if cmService != nil { - if err := r.CloudMap.DeleteEndpoints(ctx, cmService.Namespace, cmService.Name, cmService.Endpoints); err != nil { + if err := r.CloudMap.DeleteEndpoints(ctx, cmService.Namespace, cmService.Name, cmService.GetEndpoints(clusterId)); err != nil { r.Log.Error(err, "error deleting Endpoints from Cloud Map", "namespace", cmService.Namespace, "name", cmService.Name) return ctrl.Result{}, err } diff --git a/pkg/controllers/multicluster/serviceexport_controller_test.go b/pkg/controllers/multicluster/serviceexport_controller_test.go index b24022bc..5b1e6dac 100644 --- a/pkg/controllers/multicluster/serviceexport_controller_test.go +++ b/pkg/controllers/multicluster/serviceexport_controller_test.go @@ -141,7 +141,7 @@ func TestServiceExportReconciler_Reconcile_DeleteExistingService(t *testing.T) { Return(test.GetTestService(), nil) // call to delete the endpoint in the cloudmap mock.EXPECT().DeleteEndpoints(gomock.Any(), test.HttpNsName, test.SvcName, - []*model.Endpoint{test.GetTestEndpoint1(), test.GetTestEndpoint2()}).Return(nil).Times(1) + test.GetTestService().GetEndpoints(test.ClusterId1)).Return(nil).Times(1) request := ctrl.Request{ NamespacedName: types.NamespacedName{ @@ -178,8 +178,6 @@ func TestServiceExportReconciler_Reconcile_NoClusterProperty(t *testing.T) { mockSDClient := cloudmapMock.NewMockServiceDiscoveryClient(mockController) - mockSDClient.EXPECT().GetService(gomock.Any(), test.HttpNsName, test.SvcName).Return(test.GetTestService(), nil) - reconciler := getServiceExportReconciler(t, mockSDClient, fakeClient) request := ctrl.Request{ diff --git a/pkg/model/types.go b/pkg/model/types.go index 3a78a3e5..432eb4f8 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -287,6 +287,15 @@ func ConvertNamespaceType(nsType types.NamespaceType) (namespaceType NamespaceTy } } +func (svc *Service) GetEndpoints(clusterId string) (endpts []*Endpoint) { + for _, endpt := range svc.Endpoints { + if endpt.ClusterId == clusterId { + endpts = append(endpts, endpt) + } + } + return endpts +} + func (namespaceType *NamespaceType) IsUnsupported() bool { return *namespaceType == UnsupportedNamespaceType } diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index bf3bfd8e..0040c6e8 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -6,11 +6,16 @@ import ( "testing" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" + "github.com/google/go-cmp/cmp" ) var instId = "my-instance" var ip = "192.168.0.1" var clusterId = "test-mcs-clusterId" +var clusterId2 = "test-mcs-clusterid-2" +var clusterId3 = "test-mcs-clusterid-3" +var namespaceName = "test-mcs-namespace" +var serviceName = "test-mcs-service" var clusterSetId = "test-mcs-clusterSetId" var serviceType = ClusterSetIPType.String() var svcExportCreationTimestamp int64 = 1640995200000 @@ -333,3 +338,69 @@ func TestEndpoint_Equals(t *testing.T) { }) } } + +func TestGetEndpoints(t *testing.T) { + firstEndpoint := Endpoint{ + Id: instId + "-1", + IP: ip, + ServicePort: Port{ + Port: 80, + }, + ClusterId: clusterId, + } + secondEndpoint := Endpoint{ + Id: instId + "2", + IP: ip, + ServicePort: Port{ + Port: 80, + Name: "", + }, + ClusterId: clusterId2, + } + thirdEndpoint := Endpoint{ + Id: instId + "3", + IP: ip, + ServicePort: Port{ + Port: 80, + Name: "", + }, + ClusterId: clusterId2, + } + + svc := Service{ + Namespace: namespaceName, + Name: serviceName, + Endpoints: []*Endpoint{ + &firstEndpoint, &secondEndpoint, &thirdEndpoint, + }, + } + + tests := []struct { + name string + x string + wantEndpts []*Endpoint + }{ + { + name: "return-first-endpoint", + x: clusterId, + wantEndpts: []*Endpoint{&firstEndpoint}, + }, + { + name: "return-two-endpoints", + x: clusterId2, + wantEndpts: []*Endpoint{&secondEndpoint, &thirdEndpoint}, + }, + { + name: "return-nil", + x: clusterId3, + wantEndpts: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotEndpts := svc.GetEndpoints(tt.x); !cmp.Equal(gotEndpts, tt.wantEndpts) { + t.Errorf("Equals() = %v, Want = %v", gotEndpts, tt.wantEndpts) + } + }) + } +} From 887737efceed14c883d9eae64637b53ab0f0796c Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Fri, 9 Dec 2022 11:17:48 -0800 Subject: [PATCH 136/163] v0.3.1 release (#256) --- README.md | 2 +- config/controller_install_release/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3b1298ac..18fcd74e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ ## Introduction The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. We have detailed [step-by-step setup guide](https://aws.github.io/aws-cloud-map-mcs-controller-for-k8s/)! -**NOTE: The current version `v0.3.0` is in *Alpha* phase, checkout the [Graduation Criteria](#graduation-criteria) for the next steps.** +**NOTE: The current version [![GitHub Release](https://img.shields.io/github/release/aws/aws-cloud-map-mcs-controller-for-k8s.svg?style=flat&label=)]() is in *Alpha* phase, checkout the [Graduation Criteria](#graduation-criteria) for the next steps.** ## Installation diff --git a/config/controller_install_release/kustomization.yaml b/config/controller_install_release/kustomization.yaml index b870ff22..e6dd7732 100644 --- a/config/controller_install_release/kustomization.yaml +++ b/config/controller_install_release/kustomization.yaml @@ -4,4 +4,4 @@ bases: images: - name: controller newName: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s - newTag: v0.3.0 + newTag: v0.3.1 From 37a57ead69f36cb534ea25c496358938976e0879 Mon Sep 17 00:00:00 2001 From: CurtisThe <37814747+CurtisThe@users.noreply.github.com> Date: Mon, 12 Dec 2022 21:52:39 -0600 Subject: [PATCH 137/163] Ipv6 (#257) * Support for IPv6 endpoints. Co-authored-by: Jay Patel Co-authored-by: jaywasd --- Makefile | 6 + integration/kind-test/configs/ipv6.yaml | 5 + integration/kind-test/scripts/dns-test.sh | 23 +- integration/kind-test/scripts/run-helper.sh | 9 +- integration/kind-test/scripts/run-tests.sh | 5 +- integration/kind-test/scripts/setup-kind.sh | 15 +- .../shared/scenarios/export_service.go | 11 +- integration/shared/scenarios/runner/main.go | 13 +- .../multicluster/controllers_common_test.go | 2 +- .../multicluster/endpointslice_plan.go | 11 +- .../multicluster/endpointslice_plan_test.go | 40 +++ .../multicluster/serviceexport_controller.go | 5 +- pkg/controllers/multicluster/utils.go | 4 +- pkg/model/types.go | 48 ++- pkg/model/types_test.go | 331 ++++++++++++++++-- test/test-constants.go | 41 ++- 16 files changed, 498 insertions(+), 71 deletions(-) create mode 100644 integration/kind-test/configs/ipv6.yaml diff --git a/Makefile b/Makefile index 7e3f0915..f6a2fb24 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,12 @@ test: manifests generate generate-mocks fmt vet goimports lint ## Run tests go test ./... -coverprofile=cover.out -covermode=atomic kind-integration-suite: ## Provision and run integration tests with cleanup + export ADDRESS_TYPE="IPv4" && \ + make kind-integration-setup && \ + make kind-integration-run && \ + make kind-integration-cleanup + + export ADDRESS_TYPE="IPv6" && \ make kind-integration-setup && \ make kind-integration-run && \ make kind-integration-cleanup diff --git a/integration/kind-test/configs/ipv6.yaml b/integration/kind-test/configs/ipv6.yaml new file mode 100644 index 00000000..a577feda --- /dev/null +++ b/integration/kind-test/configs/ipv6.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: ipv6 + apiServerAddress: 127.0.0.1 \ No newline at end of file diff --git a/integration/kind-test/scripts/dns-test.sh b/integration/kind-test/scripts/dns-test.sh index 95463b54..627c10a8 100755 --- a/integration/kind-test/scripts/dns-test.sh +++ b/integration/kind-test/scripts/dns-test.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash +# If the IP Type env var is not set, default it to IPv4 +if [[ -z "${ADDRESS_TYPE}" ]]; then + ADDRESS_TYPE="IPv4" +fi + # Helper function to verify DNS results checkDNS() { dns_addresses_count=$(echo "$1" | wc -l | xargs) @@ -31,10 +36,20 @@ $KUBECTL_BIN wait --for=condition=ready pod/dnsutils # wait until pod is deploye # Perform a dig to cluster-local CoreDNS # TODO: parse dig outputs for more precise verification - check specifics IPs? -echo "performing dig for A/AAAA records..." -addresses=$($KUBECTL_BIN exec dnsutils -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) -exit_code=$? -echo "$addresses" +if [[ $ADDRESS_TYPE == "IPv4" ]]; then + echo "performing dig for A records for IPv4..." + addresses=$($KUBECTL_BIN exec dnsutils -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) + exit_code=$? + echo "$addresses" +elif [[ $ADDRESS_TYPE == "IPv6" ]]; then + echo "performing dig for AAAA records for IPv6..." + addresses=$($KUBECTL_BIN exec dnsutils -- dig AAAA +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short) + exit_code=$? + echo "$addresses" +else + echo "ADDRESS_TYPE invalid" + exit 1 +fi if [ "$exit_code" -ne 0 ]; then echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local" diff --git a/integration/kind-test/scripts/run-helper.sh b/integration/kind-test/scripts/run-helper.sh index 9267654b..fd002efa 100755 --- a/integration/kind-test/scripts/run-helper.sh +++ b/integration/kind-test/scripts/run-helper.sh @@ -7,8 +7,13 @@ source ./integration/kind-test/scripts/common.sh # create test namespace $KUBECTL_BIN create namespace "$NAMESPACE" +# If the IP Type env var is not set, default it to IPV4 +if [[ -z "${ADDRESS_TYPE}" ]]; then + ADDRESS_TYPE="IPv4" +fi + # ClusterIP service test -./integration/kind-test/scripts/run-tests.sh "$CLUSTERIP_SERVICE" "ClusterSetIP" +./integration/kind-test/scripts/run-tests.sh "$CLUSTERIP_SERVICE" "ClusterSetIP" $ADDRESS_TYPE exit_code=$? if [ "$exit_code" -ne 0 ] ; then echo "ERROR: Testing $CLUSTERIP_SERVICE failed" @@ -18,7 +23,7 @@ fi sleep 5 # Headless service test -./integration/kind-test/scripts/run-tests.sh "$HEADLESS_SERVICE" "Headless" +./integration/kind-test/scripts/run-tests.sh "$HEADLESS_SERVICE" "Headless" $ADDRESS_TYPE exit_code=$? if [ "$exit_code" -ne 0 ] ; then echo "ERROR: Testing $HEADLESS_SERVICE failed" diff --git a/integration/kind-test/scripts/run-tests.sh b/integration/kind-test/scripts/run-tests.sh index 1c5d1b13..95eab7bc 100755 --- a/integration/kind-test/scripts/run-tests.sh +++ b/integration/kind-test/scripts/run-tests.sh @@ -5,6 +5,7 @@ source ./integration/kind-test/scripts/common.sh export SERVICE=$1 export SERVICE_TYPE=$2 +export IP_TYPE=$3 # Deploy pods $KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-deployment.yaml" @@ -24,7 +25,7 @@ mkdir -p "$LOGS" CTL_PID=$! echo "controller PID:$CTL_PID" -go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$endpts" +go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE $IP_TYPE "$endpts" exit_code=$? if [ "$exit_code" -eq 0 ] ; then @@ -58,7 +59,7 @@ if [ "$exit_code" -eq 0 ] ; then fi if [ "$exit_code" -eq 0 ] ; then - go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$updated_endpoints" + go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE $IP_TYPE "$updated_endpoints" exit_code=$? fi diff --git a/integration/kind-test/scripts/setup-kind.sh b/integration/kind-test/scripts/setup-kind.sh index 88ba6976..2b37ff52 100755 --- a/integration/kind-test/scripts/setup-kind.sh +++ b/integration/kind-test/scripts/setup-kind.sh @@ -9,7 +9,20 @@ source ./integration/kind-test/scripts/common.sh ./integration/kind-test/scripts/ensure-jq.sh -$KIND_BIN create cluster --name "$KIND_SHORT" --image "$IMAGE" +# If the IP Type env var is not set, default it to IPv4 +if [[ -z "${ADDRESS_TYPE}" ]]; then + ADDRESS_TYPE="IPv4" +fi + +echo "ADDRESS_TYPE: $ADDRESS_TYPE" +if [[ $ADDRESS_TYPE == "IPv4" ]]; then + $KIND_BIN create cluster --name "$KIND_SHORT" --image "$IMAGE" +elif [[ $ADDRESS_TYPE == "IPv6" ]]; then + $KIND_BIN create cluster --name "$KIND_SHORT" --image "$IMAGE" --config=./integration/kind-test/configs/ipv6.yaml +else + echo "ADDRESS_TYPE invalid" +fi + $KUBECTL_BIN config use-context "$CLUSTER" make install diff --git a/integration/shared/scenarios/export_service.go b/integration/shared/scenarios/export_service.go index df609b69..535faaee 100644 --- a/integration/shared/scenarios/export_service.go +++ b/integration/shared/scenarios/export_service.go @@ -33,7 +33,7 @@ type exportServiceScenario struct { expectedSvc model.Service } -func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, clusterId string, clusterSetId string, portStr string, servicePortStr string, serviceType string, ips string) (ExportServiceScenario, error) { +func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, clusterId string, clusterSetId string, portStr string, servicePortStr string, serviceType string, addressTypeStr string, ips string) (ExportServiceScenario, error) { endpts := make([]*model.Endpoint, 0) port, parseError := strconv.ParseUint(portStr, 10, 16) @@ -44,6 +44,10 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, cl if parseError != nil { return nil, parseError } + addressType, parseError := model.GetAddressTypeFromString(addressTypeStr) + if parseError != nil { + return nil, parseError + } for _, ip := range strings.Split(ips, ",") { endpointPort := model.Port{ @@ -51,8 +55,9 @@ func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, cl Protocol: string(v1.ProtocolTCP), } endpts = append(endpts, &model.Endpoint{ - Id: model.EndpointIdFromIPAddressAndPort(ip, endpointPort), - IP: ip, + Id: model.EndpointIdFromIPAddressAndPort(ip, endpointPort), + IP: ip, + AddressType: addressType, ServicePort: model.Port{ Port: int32(servicePort), TargetPort: portStr, diff --git a/integration/shared/scenarios/runner/main.go b/integration/shared/scenarios/runner/main.go index ce94c9b4..0889445f 100644 --- a/integration/shared/scenarios/runner/main.go +++ b/integration/shared/scenarios/runner/main.go @@ -11,8 +11,8 @@ import ( ) func main() { - if len(os.Args) != 9 { - fmt.Println("Expected namespace, service, clusterId, clusterSetId, endpoint port, service port, serviceType, and endpoint IP list as arguments") + if len(os.Args) != 10 { + fmt.Println("Expected namespace, service, clusterId, clusterSetId, endpoint port, service port, serviceType, endpoint AddressType, and endpoint IP list as arguments") os.Exit(1) } @@ -23,15 +23,16 @@ func main() { port := os.Args[5] servicePort := os.Args[6] serviceType := os.Args[7] - ips := os.Args[8] + addressType := os.Args[8] + ips := os.Args[9] - testServiceExport(nsName, svcName, clusterId, clusterSetId, port, servicePort, serviceType, ips) + testServiceExport(nsName, svcName, clusterId, clusterSetId, port, servicePort, serviceType, addressType, ips) } -func testServiceExport(nsName string, svcName string, clusterId string, clusterSetId string, port string, servicePort string, serviceType string, ips string) { +func testServiceExport(nsName string, svcName string, clusterId string, clusterSetId string, port string, servicePort string, serviceType string, addressType string, ips string) { fmt.Printf("Testing service export integration for namespace %s and service %s\n", nsName, svcName) - export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, clusterId, clusterSetId, port, servicePort, serviceType, ips) + export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, clusterId, clusterSetId, port, servicePort, serviceType, addressType, ips) if err != nil { fmt.Printf("Failed to setup service export integration test scenario: %s", err.Error()) os.Exit(1) diff --git a/pkg/controllers/multicluster/controllers_common_test.go b/pkg/controllers/multicluster/controllers_common_test.go index 12cfb690..37dce06f 100644 --- a/pkg/controllers/multicluster/controllers_common_test.go +++ b/pkg/controllers/multicluster/controllers_common_test.go @@ -96,7 +96,7 @@ func endpointSliceForTest() *discovery.EndpointSlice { func endpointSliceFromEndpointsForTest(endpts []*model.Endpoint, ports []discovery.EndpointPort) *discovery.EndpointSlice { svc := k8sServiceForTest() - slice := CreateEndpointSliceStruct(svc, test.SvcName, test.ClusterId1) + slice := CreateEndpointSliceStruct(svc, test.SvcName, test.ClusterId1, endpts[0].AddressType) slice.Ports = ports testEndpoints := make([]discovery.Endpoint, 0) diff --git a/pkg/controllers/multicluster/endpointslice_plan.go b/pkg/controllers/multicluster/endpointslice_plan.go index fe05fb82..c3bb996a 100644 --- a/pkg/controllers/multicluster/endpointslice_plan.go +++ b/pkg/controllers/multicluster/endpointslice_plan.go @@ -39,6 +39,15 @@ type EndpointSlicePlan struct { ClusterId string } +// CheckAddressType TODO: Will need to improve how IP Type is determined when we implement dual stack. +func (p *EndpointSlicePlan) CheckAddressType() discovery.AddressType { + // Peek at the first endpoint for its AddressType. All endpoints in a slice will be of the same AddressType. + if len(p.Desired) == 0 { + return discovery.AddressTypeIPv4 + } + return p.Desired[0].AddressType +} + // CalculateChanges returns list of EndpointSlice Changes that need to applied func (p *EndpointSlicePlan) CalculateChanges() EndpointSliceChanges { // populate map of desired endpoints for lookup efficiency @@ -147,7 +156,7 @@ func (p *EndpointSlicePlan) getOrCreateUnfilledEndpointSlice(changes *EndpointSl } // No existing slices can fill new endpoint requirements so create a new slice - sliceToCreate := CreateEndpointSliceStruct(p.Service, p.ServiceImportName, p.ClusterId) + sliceToCreate := CreateEndpointSliceStruct(p.Service, p.ServiceImportName, p.ClusterId, p.CheckAddressType()) changes.Create = append(changes.Create, sliceToCreate) return sliceToCreate, true } diff --git a/pkg/controllers/multicluster/endpointslice_plan_test.go b/pkg/controllers/multicluster/endpointslice_plan_test.go index d058ab86..512252e8 100644 --- a/pkg/controllers/multicluster/endpointslice_plan_test.go +++ b/pkg/controllers/multicluster/endpointslice_plan_test.go @@ -11,6 +11,46 @@ import ( discovery "k8s.io/api/discovery/v1" ) +func TestCheckAddressType(t *testing.T) { + tests := []struct { + name string + want discovery.AddressType + slicePlan EndpointSlicePlan + }{ + { + name: "happy case ipv4", + want: discovery.AddressTypeIPv4, + slicePlan: EndpointSlicePlan{ + maxEndpointsPerSlice: 0, + Service: nil, + ServiceImportName: "", + Current: nil, + Desired: []*model.Endpoint{test.GetTestEndpoint1()}, + ClusterId: "", + }, + }, + { + name: "happy case ipv6", + want: discovery.AddressTypeIPv6, + slicePlan: EndpointSlicePlan{ + maxEndpointsPerSlice: 0, + Service: nil, + ServiceImportName: "", + Current: nil, + Desired: []*model.Endpoint{test.GetTestEndpointIpv6()}, + ClusterId: "", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.slicePlan.CheckAddressType(); got != tt.want { + t.Errorf("CheckAddressType() = %v, want %v", got, tt.want) + } + }) + } +} + func TestEndpointSlicePlan_CalculateChanges(t *testing.T) { type fields struct { Current []*discovery.EndpointSlice diff --git a/pkg/controllers/multicluster/serviceexport_controller.go b/pkg/controllers/multicluster/serviceexport_controller.go index dcd58978..34dabdd4 100644 --- a/pkg/controllers/multicluster/serviceexport_controller.go +++ b/pkg/controllers/multicluster/serviceexport_controller.go @@ -2,7 +2,6 @@ package controllers import ( "context" - "fmt" "time" "k8s.io/apimachinery/pkg/api/errors" @@ -251,9 +250,6 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. endpoints := make([]*model.Endpoint, 0) for _, slice := range endpointSlices.Items { - if slice.AddressType != discovery.AddressTypeIPv4 { - return nil, fmt.Errorf("unsupported address type %s for service %s", slice.AddressType, svc.Name) - } for _, endpointPort := range slice.Ports { for _, endpoint := range slice.Endpoints { port := EndpointPortToPort(endpointPort) @@ -263,6 +259,7 @@ func (r *ServiceExportReconciler) extractEndpoints(ctx context.Context, svc *v1. endpoints = append(endpoints, &model.Endpoint{ Id: model.EndpointIdFromIPAddressAndPort(IP, port), IP: IP, + AddressType: slice.AddressType, EndpointPort: port, ServicePort: servicePortMap[*endpointPort.Name], ClusterId: clusterProperties.ClusterId(), diff --git a/pkg/controllers/multicluster/utils.go b/pkg/controllers/multicluster/utils.go index 8bcc0b08..4a919431 100644 --- a/pkg/controllers/multicluster/utils.go +++ b/pkg/controllers/multicluster/utils.go @@ -251,7 +251,7 @@ func CreateEndpointForSlice(svc *v1.Service, endpoint *model.Endpoint) discovery return ep } -func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string, clusterId string) *discovery.EndpointSlice { +func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string, clusterId string, addressType discovery.AddressType) *discovery.EndpointSlice { return &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -271,7 +271,7 @@ func CreateEndpointSliceStruct(svc *v1.Service, svcImportName string, clusterId })}, Namespace: svc.Namespace, }, - AddressType: discovery.AddressTypeIPv4, + AddressType: addressType, } } diff --git a/pkg/model/types.go b/pkg/model/types.go index 432eb4f8..85c93579 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -7,6 +7,8 @@ import ( "strconv" "strings" + discovery "k8s.io/api/discovery/v1" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" ) @@ -44,6 +46,7 @@ type ServiceType string type Endpoint struct { Id string IP string + AddressType discovery.AddressType EndpointPort Port ServicePort Port ClusterId string @@ -67,6 +70,7 @@ type Port struct { // Rest are custom attributes const ( EndpointIpv4Attr = "AWS_INSTANCE_IPV4" + EndpointIpv6Attr = "AWS_INSTANCE_IPV6" EndpointPortAttr = "AWS_INSTANCE_PORT" EndpointPortNameAttr = "ENDPOINT_PORT_NAME" EndpointProtocolAttr = "ENDPOINT_PROTOCOL" @@ -96,11 +100,29 @@ func NewEndpointFromInstance(inst *types.HttpInstanceSummary) (*Endpoint, error) } // Remove and set the IP, Port, Service Port, ServiceType, ClusterId, ClusterSetId - ip, err := removeStringAttr(attributes, EndpointIpv4Attr) - if err != nil { - return nil, err + + // ASSUMPTION: Endpoints have either IPV4 OR IPV6, not both. Defaults to IPV4 if both are present. + ipv4, ipv4Exists := attributes[EndpointIpv4Attr] + ipv6, ipv6Exists := attributes[EndpointIpv6Attr] + if ipv6Exists { + ip, err := removeStringAttr(attributes, EndpointIpv6Attr) + if err != nil { + return nil, err + } + endpoint.IP = ip + endpoint.AddressType = discovery.AddressTypeIPv6 + } + if ipv4Exists { + ip, err := removeStringAttr(attributes, EndpointIpv4Attr) + if err != nil { + return nil, err + } + endpoint.IP = ip + endpoint.AddressType = discovery.AddressTypeIPv4 + } + if ipv4Exists && ipv6Exists { + fmt.Printf("WARNING: Found both address types in one Endpoint... IPv4: %s IPv6: %s\n", ipv4, ipv6) } - endpoint.IP = ip endpointPort, err := endpointPortFromAttr(attributes) if err != nil { @@ -226,9 +248,14 @@ func removeTimestampAttr(attributes map[string]string, attr string) (int64, erro func (e *Endpoint) GetCloudMapAttributes() map[string]string { attrs := make(map[string]string) + if e.AddressType == discovery.AddressTypeIPv4 { + attrs[EndpointIpv4Attr] = e.IP + } else if e.AddressType == discovery.AddressTypeIPv6 { + attrs[EndpointIpv6Attr] = e.IP + } + attrs[ClusterIdAttr] = e.ClusterId attrs[ClusterSetIdAttr] = e.ClusterSetId - attrs[EndpointIpv4Attr] = e.IP attrs[EndpointPortAttr] = strconv.Itoa(int(e.EndpointPort.Port)) attrs[EndpointProtocolAttr] = e.EndpointPort.Protocol attrs[EndpointPortNameAttr] = e.EndpointPort.Name @@ -308,3 +335,14 @@ func (p *Port) GetID() string { func (p *Port) Equals(other *Port) bool { return reflect.DeepEqual(p, other) } + +func GetAddressTypeFromString(addressTypeStr string) (discovery.AddressType, error) { + switch addressTypeStr { + case string(discovery.AddressTypeIPv4): + return discovery.AddressTypeIPv4, nil + case string(discovery.AddressTypeIPv6): + return discovery.AddressTypeIPv6, nil + default: + return "", fmt.Errorf("Invalid AddressType, could not parse from string: %s", addressTypeStr) + } +} diff --git a/pkg/model/types_test.go b/pkg/model/types_test.go index 0040c6e8..8ab1ca44 100644 --- a/pkg/model/types_test.go +++ b/pkg/model/types_test.go @@ -5,12 +5,15 @@ import ( "strconv" "testing" + discovery "k8s.io/api/discovery/v1" + "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" "github.com/google/go-cmp/cmp" ) var instId = "my-instance" -var ip = "192.168.0.1" +var ipv4 = "192.168.0.1" +var ipv6 = "2001:0db8:0001:0000:0000:0ab9:C0A8:0102" var clusterId = "test-mcs-clusterId" var clusterId2 = "test-mcs-clusterid-2" var clusterId3 = "test-mcs-clusterid-3" @@ -28,13 +31,106 @@ func TestNewEndpointFromInstance(t *testing.T) { wantErr bool }{ { - name: "happy case", + name: "happy case ipv4", + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, + Attributes: map[string]string{ + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, + EndpointIpv4Attr: ipv4, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, + ServiceExportCreationAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), + "custom-attr": "custom-val", + }, + }, + want: &Endpoint{ + Id: instId, + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, + EndpointPort: Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + ServicePort: Port{ + Name: "http", + Port: 65535, + TargetPort: "80", + Protocol: "TCP", + }, + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), + ServiceExportCreationTimestamp: svcExportCreationTimestamp, + Ready: true, + Attributes: map[string]string{ + "custom-attr": "custom-val", + }, + }, + }, + { + name: "happy case ipv6", + inst: &types.HttpInstanceSummary{ + InstanceId: &instId, + Attributes: map[string]string{ + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, + EndpointIpv6Attr: ipv6, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "65535", + ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, + ServiceExportCreationAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), + "custom-attr": "custom-val", + }, + }, + want: &Endpoint{ + Id: instId, + IP: ipv6, + AddressType: discovery.AddressTypeIPv6, + EndpointPort: Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + ServicePort: Port{ + Name: "http", + Port: 65535, + TargetPort: "80", + Protocol: "TCP", + }, + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), + ServiceExportCreationTimestamp: svcExportCreationTimestamp, + Ready: true, + Attributes: map[string]string{ + "custom-attr": "custom-val", + }, + }, + }, + { + name: "ipv4 and ipv6 defaults to ipv4", inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ ClusterIdAttr: clusterId, ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, + EndpointIpv4Attr: ipv4, + EndpointIpv6Attr: ipv6, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", @@ -49,8 +145,9 @@ func TestNewEndpointFromInstance(t *testing.T) { }, }, want: &Endpoint{ - Id: instId, - IP: ip, + Id: instId, + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, EndpointPort: Port{ Name: "http", Port: 80, @@ -77,7 +174,7 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - EndpointIpv4Attr: ip, + EndpointIpv4Attr: ipv4, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", @@ -108,7 +205,7 @@ func TestNewEndpointFromInstance(t *testing.T) { inst: &types.HttpInstanceSummary{ InstanceId: &instId, Attributes: map[string]string{ - EndpointIpv4Attr: ip, + EndpointIpv4Attr: ipv4, "custom-attr": "custom-val", }, }, @@ -120,7 +217,7 @@ func TestNewEndpointFromInstance(t *testing.T) { InstanceId: &instId, Attributes: map[string]string{ ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, + EndpointIpv4Attr: ipv4, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", @@ -140,7 +237,7 @@ func TestNewEndpointFromInstance(t *testing.T) { InstanceId: &instId, Attributes: map[string]string{ ClusterIdAttr: clusterId, - EndpointIpv4Attr: ip, + EndpointIpv4Attr: ipv4, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", @@ -161,7 +258,7 @@ func TestNewEndpointFromInstance(t *testing.T) { Attributes: map[string]string{ ClusterIdAttr: clusterId, ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, + EndpointIpv4Attr: ipv4, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", @@ -197,9 +294,54 @@ func TestEndpoint_GetAttributes(t *testing.T) { want map[string]string }{ { - name: "happy case", + name: "happy case ipv4", + endpoint: Endpoint{ + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, + EndpointPort: Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + ServicePort: Port{ + Name: "http", + Port: 30, + TargetPort: "80", + Protocol: "TCP", + }, + Ready: true, + ClusterId: clusterId, + ClusterSetId: clusterSetId, + ServiceType: ServiceType(serviceType), + ServiceExportCreationTimestamp: svcExportCreationTimestamp, + Attributes: map[string]string{ + "custom-attr": "custom-val", + }, + }, + want: map[string]string{ + ClusterIdAttr: clusterId, + ClusterSetIdAttr: clusterSetId, + EndpointIpv4Attr: ipv4, + EndpointPortAttr: "80", + EndpointProtocolAttr: "TCP", + EndpointPortNameAttr: "http", + EndpointReadyAttr: "true", + EndpointHostnameAttr: "", + EndpointNodeNameAttr: "", + ServicePortNameAttr: "http", + ServiceProtocolAttr: "TCP", + ServicePortAttr: "30", + ServiceTargetPortAttr: "80", + ServiceTypeAttr: serviceType, + ServiceExportCreationAttr: strconv.FormatInt(svcExportCreationTimestamp, 10), + "custom-attr": "custom-val", + }, + }, + { + name: "happy case ipv6", endpoint: Endpoint{ - IP: ip, + IP: ipv6, + AddressType: discovery.AddressTypeIPv6, EndpointPort: Port{ Name: "http", Port: 80, @@ -223,7 +365,7 @@ func TestEndpoint_GetAttributes(t *testing.T) { want: map[string]string{ ClusterIdAttr: clusterId, ClusterSetIdAttr: clusterSetId, - EndpointIpv4Attr: ip, + EndpointIpv6Attr: ipv6, EndpointPortAttr: "80", EndpointProtocolAttr: "TCP", EndpointPortNameAttr: "http", @@ -257,8 +399,8 @@ func TestEndpointIdFromIPAddressAndPort(t *testing.T) { want string }{ { - name: "happy case", - address: ip, + name: "happy case ipv4", + address: ipv4, port: Port{ Name: "http", Port: 80, @@ -266,6 +408,16 @@ func TestEndpointIdFromIPAddressAndPort(t *testing.T) { }, want: "tcp-192_168_0_1-80", }, + { + name: "happy case ipv6", + address: ipv6, + port: Port{ + Name: "http", + Port: 80, + Protocol: "TCP", + }, + want: "tcp-2001_0db8_0001_0000_0000_0ab9_C0A8_0102-80", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -277,9 +429,10 @@ func TestEndpointIdFromIPAddressAndPort(t *testing.T) { } func TestEndpoint_Equals(t *testing.T) { - firstEndpoint := Endpoint{ - Id: instId, - IP: ip, + firstEndpointIpv4 := Endpoint{ + Id: instId, + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, ServicePort: Port{ Port: 80, }, @@ -288,9 +441,10 @@ func TestEndpoint_Equals(t *testing.T) { }, } - secondEndpoint := Endpoint{ - Id: instId, - IP: ip, + secondEndpointIpv4 := Endpoint{ + Id: instId, + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, ServicePort: Port{ Port: 80, Name: "", @@ -300,9 +454,47 @@ func TestEndpoint_Equals(t *testing.T) { }, } - thirdEndpoint := Endpoint{ - Id: instId, - IP: ip, + thirdEndpointIpv4 := Endpoint{ + Id: instId, + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, + ServicePort: Port{ + Port: 80, + }, + Attributes: map[string]string{ + "custom-key": "different-val", + }, + } + + firstEndpointIpv6 := Endpoint{ + Id: instId, + IP: ipv6, + AddressType: discovery.AddressTypeIPv6, + ServicePort: Port{ + Port: 80, + }, + Attributes: map[string]string{ + "custom-key": "custom-val", + }, + } + + secondEndpointIpv6 := Endpoint{ + Id: instId, + IP: ipv6, + AddressType: discovery.AddressTypeIPv6, + ServicePort: Port{ + Port: 80, + Name: "", + }, + Attributes: map[string]string{ + "custom-key": "custom-val", + }, + } + + thirdEndpointIpv6 := Endpoint{ + Id: instId, + IP: ipv6, + AddressType: discovery.AddressTypeIPv6, ServicePort: Port{ Port: 80, }, @@ -318,15 +510,33 @@ func TestEndpoint_Equals(t *testing.T) { want bool }{ { - name: "identical", - x: firstEndpoint, - y: secondEndpoint, + name: "identical ipv4", + x: firstEndpointIpv4, + y: secondEndpointIpv4, + want: true, + }, + { + name: "identical ipv6", + x: firstEndpointIpv6, + y: secondEndpointIpv6, want: true, }, { - name: "different", - x: firstEndpoint, - y: thirdEndpoint, + name: "different ipv4", + x: firstEndpointIpv4, + y: thirdEndpointIpv4, + want: false, + }, + { + name: "different ipv6", + x: firstEndpointIpv6, + y: thirdEndpointIpv6, + want: false, + }, + { + name: "different ipv4 and ipv6", + x: firstEndpointIpv4, + y: firstEndpointIpv6, want: false, }, } @@ -339,18 +549,66 @@ func TestEndpoint_Equals(t *testing.T) { } } +func TestGetAddressTypeFromString(t *testing.T) { + tests := []struct { + name string + addressTypeStr string + want discovery.AddressType + wantErr bool + }{ + { + name: "happy case ipv4", + addressTypeStr: "IPv4", + want: discovery.AddressTypeIPv4, + wantErr: false, + }, + { + name: "happy case ipv6", + addressTypeStr: "IPv6", + want: discovery.AddressTypeIPv6, + wantErr: false, + }, + { + name: "empty string", + addressTypeStr: "", + want: "", + wantErr: true, + }, + { + name: "case wrong", + addressTypeStr: "IPV6", + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetAddressTypeFromString(tt.addressTypeStr) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddressTypeFromString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetAddressTypeFromString() = %v, want %v", got, tt.want) + } + }) + } +} + func TestGetEndpoints(t *testing.T) { firstEndpoint := Endpoint{ - Id: instId + "-1", - IP: ip, + Id: instId + "-1", + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, ServicePort: Port{ Port: 80, }, ClusterId: clusterId, } secondEndpoint := Endpoint{ - Id: instId + "2", - IP: ip, + Id: instId + "2", + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, ServicePort: Port{ Port: 80, Name: "", @@ -358,8 +616,9 @@ func TestGetEndpoints(t *testing.T) { ClusterId: clusterId2, } thirdEndpoint := Endpoint{ - Id: instId + "3", - IP: ip, + Id: instId + "3", + IP: ipv4, + AddressType: discovery.AddressTypeIPv4, ServicePort: Port{ Port: 80, Name: "", diff --git a/test/test-constants.go b/test/test-constants.go index ecf50f8c..15783070 100644 --- a/test/test-constants.go +++ b/test/test-constants.go @@ -3,6 +3,8 @@ package test import ( "fmt" + discovery "k8s.io/api/discovery/v1" + "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,8 +24,10 @@ const ( ClusterId2 = "test-mcs-clusterid-2" EndptId1 = "tcp-192_168_0_1-1" EndptId2 = "tcp-192_168_0_2-2" + EndptIdIpv6 = "tcp-2001_0db8_0001_0000_0000_0ab9_C0A8:0102-1" EndptIp1 = "192.168.0.1" EndptIp2 = "192.168.0.2" + EndptIpv6 = "2001:0db8:0001:0000:0000:0ab9:C0A8:0102" EndptReadyTrue = "true" EndptReadyFalse = "false" Port1 = 1 @@ -98,8 +102,9 @@ func GetTestMulticlusterService() *model.Service { func GetTestEndpoint1() *model.Endpoint { return &model.Endpoint{ - Id: EndptId1, - IP: EndptIp1, + Id: EndptId1, + IP: EndptIp1, + AddressType: discovery.AddressTypeIPv4, EndpointPort: model.Port{ Name: PortName1, Port: Port1, @@ -124,8 +129,36 @@ func GetTestEndpoint1() *model.Endpoint { func GetTestEndpoint2() *model.Endpoint { return &model.Endpoint{ - Id: EndptId2, - IP: EndptIp2, + Id: EndptId2, + IP: EndptIp2, + AddressType: discovery.AddressTypeIPv4, + EndpointPort: model.Port{ + Name: PortName2, + Port: Port2, + Protocol: Protocol2, + }, + ServicePort: model.Port{ + Name: PortName2, + Port: ServicePort2, + TargetPort: PortStr2, + Protocol: Protocol2, + }, + Ready: true, + Hostname: Hostname, + Nodename: Nodename, + ClusterId: ClusterId1, + ClusterSetId: ClusterSet, + ServiceType: model.ClusterSetIPType, + ServiceExportCreationTimestamp: SvcExportCreationTimestamp, + Attributes: map[string]string{model.K8sVersionAttr: PackageVersion}, + } +} + +func GetTestEndpointIpv6() *model.Endpoint { + return &model.Endpoint{ + Id: EndptId2, + IP: EndptIpv6, + AddressType: discovery.AddressTypeIPv6, EndpointPort: model.Port{ Name: PortName2, Port: Port2, From bb6d804367d1f1a934e804b5be77128c2200d876 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:52:25 -0800 Subject: [PATCH 138/163] Bump golang.org/x/time from 0.2.0 to 0.3.0 (#260) Bumps [golang.org/x/time](https://github.com/golang/time) from 0.2.0 to 0.3.0. - [Release notes](https://github.com/golang/time/releases) - [Commits](https://github.com/golang/time/compare/v0.2.0...v0.3.0) --- updated-dependencies: - dependency-name: golang.org/x/time dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 13f22685..e151566d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.1 - golang.org/x/time v0.2.0 + golang.org/x/time v0.3.0 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.2 diff --git a/go.sum b/go.sum index b1588741..e898528d 100644 --- a/go.sum +++ b/go.sum @@ -774,8 +774,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From feb22a2d0a2bed915d75085d1fa13bf55f660fa2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:53:29 -0800 Subject: [PATCH 139/163] Bump github.com/aws/aws-sdk-go-v2 from 1.17.1 to 1.17.3 (#261) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.17.1 to 1.17.3. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.17.1...v1.17.3) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index e151566d..c437d05a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.17.1 + github.com/aws/aws-sdk-go-v2 v1.17.3 github.com/aws/aws-sdk-go-v2/config v1.18.3 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2 github.com/go-logr/logr v1.2.3 @@ -37,7 +37,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect - github.com/aws/smithy-go v1.13.4 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index e898528d..f1d1fea3 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,9 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= +github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= +github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.18.3 h1:3kfBKcX3votFX84dm00U8RGA1sCCh3eRMOGzg5dCWfU= github.com/aws/aws-sdk-go-v2/config v1.18.3/go.mod h1:BYdrbeCse3ZnOD5+2/VE/nATOK8fEUpBtmPMdKSyhMU= github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57HOo03ELodttmagk= @@ -98,8 +99,9 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZo github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 h1:60SJ4lhvn///8ygCzYy2l53bFW/Q15bVfyjyAWo6zuw= github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= -github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 16ed15296c3a61566f43d122fb6a4d28db099b5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 11:17:58 -0800 Subject: [PATCH 140/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#259) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.18.2 to 1.18.6. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.2...config/v1.18.6) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index c437d05a..b632e42e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.3 github.com/aws/aws-sdk-go-v2/config v1.18.3 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 @@ -30,8 +30,8 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect diff --git a/go.sum b/go.sum index f1d1fea3..9ca36094 100644 --- a/go.sum +++ b/go.sum @@ -83,16 +83,18 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57 github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2 h1:TBmvwADPIuBUEhWPrLsrE0VEchnSgM/wzL33MiuPjhs= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.2/go.mod h1:/J9zJYj/yOhbRTj9EawVe32mkrrFcxCDo5vr5OulEMI= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6 h1:TXihIihIrauLYeccDQWOOZd9T7RX7NZ8vX3e+l35Dyw= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6/go.mod h1:B8K0dN2s4BdUWgKr3jWhHf0Jjzqdskb9yiC79CXw83I= github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= From 943eaff7fa12eda6ffad6bc41026ea2bdaabfd03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 11:30:39 -0800 Subject: [PATCH 141/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.18.3 to 1.18.7 (#258) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.3 to 1.18.7. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.3...config/v1.18.7) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 16 ++++++++-------- go.sum | 36 ++++++++++++++++-------------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index b632e42e..669b6844 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.3 - github.com/aws/aws-sdk-go-v2/config v1.18.3 + github.com/aws/aws-sdk-go-v2/config v1.18.7 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,15 +28,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 9ca36094..ecb17561 100644 --- a/go.sum +++ b/go.sum @@ -74,34 +74,30 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.3 h1:3kfBKcX3votFX84dm00U8RGA1sCCh3eRMOGzg5dCWfU= -github.com/aws/aws-sdk-go-v2/config v1.18.3/go.mod h1:BYdrbeCse3ZnOD5+2/VE/nATOK8fEUpBtmPMdKSyhMU= -github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57HOo03ELodttmagk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= +github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E= +github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6 h1:TXihIihIrauLYeccDQWOOZd9T7RX7NZ8vX3e+l35Dyw= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6/go.mod h1:B8K0dN2s4BdUWgKr3jWhHf0Jjzqdskb9yiC79CXw83I= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 h1:60SJ4lhvn///8ygCzYy2l53bFW/Q15bVfyjyAWo6zuw= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= -github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From 9ddd40ea88fe7a2bbe8677b1bcb4451563c796e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:09:57 -0800 Subject: [PATCH 142/163] Bump github.com/emicklei/go-restful (#266) Bumps [github.com/emicklei/go-restful](https://github.com/emicklei/go-restful) from 2.9.5+incompatible to 2.16.0+incompatible. - [Release notes](https://github.com/emicklei/go-restful/releases) - [Changelog](https://github.com/emicklei/go-restful/blob/v3/CHANGES.md) - [Commits](https://github.com/emicklei/go-restful/compare/v2.9.5...v2.16.0) --- updated-dependencies: - dependency-name: github.com/emicklei/go-restful dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 669b6844..2b8bd877 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/emicklei/go-restful v2.16.0+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect diff --git a/go.sum b/go.sum index ecb17561..b88a622b 100644 --- a/go.sum +++ b/go.sum @@ -149,8 +149,9 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM= +github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From 08d190362c6f59bdaa593eb2324ecbc00c7f27bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:10:58 -0800 Subject: [PATCH 143/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#265) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.18.6 to 1.19.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.6...service/s3/v1.19.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2b8bd877..ddbec4d2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.3 github.com/aws/aws-sdk-go-v2/config v1.18.7 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 diff --git a/go.sum b/go.sum index b88a622b..ae1aee56 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6 h1:TXihIihIrauLYeccDQWOOZd9T7RX7NZ8vX3e+l35Dyw= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.18.6/go.mod h1:B8K0dN2s4BdUWgKr3jWhHf0Jjzqdskb9yiC79CXw83I= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0 h1:RdYI8Ed49zVSKhUYNt8N8lwSVPDWHwUuN8QOrGv2+Uc= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0/go.mod h1:B8K0dN2s4BdUWgKr3jWhHf0Jjzqdskb9yiC79CXw83I= github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= From 24583f71241cad78c6905fb05f38636eb05b6ed2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Feb 2023 12:16:42 -0800 Subject: [PATCH 144/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.18.7 to 1.18.12 (#267) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.7 to 1.18.12. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.7...config/v1.18.12) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 22 +++++++++++----------- go.sum | 41 ++++++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index ddbec4d2..f9f86977 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.17.3 - github.com/aws/aws-sdk-go-v2/config v1.18.7 + github.com/aws/aws-sdk-go-v2 v1.17.4 + github.com/aws/aws-sdk-go-v2/config v1.18.12 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,15 +28,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index ae1aee56..08d8d9ae 100644 --- a/go.sum +++ b/go.sum @@ -74,30 +74,33 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E= -github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA= -github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck= -github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= +github.com/aws/aws-sdk-go-v2 v1.17.4 h1:wyC6p9Yfq6V2y98wfDsj6OnNQa4w2BLGCLIxzNhwOGY= +github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.12 h1:fKs/I4wccmfrNRO9rdrbMO1NgLxct6H9rNMiPdBxHWw= +github.com/aws/aws-sdk-go-v2/config v1.18.12/go.mod h1:J36fOhj1LQBr+O4hJCiT8FwVvieeoSGOtPuvhKlsNu8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.12 h1:Cb+HhuEnV19zHRaYYVglwvdHGMJWbdsyP4oHhw04xws= +github.com/aws/aws-sdk-go-v2/credentials v1.13.12/go.mod h1:37HG2MBroXK3jXfxVGtbM2J48ra2+Ltu+tmwr/jO0KA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 h1:3aMfcTmoXtTZnaT86QlVaYh+BRMbvrrmZwIQ5jWqCZQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22/go.mod h1:YGSIJyQ6D6FjKMQh16hVFSIUD54L4F7zTGePqYMYYJU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 h1:r+XwaCLpIvCKjBIYy/HVZujQS9tsz5ohHG3ZIe0wKoE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 h1:7AwGYXDdqRQYsluvKFmWoqpcOQJ4bH634SkYf3FNj/A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 h1:J4xhFd6zHhdF9jPP0FQJ6WknzBboGMBNjKOv4iTuw4A= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 h1:LjFQf8hFuMO22HkV5VWGLBvmCLBCLPivUAmpdpnp4Vs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22/go.mod h1:xt0Au8yPIwYXf/GYPy/vl4K3CgwhfQMYbrH7DlUUIws= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0 h1:RdYI8Ed49zVSKhUYNt8N8lwSVPDWHwUuN8QOrGv2+Uc= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0/go.mod h1:B8K0dN2s4BdUWgKr3jWhHf0Jjzqdskb9yiC79CXw83I= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 h1:lQKN/LNa3qqu2cDOQZybP7oL4nMGGiFqob0jZJaR8/4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.1/go.mod h1:IgV8l3sj22nQDd5qcAGY0WenwCzCphqdbFOpfktZPrI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 h1:0bLhH6DRAqox+g0LatcjGKjjhU6Eudyys6HB6DJVPj8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1/go.mod h1:O1YSOg3aekZibh2SngvCRRG+cRHKKlYgxf/JBF/Kr/k= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 h1:s49mSnsBZEXjfGBkRfmK+nPqzT7Lt3+t2SmAKNyHblw= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.3/go.mod h1:b+psTJn33Q4qGoDaM7ZiOVVG8uVjGI6HaZ8WBHdgDgU= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From 451f13436c30def66bc31133e8e8e88d84f5d4fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Feb 2023 22:11:40 -0800 Subject: [PATCH 145/163] Bump golang.org/x/net from 0.0.0-20220722155237-a158d28d115b to 0.7.0 (#274) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.0.0-20220722155237-a158d28d115b to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/commits/v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f9f86977..5edc3e72 100644 --- a/go.mod +++ b/go.mod @@ -74,11 +74,11 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index 08d8d9ae..b3cd4cf5 100644 --- a/go.sum +++ b/go.sum @@ -661,8 +661,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -758,11 +759,13 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -771,8 +774,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 3d121eee5b8272122d60f317413cf524ed5dd1b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Feb 2023 22:37:13 -0800 Subject: [PATCH 146/163] Bump golang.org/x/crypto from 0.0.0-20220214200702-86341886e292 to 0.1.0 (#275) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.0.0-20220214200702-86341886e292 to 0.1.0. - [Release notes](https://github.com/golang/crypto/releases) - [Commits](https://github.com/golang/crypto/commits/v0.1.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5edc3e72..dd792d01 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect + golang.org/x/crypto v0.1.0 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sys v0.5.0 // indirect diff --git a/go.sum b/go.sum index b3cd4cf5..eb06b9dc 100644 --- a/go.sum +++ b/go.sum @@ -571,8 +571,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -662,6 +663,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -760,10 +762,12 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -775,6 +779,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 9d6684a2890b195bea9b93b894458dcc8838050c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 17:45:06 +0530 Subject: [PATCH 147/163] Bump github.com/aws/aws-sdk-go-v2 from 1.17.4 to 1.17.5 (#279) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.17.4 to 1.17.5. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.17.4...v1.17.5) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index dd792d01..a401d1d0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.17.4 + github.com/aws/aws-sdk-go-v2 v1.17.5 github.com/aws/aws-sdk-go-v2/config v1.18.12 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0 github.com/go-logr/logr v1.2.3 diff --git a/go.sum b/go.sum index eb06b9dc..a78b0a98 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,9 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.17.4 h1:wyC6p9Yfq6V2y98wfDsj6OnNQa4w2BLGCLIxzNhwOGY= github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.17.5 h1:TzCUW1Nq4H8Xscph5M/skINUitxM5UBAyvm2s7XBzL4= +github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.18.12 h1:fKs/I4wccmfrNRO9rdrbMO1NgLxct6H9rNMiPdBxHWw= github.com/aws/aws-sdk-go-v2/config v1.18.12/go.mod h1:J36fOhj1LQBr+O4hJCiT8FwVvieeoSGOtPuvhKlsNu8= github.com/aws/aws-sdk-go-v2/credentials v1.13.12 h1:Cb+HhuEnV19zHRaYYVglwvdHGMJWbdsyP4oHhw04xws= From c1254a39ae52d27a2987a55499fddc44b5b01422 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 17:45:36 +0530 Subject: [PATCH 148/163] Bump github.com/stretchr/testify from 1.8.1 to 1.8.2 (#277) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.1 to 1.8.2. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.1...v1.8.2) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a401d1d0..84770c5d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 golang.org/x/time v0.3.0 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 diff --git a/go.sum b/go.sum index a78b0a98..731aca43 100644 --- a/go.sum +++ b/go.sum @@ -503,8 +503,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= From 35f92a70fd85ef6a5cd31d231949de3983b647cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 18:08:08 +0530 Subject: [PATCH 149/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#278) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.19.0 to 1.19.4. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.19.0...service/efs/v1.19.4) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 84770c5d..a6224230 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.5 github.com/aws/aws-sdk-go-v2/config v1.18.12 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 @@ -30,8 +30,8 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect diff --git a/go.sum b/go.sum index 731aca43..2e3957bc 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.5 h1:TzCUW1Nq4H8Xscph5M/skINUitxM5UBAyvm2s7XBzL4= github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= @@ -84,18 +83,18 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.12 h1:Cb+HhuEnV19zHRaYYVglwvdHGMJ github.com/aws/aws-sdk-go-v2/credentials v1.13.12/go.mod h1:37HG2MBroXK3jXfxVGtbM2J48ra2+Ltu+tmwr/jO0KA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 h1:3aMfcTmoXtTZnaT86QlVaYh+BRMbvrrmZwIQ5jWqCZQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22/go.mod h1:YGSIJyQ6D6FjKMQh16hVFSIUD54L4F7zTGePqYMYYJU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 h1:r+XwaCLpIvCKjBIYy/HVZujQS9tsz5ohHG3ZIe0wKoE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 h1:7AwGYXDdqRQYsluvKFmWoqpcOQJ4bH634SkYf3FNj/A= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 h1:9/aKwwus0TQxppPXFmf010DFrE+ssSbzroLVYINA+xE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 h1:b/Vn141DBuLVgXbhRWIrl9g+ww7G+ScV5SzniWR13jQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 h1:J4xhFd6zHhdF9jPP0FQJ6WknzBboGMBNjKOv4iTuw4A= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 h1:LjFQf8hFuMO22HkV5VWGLBvmCLBCLPivUAmpdpnp4Vs= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22/go.mod h1:xt0Au8yPIwYXf/GYPy/vl4K3CgwhfQMYbrH7DlUUIws= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0 h1:RdYI8Ed49zVSKhUYNt8N8lwSVPDWHwUuN8QOrGv2+Uc= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.0/go.mod h1:B8K0dN2s4BdUWgKr3jWhHf0Jjzqdskb9yiC79CXw83I= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 h1:xxWtdHTL6YO5nqXYV0s6rYV7NpAc95EjztJ351oRdqA= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4/go.mod h1:UBNPuS7L4/FIjxAS92CO/5ljnaKHyAe1GeoLjqR8pvA= github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 h1:lQKN/LNa3qqu2cDOQZybP7oL4nMGGiFqob0jZJaR8/4= github.com/aws/aws-sdk-go-v2/service/sso v1.12.1/go.mod h1:IgV8l3sj22nQDd5qcAGY0WenwCzCphqdbFOpfktZPrI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 h1:0bLhH6DRAqox+g0LatcjGKjjhU6Eudyys6HB6DJVPj8= From ceecd6faf4898a27387ae2413485c060114fa502 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 18:55:47 +0530 Subject: [PATCH 150/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.18.12 to 1.18.15 (#276) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.12 to 1.18.15. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.12...config/v1.18.15) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 16 ++++++++-------- go.sum | 35 ++++++++++++++++------------------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index a6224230..88a54abd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.5 - github.com/aws/aws-sdk-go-v2/config v1.18.12 + github.com/aws/aws-sdk-go-v2/config v1.18.15 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,15 +28,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.15 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 2e3957bc..aef8db5b 100644 --- a/go.sum +++ b/go.sum @@ -74,33 +74,30 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.5 h1:TzCUW1Nq4H8Xscph5M/skINUitxM5UBAyvm2s7XBzL4= github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.12 h1:fKs/I4wccmfrNRO9rdrbMO1NgLxct6H9rNMiPdBxHWw= -github.com/aws/aws-sdk-go-v2/config v1.18.12/go.mod h1:J36fOhj1LQBr+O4hJCiT8FwVvieeoSGOtPuvhKlsNu8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.12 h1:Cb+HhuEnV19zHRaYYVglwvdHGMJWbdsyP4oHhw04xws= -github.com/aws/aws-sdk-go-v2/credentials v1.13.12/go.mod h1:37HG2MBroXK3jXfxVGtbM2J48ra2+Ltu+tmwr/jO0KA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 h1:3aMfcTmoXtTZnaT86QlVaYh+BRMbvrrmZwIQ5jWqCZQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22/go.mod h1:YGSIJyQ6D6FjKMQh16hVFSIUD54L4F7zTGePqYMYYJU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= +github.com/aws/aws-sdk-go-v2/config v1.18.15 h1:509yMO0pJUGUugBP2H9FOFyV+7Mz7sRR+snfDN5W4NY= +github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15 h1:0rZQIi6deJFjOEgHI9HI2eZcLPPEGQPictX66oRFLL8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 h1:Kbiv9PGnQfG/imNI4L/heyUXvzKmcWSBeDvkrQz5pFc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 h1:9/aKwwus0TQxppPXFmf010DFrE+ssSbzroLVYINA+xE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 h1:b/Vn141DBuLVgXbhRWIrl9g+ww7G+ScV5SzniWR13jQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 h1:J4xhFd6zHhdF9jPP0FQJ6WknzBboGMBNjKOv4iTuw4A= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 h1:LjFQf8hFuMO22HkV5VWGLBvmCLBCLPivUAmpdpnp4Vs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22/go.mod h1:xt0Au8yPIwYXf/GYPy/vl4K3CgwhfQMYbrH7DlUUIws= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 h1:IVx9L7YFhpPq0tTnGo8u8TpluFu7nAn9X3sUDMb11c0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 h1:QoOybhwRfciWUBbZ0gp9S7XaDnCuSTeK/fySB99V1ls= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 h1:xxWtdHTL6YO5nqXYV0s6rYV7NpAc95EjztJ351oRdqA= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4/go.mod h1:UBNPuS7L4/FIjxAS92CO/5ljnaKHyAe1GeoLjqR8pvA= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 h1:lQKN/LNa3qqu2cDOQZybP7oL4nMGGiFqob0jZJaR8/4= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.1/go.mod h1:IgV8l3sj22nQDd5qcAGY0WenwCzCphqdbFOpfktZPrI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 h1:0bLhH6DRAqox+g0LatcjGKjjhU6Eudyys6HB6DJVPj8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1/go.mod h1:O1YSOg3aekZibh2SngvCRRG+cRHKKlYgxf/JBF/Kr/k= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 h1:s49mSnsBZEXjfGBkRfmK+nPqzT7Lt3+t2SmAKNyHblw= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.3/go.mod h1:b+psTJn33Q4qGoDaM7ZiOVVG8uVjGI6HaZ8WBHdgDgU= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 h1:qJdM48OOLl1FBSzI7ZrA1ZfLwOyCYqkXV5lko1hYDBw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 h1:YRkWXQveFb0tFC0TLktmmhGsOcCgLwvq88MC2al47AA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 h1:L1600eLr0YvTT7gNh3Ni24yGI7NSHkq9Gp62vijPRCs= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From c586797fb2e2ee66449b2fdc296ada891d75fafe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 12:47:12 +0530 Subject: [PATCH 151/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.18.15 to 1.18.19 (#283) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.15 to 1.18.19. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.15...config/v1.18.19) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 22 +++++++++++----------- go.sum | 41 ++++++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 88a54abd..cb5c77d8 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.17.5 - github.com/aws/aws-sdk-go-v2/config v1.18.15 + github.com/aws/aws-sdk-go-v2 v1.17.7 + github.com/aws/aws-sdk-go-v2/config v1.18.19 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 @@ -28,15 +28,15 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.15 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index aef8db5b..76116a43 100644 --- a/go.sum +++ b/go.sum @@ -74,30 +74,33 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.5 h1:TzCUW1Nq4H8Xscph5M/skINUitxM5UBAyvm2s7XBzL4= github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.15 h1:509yMO0pJUGUugBP2H9FOFyV+7Mz7sRR+snfDN5W4NY= -github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= -github.com/aws/aws-sdk-go-v2/credentials v1.13.15 h1:0rZQIi6deJFjOEgHI9HI2eZcLPPEGQPictX66oRFLL8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 h1:Kbiv9PGnQfG/imNI4L/heyUXvzKmcWSBeDvkrQz5pFc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 h1:9/aKwwus0TQxppPXFmf010DFrE+ssSbzroLVYINA+xE= +github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= +github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 h1:b/Vn141DBuLVgXbhRWIrl9g+ww7G+ScV5SzniWR13jQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 h1:IVx9L7YFhpPq0tTnGo8u8TpluFu7nAn9X3sUDMb11c0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 h1:QoOybhwRfciWUBbZ0gp9S7XaDnCuSTeK/fySB99V1ls= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 h1:xxWtdHTL6YO5nqXYV0s6rYV7NpAc95EjztJ351oRdqA= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4/go.mod h1:UBNPuS7L4/FIjxAS92CO/5ljnaKHyAe1GeoLjqR8pvA= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 h1:qJdM48OOLl1FBSzI7ZrA1ZfLwOyCYqkXV5lko1hYDBw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 h1:YRkWXQveFb0tFC0TLktmmhGsOcCgLwvq88MC2al47AA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 h1:L1600eLr0YvTT7gNh3Ni24yGI7NSHkq9Gp62vijPRCs= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= From 6bef32652d3b5a983b845405cb2f5e3d39852d0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 13:03:51 +0530 Subject: [PATCH 152/163] Bump github.com/aws/aws-sdk-go-v2/service/servicediscovery (#282) Bumps [github.com/aws/aws-sdk-go-v2/service/servicediscovery](https://github.com/aws/aws-sdk-go-v2) from 1.19.4 to 1.21.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/efs/v1.19.4...service/s3/v1.21.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/servicediscovery dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index cb5c77d8..3070e9d9 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2 v1.17.7 github.com/aws/aws-sdk-go-v2/config v1.18.19 - github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 + github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 diff --git a/go.sum b/go.sum index 76116a43..84be62b1 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= @@ -83,18 +82,16 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1 github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4 h1:xxWtdHTL6YO5nqXYV0s6rYV7NpAc95EjztJ351oRdqA= -github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.19.4/go.mod h1:UBNPuS7L4/FIjxAS92CO/5ljnaKHyAe1GeoLjqR8pvA= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0 h1:8Cq/VTVv8EbgDZo3G/0Rk5iUkAzvf+ydvw6ExKscj/w= +github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0/go.mod h1:T9ArVTDM6TUdMyfMGbULOLZMPwEnFhw1qjAoEj0VoHM= github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= From c55daf6c38be230ee31975dccc6bbc57c5545de7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:37:22 +0530 Subject: [PATCH 153/163] Bump github.com/go-logr/logr from 1.2.3 to 1.2.4 (#281) Bumps [github.com/go-logr/logr](https://github.com/go-logr/logr) from 1.2.3 to 1.2.4. - [Release notes](https://github.com/go-logr/logr/releases) - [Changelog](https://github.com/go-logr/logr/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-logr/logr/compare/v1.2.3...v1.2.4) --- updated-dependencies: - dependency-name: github.com/go-logr/logr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3070e9d9..c6ac7612 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.17.7 github.com/aws/aws-sdk-go-v2/config v1.18.19 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0 - github.com/go-logr/logr v1.2.3 + github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 84be62b1..e05b529c 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= From a66f87a63522b60072563fe5fc6697d8db60a45a Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Tue, 13 Feb 2024 11:00:34 -0800 Subject: [PATCH 154/163] Update README.md (#314) * Update README.md * Update README.md * Update step by step guide link --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 18fcd74e..7458d960 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/aws/aws-cloud-map-mcs-controller-for-k8s)](https://goreportcard.com/report/github.com/aws/aws-cloud-map-mcs-controller-for-k8s) ## Introduction -The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. We have detailed [step-by-step setup guide](https://aws.github.io/aws-cloud-map-mcs-controller-for-k8s/)! +The AWS Cloud Map Multi-cluster Service Discovery Controller for Kubernetes (K8s) implements the Kubernetes [KEP-1645: Multi-Cluster Services API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/1645-multi-cluster-services-api) and [KEP-2149: ClusterId for ClusterSet identification](https://github.com/kubernetes/enhancements/tree/master/keps/sig-multicluster/2149-clusterid), which allows services to communicate across multiple clusters. The implementation relies on [AWS Cloud Map](https://aws.amazon.com/cloud-map/) for enabling cross-cluster service discovery. We have detailed [step-by-step setup guide](https://aws.amazon.com/blogs/opensource/kubernetes-multi-cluster-service-discovery-using-open-source-aws-cloud-map-mcs-controller/)! -**NOTE: The current version [![GitHub Release](https://img.shields.io/github/release/aws/aws-cloud-map-mcs-controller-for-k8s.svg?style=flat&label=)]() is in *Alpha* phase, checkout the [Graduation Criteria](#graduation-criteria) for the next steps.** +**⚠ NOTE: The current version [![GitHub Release](https://img.shields.io/github/release/aws/aws-cloud-map-mcs-controller-for-k8s.svg?style=flat&label=)]() is in *Alpha* phase, and NOT intended for production use. The support will be limited to critical bug fixes.** + +*Checkout the [Graduation Criteria](#graduation-criteria) for moving the project to the next phase.* ## Installation @@ -28,7 +30,7 @@ Perform the following installation steps on each participating cluster. #### Network -> ⚠ **The AWS Cloud Map MCS Controller for K8s provides service discovery and communication across multiple clusters, therefore implementations depend on end-end network connectivity between workloads provisioned within each participating cluster.** +> **The AWS Cloud Map MCS Controller for K8s provides service discovery and communication across multiple clusters, therefore implementations depend on end-end network connectivity between workloads provisioned within each participating cluster.** - In deployment scenarios where participating clusters are provisioned into separate VPCs, connectivity will depend on correctly configured [VPC Peering](https://docs.aws.amazon.com/vpc/latest/peering/create-vpc-peering-connection.html), [inter-VPC routing](https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-routing.html), and Security Group configuration. The [VPC Reachability Analyzer](https://docs.aws.amazon.com/vpc/latest/reachability/getting-started.html) can be used to test and validate end-end connectivity between worker nodes within each cluster. - Undefined behavior may occur if controllers are deployed without the required network connectivity between clusters. From 12353020787799dae3cc665bbbe5ebd20b1da781 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:28:48 -0700 Subject: [PATCH 155/163] Bump github.com/stretchr/testify from 1.8.2 to 1.8.4 (#290) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.2 to 1.8.4. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.2...v1.8.4) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c6ac7612..68851434 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 golang.org/x/time v0.3.0 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 diff --git a/go.sum b/go.sum index e05b529c..9994d552 100644 --- a/go.sum +++ b/go.sum @@ -499,8 +499,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= From 632d1366a7828c5323a4ae0036af7b6739bc695b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:29:17 -0700 Subject: [PATCH 156/163] Bump github.com/aws/aws-sdk-go-v2 from 1.17.7 to 1.18.1 (#293) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.17.7 to 1.18.1. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.17.7...v1.18.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 68851434..a3dcc285 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.17.7 + github.com/aws/aws-sdk-go-v2 v1.18.1 github.com/aws/aws-sdk-go-v2/config v1.18.19 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0 github.com/go-logr/logr v1.2.4 diff --git a/go.sum b/go.sum index 9994d552..b541ef63 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,9 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= +github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= From a2a6b375fb6bb5d0d232e88e8a58f14363713404 Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Tue, 22 Oct 2024 15:38:31 -0700 Subject: [PATCH 157/163] Update github actions (#322) --- .github/workflows/build.yml | 14 +++++--------- .github/workflows/codeql-analysis.yml | 8 ++++---- .github/workflows/deploy.yml | 6 +++--- .github/workflows/integration-test.yml | 6 +++--- .github/workflows/mkdocs.yml | 4 ++-- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7dbed56b..d6cffed5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,26 +13,22 @@ jobs: steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: 1.17 - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Unit tests run: make test - name: Upload code coverage - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: cover.out - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v5 with: - version: v1.45.2 - # Optional: if set to true then the action don't cache or restore ~/go/pkg. - skip-pkg-cache: true - # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. - skip-build-cache: true + version: v1.50.1 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 802ce6b5..1fd747ee 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 254a796e..8893a1d9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,10 +14,10 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.17 @@ -25,7 +25,7 @@ jobs: run: make docker-build - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ffd15990..caae758a 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -15,15 +15,15 @@ jobs: id-token: write steps: - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-west-2 role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} role-session-name: IntegrationTestSession - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.17 - name: Set up env diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index 9cec4626..314e33ba 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -8,10 +8,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout main - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.x From 86141144ba7f2cdd1c7140e5b5ee222d78a18606 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 15:44:17 -0700 Subject: [PATCH 158/163] Bump golang.org/x/crypto from 0.1.0 to 0.17.0 (#320) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.1.0 to 0.17.0. - [Commits](https://github.com/golang/crypto/compare/v0.1.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 28 +++++++++++++++++----------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index a3dcc285..f3bd1e96 100644 --- a/go.mod +++ b/go.mod @@ -73,12 +73,12 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index b541ef63..b38a3613 100644 --- a/go.sum +++ b/go.sum @@ -570,8 +570,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -609,6 +609,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -661,9 +662,9 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -690,6 +691,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -760,14 +762,16 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -777,9 +781,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -849,6 +854,7 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 7c038ef1df83e87d17b567179991ebc9a10f6155 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 15:44:51 -0700 Subject: [PATCH 159/163] Bump google.golang.org/protobuf from 1.28.0 to 1.33.0 (#319) Bumps google.golang.org/protobuf from 1.28.0 to 1.33.0. --- updated-dependencies: - dependency-name: google.golang.org/protobuf dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f3bd1e96..88ff87a9 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( golang.org/x/text v0.14.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b38a3613..7cc5c8d0 100644 --- a/go.sum +++ b/go.sum @@ -971,8 +971,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 25cd6c1b46e05d60d41aebcae97e7ae6cec4c17e Mon Sep 17 00:00:00 2001 From: Akash Rungta Date: Wed, 23 Oct 2024 12:02:50 -0700 Subject: [PATCH 160/163] Update go version (#323) * Update go version * Update go version --- .github/workflows/build.yml | 2 +- .github/workflows/deploy.yml | 2 +- .github/workflows/integration-test.yml | 2 +- CONTRIBUTING.md | 2 +- Dockerfile | 2 +- go.mod | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d6cffed5..65185648 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.17 + go-version: 1.19 - name: checkout uses: actions/checkout@v4 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8893a1d9..fc5588d3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.17 + go-version: 1.19 - name: Build image run: make docker-build diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index caae758a..4214f557 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.17 + go-version: 1.19 - name: Set up env run: source ~/.bashrc - name: Start clean diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd948880..3fdd9f64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ make clean In order to build and run locally: -* Make sure to have `kubectl` [installed](https://kubernetes.io/docs/tasks/tools/install-kubectl/), at least version `1.17` or above. +* Make sure to have `kubectl` [installed](https://kubernetes.io/docs/tasks/tools/install-kubectl/), at least version `1.19` or above. * Make sure to have `kind` [installed](https://kind.sigs.k8s.io/docs/user/quick-start/#installation). * Make sure, you have access to AWS Cloud Map. As per exercise below, AWS Cloud Map namespace `example` of the type [HttpNamespace](https://docs.aws.amazon.com/cloud-map/latest/api/API_CreateHttpNamespace.html) will be automatically created. diff --git a/Dockerfile b/Dockerfile index 130b3a4e..1f742f3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.17 as builder +FROM golang:1.19 as builder WORKDIR /workspace diff --git a/go.mod b/go.mod index 88ff87a9..9b3fc08e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s -go 1.17 +go 1.19 require ( github.com/aws/aws-sdk-go-v2 v1.18.1 From 6a4c981aac47d8a4890785f95eec6bc30c41f5a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:42:12 -0700 Subject: [PATCH 161/163] Bump github.com/aws/aws-sdk-go-v2/config from 1.18.19 to 1.20.0 (#307) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.19 to 1.20.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/v1.20.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.19...v1.20.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Akash Rungta --- go.mod | 24 ++++++++++++------------ go.sum | 45 ++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 9b3fc08e..f9400cf5 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/aws/aws-cloud-map-mcs-controller-for-k8s go 1.19 require ( - github.com/aws/aws-sdk-go-v2 v1.18.1 - github.com/aws/aws-sdk-go-v2/config v1.18.19 + github.com/aws/aws-sdk-go-v2 v1.22.0 + github.com/aws/aws-sdk-go-v2/config v1.20.0 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0 github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 @@ -28,16 +28,16 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect - github.com/aws/smithy-go v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.16.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.18.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.24.0 // indirect + github.com/aws/smithy-go v1.16.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 7cc5c8d0..6e00f6e8 100644 --- a/go.sum +++ b/go.sum @@ -75,32 +75,35 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2 v1.22.0 h1:CpTS3XO3MWNel8ohoazkLZC6scvkYL2k+m0yzFJ17Hg= +github.com/aws/aws-sdk-go-v2 v1.22.0/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= +github.com/aws/aws-sdk-go-v2/config v1.20.0 h1:q2+/mqFhY0J9m3Tb5RGFE3R4sdaUkIe4k2EuDfE3c08= +github.com/aws/aws-sdk-go-v2/config v1.20.0/go.mod h1:7+1riCZXyT+sAGvneR5j+Zl1GyfbBUNQurpQTE6FP6k= +github.com/aws/aws-sdk-go-v2/credentials v1.14.0 h1:LQquqPE7cL55RQmA/UBoBKehDlEtMnQKm3B0Q672ePE= +github.com/aws/aws-sdk-go-v2/credentials v1.14.0/go.mod h1:q/3oaTPlamrQWHPwJe56Mjq9g1TYDgddvgTgWJtHTmE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.0 h1:lF/cVllNAPKgjDwN2RsQUX9g/f6hXer9f10ubLFSoug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.0/go.mod h1:c28nJNzMVVb9TQpZ5q4tzZvwEJwf/7So7Ie2s90l1Fw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.0 h1:tN6dNNE4SzMuyMnVtQJXGVKX177/d5Zy4MuA1HA4KUc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.0/go.mod h1:F6MXWETIeetAHwFHyoHEqrcB3NpijFv9nLP5h9CXtT0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.0 h1:bfdsbTARDjaC/dSYGMO+E0psxFU4hTvCLnqYAfZ3D38= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.0/go.mod h1:Jg8XVv5M2V2wiAMvBFx+O59jg6Yr8vhP0bgNF/IuquM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.4.0 h1:21tlTXq3ev10yLMAjXZzpkZbrl49h3ElSjmxD57tD/E= +github.com/aws/aws-sdk-go-v2/internal/ini v1.4.0/go.mod h1:d9YrBHJhyzDCv5UsEVRizHlFV6Q0sLemFq6uxuqWfUw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.0 h1:dJnwy5Awv+uvfk73aRENVbv1cSQQ60ydCkPaun097KM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.0/go.mod h1:RsPWWy7u/hwmFX57sQ7MLvrvJeYyNkiMm5BaavpoU18= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0 h1:8Cq/VTVv8EbgDZo3G/0Rk5iUkAzvf+ydvw6ExKscj/w= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.21.0/go.mod h1:T9ArVTDM6TUdMyfMGbULOLZMPwEnFhw1qjAoEj0VoHM= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/aws-sdk-go-v2/service/sso v1.16.0 h1:ZIlR6Wr/EgYwBdEz1NWBqdUsTh0mV7A68pId3YZl6H0= +github.com/aws/aws-sdk-go-v2/service/sso v1.16.0/go.mod h1:O7B5cpuhhJKefAKkM7onb0McmpHyKnsH4RrHJhOyq7M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.18.0 h1:3BZyJei4k1SHdSAFhg9Qg15NnG3v5zosZyFWPm7df/A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.18.0/go.mod h1:Td8EvzggonY02wLaqSpwybI3GbmA0PWoprKGil2uwJg= +github.com/aws/aws-sdk-go-v2/service/sts v1.24.0 h1:f/V5Y9OaHuNRrA9MntNQNAtMFXqhKj8HTEPnH81eXMI= +github.com/aws/aws-sdk-go-v2/service/sts v1.24.0/go.mod h1:HnCUMNz2XqwnEEk5X6oeDYB2HgOLFpJ/LyfilN8WErs= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= +github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 73f6c2a8779350bc755561547d1697706ab86c81 Mon Sep 17 00:00:00 2001 From: Jungrae Date: Thu, 13 Mar 2025 13:06:46 +0800 Subject: [PATCH 162/163] Add sync code when reconcille endpoint --- pkg/controllers/multicluster/cloudmap_controller.go | 6 +++++- pkg/controllers/multicluster/endpointslice_plan.go | 11 ++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index 9a945ba8..e1db9864 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -125,7 +125,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa } func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Service) error { - r.Log.Debug("syncing service", "namespace", svc.Namespace, "service", svc.Name) + r.Log.Info("syncing service", "namespace", svc.Namespace, "service", svc.Name) importedSvcPorts := ExtractServicePorts(svc.Endpoints) @@ -158,6 +158,7 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se clusterImportedSvcPorts := ExtractServicePorts(endpoints) derivedService, err := r.getDerivedService(ctx, svc.Namespace, svc.Name, clusterId) + if err != nil { if !errors.IsNotFound(err) { return err @@ -256,6 +257,9 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport } changes := plan.CalculateChanges() + // // log changes and endpoints + // r.Log.Info("changes", "create", len(changes.Create), "update", len(changes.Update), "delete", len(changes.Delete)) + // r.Log.Info("desiredEndpoints", "endpoints", desiredEndpoints) for _, sliceToUpdate := range changes.Update { r.Log.Debug("updating EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name) diff --git a/pkg/controllers/multicluster/endpointslice_plan.go b/pkg/controllers/multicluster/endpointslice_plan.go index c3bb996a..1653ea83 100644 --- a/pkg/controllers/multicluster/endpointslice_plan.go +++ b/pkg/controllers/multicluster/endpointslice_plan.go @@ -88,9 +88,18 @@ func (p *EndpointSlicePlan) trimSlices(desiredEndpoints map[string]*model.Endpoi // remove all undesired existing endpoints in slices for _, existingSlice := range p.Current { updatedEndpointList := make([]discovery.Endpoint, 0) + + sliceNeedsUpdateConditions := false + for _, existingEndpoint := range existingSlice.Endpoints { key := existingEndpoint.Addresses[0] + if _, found := desiredEndpoints[key]; found { + // if different ready status, set sliceNeedsUpdateConditions to true + if existingEndpoint.Conditions.Ready != &desiredEndpoints[key].Ready { + sliceNeedsUpdateConditions = true + existingEndpoint.Conditions.Ready = &desiredEndpoints[key].Ready + } updatedEndpointList = append(updatedEndpointList, existingEndpoint) delete(desiredEndpoints, key) } @@ -111,7 +120,7 @@ func (p *EndpointSlicePlan) trimSlices(desiredEndpoints map[string]*model.Endpoi } // slice needs to be updated if endpoint list changed - if len(updatedEndpointList) != len(existingSlice.Endpoints) { + if len(updatedEndpointList) != len(existingSlice.Endpoints) || sliceNeedsUpdateConditions { existingSlice.Endpoints = updatedEndpointList sliceNeedsUpdate = true } From f2f80caa70707e0406519c0e4672406b6458f883 Mon Sep 17 00:00:00 2001 From: Jungrae Date: Thu, 13 Mar 2025 13:38:21 +0800 Subject: [PATCH 163/163] fx --- pkg/controllers/multicluster/cloudmap_controller.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/controllers/multicluster/cloudmap_controller.go b/pkg/controllers/multicluster/cloudmap_controller.go index e1db9864..9a945ba8 100644 --- a/pkg/controllers/multicluster/cloudmap_controller.go +++ b/pkg/controllers/multicluster/cloudmap_controller.go @@ -125,7 +125,7 @@ func (r *CloudMapReconciler) reconcileNamespace(ctx context.Context, namespaceNa } func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Service) error { - r.Log.Info("syncing service", "namespace", svc.Namespace, "service", svc.Name) + r.Log.Debug("syncing service", "namespace", svc.Namespace, "service", svc.Name) importedSvcPorts := ExtractServicePorts(svc.Endpoints) @@ -158,7 +158,6 @@ func (r *CloudMapReconciler) reconcileService(ctx context.Context, svc *model.Se clusterImportedSvcPorts := ExtractServicePorts(endpoints) derivedService, err := r.getDerivedService(ctx, svc.Namespace, svc.Name, clusterId) - if err != nil { if !errors.IsNotFound(err) { return err @@ -257,9 +256,6 @@ func (r *CloudMapReconciler) updateEndpointSlices(ctx context.Context, svcImport } changes := plan.CalculateChanges() - // // log changes and endpoints - // r.Log.Info("changes", "create", len(changes.Create), "update", len(changes.Update), "delete", len(changes.Delete)) - // r.Log.Info("desiredEndpoints", "endpoints", desiredEndpoints) for _, sliceToUpdate := range changes.Update { r.Log.Debug("updating EndpointSlice", "namespace", sliceToUpdate.Namespace, "name", sliceToUpdate.Name)