From dcbf0f1ce53c6bf6bfd993e8f9adfaaa7230c829 Mon Sep 17 00:00:00 2001 From: ivelichkovich Date: Fri, 17 Jan 2020 14:45:10 -0600 Subject: [PATCH] add resource quotas --- docs/how_to/README.md | 1 + docs/how_to/namespaces/quotas.md | 44 ++++++++++++++++++++++++++++++++ examples/example.toml | 9 +++++++ internal/app/kube_helpers.go | 44 ++++++++++++++++++++++++++++++++ internal/app/namespace.go | 17 ++++++++++++ internal/app/release_test.go | 2 +- internal/app/state_test.go | 28 ++++++++++---------- 7 files changed, 130 insertions(+), 15 deletions(-) create mode 100644 docs/how_to/namespaces/quotas.md diff --git a/docs/how_to/README.md b/docs/how_to/README.md index 7eb49fb7..1f24bfbf 100644 --- a/docs/how_to/README.md +++ b/docs/how_to/README.md @@ -17,6 +17,7 @@ It is recommended that you also check the [DSF spec](../desired_state_specificat - [Label namespaces](namespaces/labels_and_annotations.md) - [Set resource limits for namespaces](namespaces/limits.md) - [Protecting namespaces](namespaces/protection.md) + - [Namespace resource quotas](namespaces/quotas.md) - Defining Helm repositories - [Using default helm repos](helm_repos/default.md) - [Using private repos in Google GCS](helm_repos/gcs.md) diff --git a/docs/how_to/namespaces/quotas.md b/docs/how_to/namespaces/quotas.md new file mode 100644 index 00000000..348c9213 --- /dev/null +++ b/docs/how_to/namespaces/quotas.md @@ -0,0 +1,44 @@ +--- +version: 3.3.0 +--- + +# Define resource quotas for namespaces + +You can define namespaces to be used in your cluster. If they don't exist, Helmsman will create them for you. You can also define how much resource limits to set for each namespace. + +You can read more about the `Quotas` specification [here](https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/quota-memory-cpu-namespace/#create-a-resourcequota). + +```toml +#... +[namespaces] + + [namespaces.helmsman1] + + [namespaces.helmsman1.quotas] + "limits.cpu" = "10" + "limits.memory" = "30Gi" + pods = "25" + "requests.cpu" = "10" + "requests.memory" = "30Gi" + + [[namespaces.helmsman1.quotas.customQuotas]] + name = "requests.nvidia.com/gpu" + value = "2" +#... +``` + +```yaml +namespaces: + helmsman1: + quotas: + limits.cpu: '10' + limits.memory: '30Gi' + pods: '25' + requests.cpu: '10' + requests.memory: '30Gi' + customQuotas: + - name: 'requests.nvidia.com/gpu' + value: '2' +``` + +The example above will create one namespace - helmsman1 - with resource quotas defined for the helmsman1 namespace. \ No newline at end of file diff --git a/examples/example.toml b/examples/example.toml index bef218c5..04c99ab3 100644 --- a/examples/example.toml +++ b/examples/example.toml @@ -58,6 +58,15 @@ context= "test-infra" # defaults to "default" if not provided protected = false [namespaces.staging.labels] env = "staging" + [namespaces.staging.quotas] + "limits.cpu" = "10" + "limits.memory" = "30Gi" + pods = "25" + "requests.cpu" = "10" + "requests.memory" = "30Gi" + [[namespaces.helmsman1.quotas.customQuotas]] + name = "requests.nvidia.com/gpu" + value = "2" # define any private/public helm charts repos you would like to get charts from diff --git a/internal/app/kube_helpers.go b/internal/app/kube_helpers.go index 9c720d1c..4e2f729e 100644 --- a/internal/app/kube_helpers.go +++ b/internal/app/kube_helpers.go @@ -29,6 +29,7 @@ func addNamespaces(s *state) { labelNamespace(name, cfg.Labels) annotateNamespace(name, cfg.Annotations) setLimits(name, cfg.Limits) + setQuotas(name, cfg.Quotas) }(nsName, ns, &wg) } wg.Wait() @@ -140,6 +141,49 @@ spec: } +func setQuotas(ns string, quotas *quotas) { + if quotas == nil { + return + } + + definition := ` +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: resource-quota +spec: + hard: +` + + for _, customQuota := range quotas.CustomQuotas { + definition = definition + Indent(customQuota.Name+": '"+customQuota.Value+"'\n", strings.Repeat(" ", 4)) + } + + //Special formatting for custom quotas so manually write these and then set to nil for marshalling + quotas.CustomQuotas = nil + + d, err := yaml.Marshal("as) + if err != nil { + log.Fatal(err.Error()) + } + + definition = definition + Indent(string(d), strings.Repeat(" ", 4)) + + if err := ioutil.WriteFile("temp-ResourceQuota.yaml", []byte(definition), 0666); err != nil { + log.Fatal(err.Error()) + } + + cmd := kubectl([]string{"apply", "-f", "temp-ResourceQuota.yaml", "-n", ns}, "Creating ResourceQuota in namespace [ "+ns+" ]") + result := cmd.exec() + + deleteFile("temp-ResourceQuota.yaml") + + if result.code != 0 { + log.Fatal("ERROR: failed to create ResourceQuota in namespace [ " + ns + " ]: " + result.errors) + } +} + // createContext creates a context -connecting to a k8s cluster- in kubectl config. // It returns true if successful, false otherwise func createContext(s *state) error { diff --git a/internal/app/namespace.go b/internal/app/namespace.go index 37cb7d5b..9cbc918a 100644 --- a/internal/app/namespace.go +++ b/internal/app/namespace.go @@ -10,6 +10,12 @@ type resources struct { Memory string `yaml:"memory,omitempty"` } +// custom resource type +type customResource struct { + Name string `yaml:"name,omitempty"` + Value string `yaml:"value,omitempty"` +} + // limits type type limits []struct { Max resources `yaml:"max,omitempty"` @@ -20,12 +26,23 @@ type limits []struct { Type string `yaml:"type"` } +// quota type +type quotas struct { + Pods string `yaml:"pods,omitempty"` + CPULimits string `yaml:"limits.cpu,omitempty"` + CPURequests string `yaml:"requests.cpu,omitempty"` + MemoryLimits string `yaml:"limits.memory,omitempty"` + MemoryRequests string `yaml:"requests.memory,omitempty"` + CustomQuotas []customResource `yaml:"customQuotas,omitempty"` +} + // namespace type represents the fields of a namespace type namespace struct { Protected bool `yaml:"protected"` Limits limits `yaml:"limits,omitempty"` Labels map[string]string `yaml:"labels"` Annotations map[string]string `yaml:"annotations"` + Quotas *quotas `yaml:"quotas,omitempty"` } // print prints the namespace diff --git a/internal/app/release_test.go b/internal/app/release_test.go index 99ce5b4f..88ea74cb 100644 --- a/internal/app/release_test.go +++ b/internal/app/release_test.go @@ -26,7 +26,7 @@ func Test_validateRelease(t *testing.T) { Metadata: make(map[string]string), Certificates: make(map[string]string), Settings: (config{}), - Namespaces: map[string]namespace{"namespace": namespace{false, limits{}, make(map[string]string), make(map[string]string)}}, + Namespaces: map[string]namespace{"namespace": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}}, HelmRepos: make(map[string]string), Apps: make(map[string]*release), } diff --git a/internal/app/state_test.go b/internal/app/state_test.go index 3599950b..c919a099 100644 --- a/internal/app/state_test.go +++ b/internal/app/state_test.go @@ -34,7 +34,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "https://192.168.99.100:8443", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -58,7 +58,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "https://192.168.99.100:8443", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -79,7 +79,7 @@ func Test_state_validate(t *testing.T) { KubeContext: "minikube", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -103,7 +103,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "https://192.168.99.100:8443", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -127,7 +127,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "$URI", // unset env }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -151,7 +151,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "https//192.168.99.100:8443", // invalid url }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -174,7 +174,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "https://192.168.99.100:8443", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -195,7 +195,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "https://192.168.99.100:8443", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -219,7 +219,7 @@ func Test_state_validate(t *testing.T) { ClusterURI: "https://192.168.99.100:8443", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -237,7 +237,7 @@ func Test_state_validate(t *testing.T) { KubeContext: "minikube", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -287,7 +287,7 @@ func Test_state_validate(t *testing.T) { KubeContext: "minikube", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: nil, Apps: make(map[string]*release), @@ -302,7 +302,7 @@ func Test_state_validate(t *testing.T) { KubeContext: "minikube", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{}, Apps: make(map[string]*release), @@ -317,7 +317,7 @@ func Test_state_validate(t *testing.T) { KubeContext: "minikube", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com", @@ -335,7 +335,7 @@ func Test_state_validate(t *testing.T) { KubeContext: "minikube", }, Namespaces: map[string]namespace{ - "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)}, + "staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), "as{}}, }, HelmRepos: map[string]string{ "stable": "https://kubernetes-charts.storage.googleapis.com",