Skip to content

Commit

Permalink
add resource quotas
Browse files Browse the repository at this point in the history
  • Loading branch information
@ivelichkovich
ivelichkovich committed Apr 1, 2020
1 parent caa2ba6 commit dcbf0f1
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 15 deletions.
1 change: 1 addition & 0 deletions docs/how_to/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ It is recommended that you also check the [DSF spec](../desired_state_specificat
- [Label namespaces](namespaces/labels_and_annotations.md)
- [Set resource limits for namespaces](namespaces/limits.md)
- [Protecting namespaces](namespaces/protection.md)
- [Namespace resource quotas](namespaces/quotas.md)
- Defining Helm repositories
- [Using default helm repos](helm_repos/default.md)
- [Using private repos in Google GCS](helm_repos/gcs.md)
Expand Down
44 changes: 44 additions & 0 deletions docs/how_to/namespaces/quotas.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
---
version: 3.3.0
---

# Define resource quotas for namespaces

You can define namespaces to be used in your cluster. If they don't exist, Helmsman will create them for you. You can also define how much resource limits to set for each namespace.

You can read more about the `Quotas` specification [here](https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/quota-memory-cpu-namespace/#create-a-resourcequota).

```toml
#...
[namespaces]

[namespaces.helmsman1]

[namespaces.helmsman1.quotas]
"limits.cpu" = "10"
"limits.memory" = "30Gi"
pods = "25"
"requests.cpu" = "10"
"requests.memory" = "30Gi"

[[namespaces.helmsman1.quotas.customQuotas]]
name = "requests.nvidia.com/gpu"
value = "2"
#...
```

```yaml
namespaces:
helmsman1:
quotas:
limits.cpu: '10'
limits.memory: '30Gi'
pods: '25'
requests.cpu: '10'
requests.memory: '30Gi'
customQuotas:
- name: 'requests.nvidia.com/gpu'
value: '2'
```
The example above will create one namespace - helmsman1 - with resource quotas defined for the helmsman1 namespace.
9 changes: 9 additions & 0 deletions examples/example.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,15 @@ context= "test-infra" # defaults to "default" if not provided
protected = false
[namespaces.staging.labels]
env = "staging"
[namespaces.staging.quotas]
"limits.cpu" = "10"
"limits.memory" = "30Gi"
pods = "25"
"requests.cpu" = "10"
"requests.memory" = "30Gi"
[[namespaces.helmsman1.quotas.customQuotas]]
name = "requests.nvidia.com/gpu"
value = "2"


# define any private/public helm charts repos you would like to get charts from
Expand Down
44 changes: 44 additions & 0 deletions internal/app/kube_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func addNamespaces(s *state) {
labelNamespace(name, cfg.Labels)
annotateNamespace(name, cfg.Annotations)
setLimits(name, cfg.Limits)
setQuotas(name, cfg.Quotas)
}(nsName, ns, &wg)
}
wg.Wait()
Expand Down Expand Up @@ -140,6 +141,49 @@ spec:

}

func setQuotas(ns string, quotas *quotas) {
if quotas == nil {
return
}

definition := `
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: resource-quota
spec:
hard:
`

for _, customQuota := range quotas.CustomQuotas {
definition = definition + Indent(customQuota.Name+": '"+customQuota.Value+"'\n", strings.Repeat(" ", 4))
}

//Special formatting for custom quotas so manually write these and then set to nil for marshalling
quotas.CustomQuotas = nil

d, err := yaml.Marshal(&quotas)
if err != nil {
log.Fatal(err.Error())
}

definition = definition + Indent(string(d), strings.Repeat(" ", 4))

if err := ioutil.WriteFile("temp-ResourceQuota.yaml", []byte(definition), 0666); err != nil {
log.Fatal(err.Error())
}

cmd := kubectl([]string{"apply", "-f", "temp-ResourceQuota.yaml", "-n", ns}, "Creating ResourceQuota in namespace [ "+ns+" ]")
result := cmd.exec()

deleteFile("temp-ResourceQuota.yaml")

if result.code != 0 {
log.Fatal("ERROR: failed to create ResourceQuota in namespace [ " + ns + " ]: " + result.errors)
}
}

// createContext creates a context -connecting to a k8s cluster- in kubectl config.
// It returns true if successful, false otherwise
func createContext(s *state) error {
Expand Down
17 changes: 17 additions & 0 deletions internal/app/namespace.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ type resources struct {
Memory string `yaml:"memory,omitempty"`
}

// custom resource type
type customResource struct {
Name string `yaml:"name,omitempty"`
Value string `yaml:"value,omitempty"`
}

// limits type
type limits []struct {
Max resources `yaml:"max,omitempty"`
Expand All @@ -20,12 +26,23 @@ type limits []struct {
Type string `yaml:"type"`
}

// quota type
type quotas struct {
Pods string `yaml:"pods,omitempty"`
CPULimits string `yaml:"limits.cpu,omitempty"`
CPURequests string `yaml:"requests.cpu,omitempty"`
MemoryLimits string `yaml:"limits.memory,omitempty"`
MemoryRequests string `yaml:"requests.memory,omitempty"`
CustomQuotas []customResource `yaml:"customQuotas,omitempty"`
}

// namespace type represents the fields of a namespace
type namespace struct {
Protected bool `yaml:"protected"`
Limits limits `yaml:"limits,omitempty"`
Labels map[string]string `yaml:"labels"`
Annotations map[string]string `yaml:"annotations"`
Quotas *quotas `yaml:"quotas,omitempty"`
}

// print prints the namespace
Expand Down
2 changes: 1 addition & 1 deletion internal/app/release_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func Test_validateRelease(t *testing.T) {
Metadata: make(map[string]string),
Certificates: make(map[string]string),
Settings: (config{}),
Namespaces: map[string]namespace{"namespace": namespace{false, limits{}, make(map[string]string), make(map[string]string)}},
Namespaces: map[string]namespace{"namespace": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}}},
HelmRepos: make(map[string]string),
Apps: make(map[string]*release),
}
Expand Down
28 changes: 14 additions & 14 deletions internal/app/state_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "https://192.168.99.100:8443",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -58,7 +58,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "https://192.168.99.100:8443",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -79,7 +79,7 @@ func Test_state_validate(t *testing.T) {
KubeContext: "minikube",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -103,7 +103,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "https://192.168.99.100:8443",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -127,7 +127,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "$URI", // unset env
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -151,7 +151,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "https//192.168.99.100:8443", // invalid url
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -174,7 +174,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "https://192.168.99.100:8443",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -195,7 +195,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "https://192.168.99.100:8443",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -219,7 +219,7 @@ func Test_state_validate(t *testing.T) {
ClusterURI: "https://192.168.99.100:8443",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -237,7 +237,7 @@ func Test_state_validate(t *testing.T) {
KubeContext: "minikube",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand Down Expand Up @@ -287,7 +287,7 @@ func Test_state_validate(t *testing.T) {
KubeContext: "minikube",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: nil,
Apps: make(map[string]*release),
Expand All @@ -302,7 +302,7 @@ func Test_state_validate(t *testing.T) {
KubeContext: "minikube",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{},
Apps: make(map[string]*release),
Expand All @@ -317,7 +317,7 @@ func Test_state_validate(t *testing.T) {
KubeContext: "minikube",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand All @@ -335,7 +335,7 @@ func Test_state_validate(t *testing.T) {
KubeContext: "minikube",
},
Namespaces: map[string]namespace{
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string)},
"staging": namespace{false, limits{}, make(map[string]string), make(map[string]string), &quotas{}},
},
HelmRepos: map[string]string{
"stable": "https://kubernetes-charts.storage.googleapis.com",
Expand Down

0 comments on commit dcbf0f1

Please sign in to comment.