From f0deb59b4fbf7bcc62597a1299f4544638daa81a Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 8 Sep 2023 01:42:33 -0700 Subject: [PATCH] ulimit-adjuster: new sample plugin Signed-off-by: Samuel Karp --- README.md | 1 + plugins/ulimit-adjuster/README.md | 36 ++++ plugins/ulimit-adjuster/adjuster.go | 161 +++++++++++++++++ plugins/ulimit-adjuster/adjuster_test.go | 163 ++++++++++++++++++ .../ulimit-adjuster/sample-ulimit-adjust.yaml | 23 +++ 5 files changed, 384 insertions(+) create mode 100644 plugins/ulimit-adjuster/README.md create mode 100644 plugins/ulimit-adjuster/adjuster.go create mode 100644 plugins/ulimit-adjuster/adjuster_test.go create mode 100644 plugins/ulimit-adjuster/sample-ulimit-adjust.yaml diff --git a/README.md b/README.md index bbc467a5..fbb80964 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,7 @@ The following sample plugins exist for NRI: - [differ](plugins/differ) - [device injector](plugins/device-injector) - [OCI hook injector](plugins/hook-injector) + - [ulimit adjuster](plugins/ulimit-adjuster) - [NRI v0.1.0 plugin adapter](plugins/v010-adapter) Please see the documentation of these plugins for further details diff --git a/plugins/ulimit-adjuster/README.md b/plugins/ulimit-adjuster/README.md new file mode 100644 index 00000000..b6ee18c8 --- /dev/null +++ b/plugins/ulimit-adjuster/README.md @@ -0,0 +1,36 @@ +## ulimit Adjuster Plugin + +This sample plugin can adjust ulimits for containers using pod annotations. + +### Annotations + +ulimits are annotated using the key +`ulimits.nri.containerd.io/container.$CONTAINER_NAME`, which adjusts ulimits +for `$CONTAINER_NAME`. The ulimit names are the valid names of Linux resource +limits, which can be seen on the +[`setrlimit(2)` manual page](https://linux.die.net/man/2/setrlimit). + +The annotation syntax for ulimit adjustment is + +``` +- type: RLIMIT_NOFILE + soft: 1024 + hard: 4096 +- path: RLIMIT_MEMLOCK + soft: 1073741824 + hard: 1073741824 + ... +``` + +All fields are mandatory (`soft` and `hard` will be interpreted as 0 if +missing). The `type` field accepts names in uppercase letters +("RLIMIT_NOFILE"), lowercase letters ("rlimit_memlock"), and omitting the +"RLIMIT_" prefix ("nproc"). + +## Testing + +You can test this plugin using a kubernetes cluster/node with a container +runtime that has NRI support enabled. Start the plugin on the target node +(`ulimit-adjuster -idx 10`), create a pod with some annotated ulimits, then +verify that those get adjusted in the container. See the +[sample pod spec](sample-ulimit-adjust.yaml) for an example. \ No newline at end of file diff --git a/plugins/ulimit-adjuster/adjuster.go b/plugins/ulimit-adjuster/adjuster.go new file mode 100644 index 00000000..006d0e43 --- /dev/null +++ b/plugins/ulimit-adjuster/adjuster.go @@ -0,0 +1,161 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + + "github.com/containerd/containerd/log" + "github.com/sirupsen/logrus" + "sigs.k8s.io/yaml" + + "github.com/containerd/nri/pkg/api" + "github.com/containerd/nri/pkg/stub" +) + +const ( + ulimitKey = "ulimits.nri.containerd.io" + rlimitPrefix = "RLIMIT_" +) + +var ( + valid = map[string]struct{}{ + "AS": {}, + "CORE": {}, + "CPU": {}, + "DATA": {}, + "FSIZE": {}, + "LOCKS": {}, + "MEMLOCK": {}, + "MSGQUEUE": {}, + "NICE": {}, + "NOFILE": {}, + "NPROC": {}, + "RSS": {}, + "RTPRIO": {}, + "RTTIME": {}, + "SIGPENDING": {}, + "STACK": {}, + } +) + +func main() { + var ( + pluginName string + pluginIdx string + verbose bool + opts []stub.Option + ) + + l := logrus.StandardLogger() + l.SetFormatter(&logrus.TextFormatter{ + PadLevelText: true, + }) + + flag.StringVar(&pluginName, "name", "", "plugin name to register to NRI") + flag.StringVar(&pluginIdx, "idx", "", "plugin index to register to NRI") + flag.BoolVar(&verbose, "verbose", false, "enable (more) verbose logging") + flag.Parse() + ctx := log.WithLogger(context.Background(), l.WithField("name", pluginName).WithField("idx", pluginIdx)) + log.G(ctx).WithField("verbose", verbose).Info("starting plugin") + + if verbose { + l.SetLevel(logrus.DebugLevel) + } + + if pluginName != "" { + opts = append(opts, stub.WithPluginName(pluginName)) + } + if pluginIdx != "" { + opts = append(opts, stub.WithPluginIdx(pluginIdx)) + } + + p := &plugin{l: log.G(ctx)} + var err error + if p.stub, err = stub.New(p, opts...); err != nil { + log.G(ctx).Fatalf("failed to create plugin stub: %v", err) + } + + if err := p.stub.Run(context.Background()); err != nil { + log.G(ctx).Errorf("plugin exited with error %v", err) + os.Exit(1) + } +} + +type plugin struct { + stub stub.Stub + l *logrus.Entry +} + +func (p *plugin) CreateContainer( + ctx context.Context, + pod *api.PodSandbox, + container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { + if pod != nil { + p.l = p.l.WithField("pod", pod.Name) + } + if container != nil { + p.l = p.l.WithField("container", container.Name) + } + ctx = log.WithLogger(ctx, p.l) + log.G(ctx).Debug("create container") + + ulimits, err := parseUlimits(ctx, container.Name, pod.Annotations) + if err != nil { + log.G(ctx).WithError(err).Debug("failed to parse annotations") + return nil, nil, err + } + adjust := &api.ContainerAdjustment{} + for _, u := range ulimits { + log.G(ctx).WithField("type", u.Type).WithField("hard", u.Hard).WithField("soft", u.Soft).Debug("adjust rlimit") + adjust.AddRlimit(u.Type, u.Hard, u.Soft) + } + return adjust, nil, nil +} + +type ulimit struct { + Type string `json:"type"` + Hard uint64 `json:"hard"` + Soft uint64 `json:"soft"` +} + +func parseUlimits(ctx context.Context, container string, annotations map[string]string) ([]ulimit, error) { + key := ulimitKey + "/container." + container + val, ok := annotations[key] + if !ok { + log.G(ctx).Debugf("no annotations found with key %q", key) + return nil, nil + } + ulimits := make([]ulimit, 0) + if err := yaml.Unmarshal([]byte(val), &ulimits); err != nil { + return nil, err + } + for i := range ulimits { + u := ulimits[i] + typ := strings.TrimPrefix(strings.ToUpper(u.Type), rlimitPrefix) + if _, ok := valid[typ]; !ok { + log.G(ctx).WithField("raw", u.Type).WithField("trimmed", typ).Debug("failed to parse type") + return nil, fmt.Errorf("failed to parse type: %q", u.Type) + } + ulimits[i].Type = rlimitPrefix + typ + } + return ulimits, nil +} diff --git a/plugins/ulimit-adjuster/adjuster_test.go b/plugins/ulimit-adjuster/adjuster_test.go new file mode 100644 index 00000000..f9856af0 --- /dev/null +++ b/plugins/ulimit-adjuster/adjuster_test.go @@ -0,0 +1,163 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseAnnotations(t *testing.T) { + tests := map[string]struct { + container string + annotations map[string]string + expected []ulimit + errStr string + }{ + "no-annotations": { + container: "foo", + }, + "unrelated-annotation": { + container: "foo", + annotations: map[string]string{"bar": "baz"}, + }, + "one-valid": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: RLIMIT_NOFILE + soft: 123 + hard: 456 +`}, + expected: []ulimit{{ + Type: "RLIMIT_NOFILE", + Hard: 456, + Soft: 123, + }}, + }, + "multiple-valid": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: RLIMIT_NOFILE + soft: 123 + hard: 456 +- type: RLIMIT_NPROC + soft: 456 + hard: 789 +`}, + expected: []ulimit{{ + Type: "RLIMIT_NOFILE", + Hard: 456, + Soft: 123, + }, { + Type: "RLIMIT_NPROC", + Hard: 789, + Soft: 456, + }}, + }, + "missing-prefix": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: AS + soft: 123 + hard: 456 +`}, + expected: []ulimit{{ + Type: "RLIMIT_AS", + Hard: 456, + Soft: 123, + }}, + }, + "lower-case": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: rlimit_core + soft: 123 + hard: 456 +`}, + expected: []ulimit{{ + Type: "RLIMIT_CORE", + Hard: 456, + Soft: 123, + }}, + }, + "lower-case-missing-prefix": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: cpu + soft: 123 + hard: 456 +`}, + expected: []ulimit{{ + Type: "RLIMIT_CPU", + Hard: 456, + Soft: 123, + }}, + }, + "invalid-prefix": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: ULIMIT_NOFILE + soft: 123 + hard: 456 +`}, + errStr: `failed to parse type: "ULIMIT_NOFILE"`, + }, + "invalid-rlimit": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: RLIMIT_FOO + soft: 123 + hard: 456 +`}, + errStr: `failed to parse type: "RLIMIT_FOO"`, + }, + "one-invalid": { + container: "foo", + annotations: map[string]string{ + "ulimits.nri.containerd.io/container.foo": ` +- type: RLIMIT_NICE + soft: 456 + hard: 789 +- type: RLIMIT_BAR + soft: 123 + hard: 456 +`}, + errStr: `failed to parse type: "RLIMIT_BAR"`, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + ulimits, err := parseUlimits(context.Background(), tc.container, tc.annotations) + if tc.errStr != "" { + assert.EqualError(t, err, tc.errStr) + assert.Nil(t, ulimits) + } else { + assert.NoError(t, err) + assert.EqualValues(t, tc.expected, ulimits) + } + }) + } +} diff --git a/plugins/ulimit-adjuster/sample-ulimit-adjust.yaml b/plugins/ulimit-adjuster/sample-ulimit-adjust.yaml new file mode 100644 index 00000000..343be24f --- /dev/null +++ b/plugins/ulimit-adjuster/sample-ulimit-adjust.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: sleep + annotations: + ulimits.nri.containerd.io/container.sleep: | + - type: memlock + hard: 987654 + soft: 645321 + - type: RLIMIT_NOFILE + hard: 4096 + soft: 1024 + - type: nproc + hard: 9000 +spec: + containers: + - name: sleep + image: ubuntu:latest + command: + - /bin/bash + - -c + - "ulimit -a; ulimit -Ha; sleep inf" + terminationGracePeriodSeconds: 3