From 9b1a08f2bfd0a0eb0472c4e2cdd3133ec0580c9c Mon Sep 17 00:00:00 2001 From: thediveo Date: Wed, 17 Jan 2024 20:14:43 +0100 Subject: [PATCH 1/5] test: log list-network timing for Docker, podman Signed-off-by: thediveo --- decorator/dockernet/dockernet.go | 4 ++++ decorator/podmannet/podmannet.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/decorator/dockernet/dockernet.go b/decorator/dockernet/dockernet.go index b274b13..2a4c0cb 100644 --- a/decorator/dockernet/dockernet.go +++ b/decorator/dockernet/dockernet.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/siemens/ghostwire/v2/decorator" "github.com/siemens/ghostwire/v2/network" @@ -69,6 +70,7 @@ type dockerNetworks struct { func makeDockerNetworks(ctx context.Context, engine *model.ContainerEngine, allnetns network.NetworkNamespaces) ( docknets dockerNetworks, ) { + start := time.Now() dockerclient, err := client.NewClientWithOpts( client.WithHost(engine.API), client.WithAPIVersionNegotiation()) @@ -79,6 +81,8 @@ func makeDockerNetworks(ctx context.Context, engine *model.ContainerEngine, alln } networks, _ := dockerclient.NetworkList(ctx, types.NetworkListOptions{}) _ = dockerclient.Close() + span := time.Since(start) + log.Debugf("docker(%d) network ls took %s", engine.PID, span) netnsid, _ := ops.NamespacePath(fmt.Sprintf("/proc/%d/ns/net", engine.PID)).ID() docknets.networks = networks docknets.engine = engine diff --git a/decorator/podmannet/podmannet.go b/decorator/podmannet/podmannet.go index 5ba14ec..a17540f 100644 --- a/decorator/podmannet/podmannet.go +++ b/decorator/podmannet/podmannet.go @@ -7,6 +7,7 @@ package podmannet import ( "context" "fmt" + "time" "github.com/siemens/ghostwire/v2/decorator" "github.com/siemens/ghostwire/v2/decorator/dockernet" @@ -47,6 +48,7 @@ type podmanNetworks struct { func makePodmanNetworks(ctx context.Context, engine *model.ContainerEngine, allnetns network.NetworkNamespaces) ( podmannets podmanNetworks, ) { + start := time.Now() libpodclient, err := newLibpodClient(engine.API) if err != nil { log.Warnf("cannot discover podman-managed networks from API %s, reason: %s", @@ -62,6 +64,8 @@ func makePodmanNetworks(ctx context.Context, engine *model.ContainerEngine, alln libpodclient.libpodVersion = info.Version.APIVersion networks, _ := libpodclient.networkList(ctx) _ = libpodclient.Close() + span := time.Since(start) + log.Debugf("podman(%d) network ls took %s", engine.PID, span) netnsid, _ := ops.NamespacePath(fmt.Sprintf("/proc/%d/ns/net", engine.PID)).ID() podmannets.networks = networks podmannets.engine = engine From 73d8f4d41f7c3363d76fc3905ba30a48f6be9aa4 Mon Sep 17 00:00:00 2001 From: thediveo Date: Wed, 17 Jan 2024 21:44:36 +0100 Subject: [PATCH 2/5] perf: don't use /info libpod endpoint for API version detection, as it is very slow Signed-off-by: thediveo --- decorator/podmannet/libpodclient.go | 25 +++---------------------- decorator/podmannet/podmannet.go | 10 ++-------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/decorator/podmannet/libpodclient.go b/decorator/podmannet/libpodclient.go index ba05be9..c46b0de 100644 --- a/decorator/podmannet/libpodclient.go +++ b/decorator/podmannet/libpodclient.go @@ -33,7 +33,7 @@ type Client struct { // // Please note that this libpod API client is absolutely minimalist and just // suffices for querying the podman-managed networks. -func newLibpodClient(endpoint string) (*Client, error) { +func newLibpodClient(endpoint string, libpodapiversion string) (*Client, error) { epurl, err := url.Parse(endpoint) if err != nil { return nil, fmt.Errorf("invalid endpoint, reason: %w", err) @@ -47,7 +47,8 @@ func newLibpodClient(endpoint string) (*Client, error) { DisableCompression: true, }, }, - endpointURL: epurl, + endpointURL: epurl, + libpodVersion: libpodapiversion, } dialer := &net.Dialer{ // same as Docker's unix socket default transport configuration, see @@ -121,26 +122,6 @@ func ensureReaderClosed(resp *http.Response) { resp.Body.Close() } -// essentialLibpodInformation grabs just the API version information from the -// JSON salad returned by a “/vX/libpod/info” endpoint. -type essentialLibpodInformation struct { - Version struct { - APIVersion string // major.minor.patch, without "v" prefix - } `json:"version"` -} - -// info returns the “essential” libpod information, that is, the libpod API -// version. -func (c *Client) info(ctx context.Context) (essentialLibpodInformation, error) { - resp, err := c.get(ctx, "/info") - var info essentialLibpodInformation - if err != nil { - return info, err - } - err = json.NewDecoder(resp.Body).Decode(&info) - return info, err -} - // NetworkResource grabs just the few things from a podman network we're // interested here for the purposes of correctly decorating network interfaces // with podman network names. We simply ignore all the other JSON salad returned diff --git a/decorator/podmannet/podmannet.go b/decorator/podmannet/podmannet.go index a17540f..8e43d45 100644 --- a/decorator/podmannet/podmannet.go +++ b/decorator/podmannet/podmannet.go @@ -49,19 +49,13 @@ func makePodmanNetworks(ctx context.Context, engine *model.ContainerEngine, alln podmannets podmanNetworks, ) { start := time.Now() - libpodclient, err := newLibpodClient(engine.API) + libpodclient, err := newLibpodClient(engine.API, "4") if err != nil { log.Warnf("cannot discover podman-managed networks from API %s, reason: %s", engine.API, err.Error()) return } - info, err := libpodclient.info(ctx) - if err != nil { - log.Warnf("cannot discover podman-managed networks from API %s, reason: %s", - engine.API, err.Error()) - return - } - libpodclient.libpodVersion = info.Version.APIVersion + libpodclient.libpodVersion = "4" networks, _ := libpodclient.networkList(ctx) _ = libpodclient.Close() span := time.Since(start) From d9ffd27870ee2484e27119dd16bc7f4bf0b56309 Mon Sep 17 00:00:00 2001 From: thediveo Date: Wed, 17 Jan 2024 22:16:03 +0100 Subject: [PATCH 3/5] fix: use _ping endpoint for libpod API version detection Signed-off-by: thediveo --- decorator/podmannet/libpodclient.go | 20 ++++++++++++++------ decorator/podmannet/podmannet.go | 4 ++-- decorator/podmannet/podmannet_test.go | 3 --- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/decorator/podmannet/libpodclient.go b/decorator/podmannet/libpodclient.go index c46b0de..9b0ecf7 100644 --- a/decorator/podmannet/libpodclient.go +++ b/decorator/podmannet/libpodclient.go @@ -33,7 +33,7 @@ type Client struct { // // Please note that this libpod API client is absolutely minimalist and just // suffices for querying the podman-managed networks. -func newLibpodClient(endpoint string, libpodapiversion string) (*Client, error) { +func newLibpodClient(endpoint string) (*Client, error) { epurl, err := url.Parse(endpoint) if err != nil { return nil, fmt.Errorf("invalid endpoint, reason: %w", err) @@ -47,8 +47,7 @@ func newLibpodClient(endpoint string, libpodapiversion string) (*Client, error) DisableCompression: true, }, }, - endpointURL: epurl, - libpodVersion: libpodapiversion, + endpointURL: epurl, } dialer := &net.Dialer{ // same as Docker's unix socket default transport configuration, see @@ -81,11 +80,11 @@ func (c *Client) Close() error { // this seems to be version-independent, but still needs any version in its // endpoint path. func (c *Client) apiPath(apipath string) string { - if c.libpodVersion == "" { + if apipath == "/_ping" { // use only for initial libpod info (API version) retrieval; please note - // that all libpod API endpoints are versioned, there are not + // that all libpod API endpoints are versioned, there are no // un-versioned endpoints like the Docker API does. - return path.Join("/v0/libpod", apipath) + return apipath } return path.Join("/v"+c.libpodVersion+"/libpod", apipath) } @@ -122,6 +121,15 @@ func ensureReaderClosed(resp *http.Response) { resp.Body.Close() } +func (c *Client) ping(ctx context.Context) (libpodAPIVersion string) { + resp, err := c.get(ctx, "/_ping") + defer ensureReaderClosed(resp) + if err != nil { + return "" + } + return resp.Header.Get("Libpod-Api-Version") +} + // NetworkResource grabs just the few things from a podman network we're // interested here for the purposes of correctly decorating network interfaces // with podman network names. We simply ignore all the other JSON salad returned diff --git a/decorator/podmannet/podmannet.go b/decorator/podmannet/podmannet.go index 8e43d45..54fb30a 100644 --- a/decorator/podmannet/podmannet.go +++ b/decorator/podmannet/podmannet.go @@ -49,13 +49,13 @@ func makePodmanNetworks(ctx context.Context, engine *model.ContainerEngine, alln podmannets podmanNetworks, ) { start := time.Now() - libpodclient, err := newLibpodClient(engine.API, "4") + libpodclient, err := newLibpodClient(engine.API) if err != nil { log.Warnf("cannot discover podman-managed networks from API %s, reason: %s", engine.API, err.Error()) return } - libpodclient.libpodVersion = "4" + libpodclient.libpodVersion = libpodclient.ping(ctx) networks, _ := libpodclient.networkList(ctx) _ = libpodclient.Close() span := time.Since(start) diff --git a/decorator/podmannet/podmannet_test.go b/decorator/podmannet/podmannet_test.go index 369211c..02d7479 100644 --- a/decorator/podmannet/podmannet_test.go +++ b/decorator/podmannet/podmannet_test.go @@ -29,9 +29,6 @@ const ( pindName = "ghostwire-pind" pindImageName = "siemens/ghostwire-pind" - spinupTimeout = 10 * time.Second - spinupPolling = 500 * time.Millisecond - goroutinesUnwindTimeout = 2 * time.Second goroutinesUnwindPolling = 250 * time.Millisecond ) From 6aacc311d5ffc72e477478a14c735f3da251d596 Mon Sep 17 00:00:00 2001 From: thediveo Date: Thu, 18 Jan 2024 12:46:58 +0100 Subject: [PATCH 4/5] chore: refactoring Signed-off-by: thediveo --- decorator/dockernet/dockernet.go | 4 ---- decorator/podmannet/libpodclient.go | 7 +++++++ decorator/podmannet/podmannet.go | 4 ---- defs_version.go | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/decorator/dockernet/dockernet.go b/decorator/dockernet/dockernet.go index 2a4c0cb..b274b13 100644 --- a/decorator/dockernet/dockernet.go +++ b/decorator/dockernet/dockernet.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "strings" - "time" "github.com/siemens/ghostwire/v2/decorator" "github.com/siemens/ghostwire/v2/network" @@ -70,7 +69,6 @@ type dockerNetworks struct { func makeDockerNetworks(ctx context.Context, engine *model.ContainerEngine, allnetns network.NetworkNamespaces) ( docknets dockerNetworks, ) { - start := time.Now() dockerclient, err := client.NewClientWithOpts( client.WithHost(engine.API), client.WithAPIVersionNegotiation()) @@ -81,8 +79,6 @@ func makeDockerNetworks(ctx context.Context, engine *model.ContainerEngine, alln } networks, _ := dockerclient.NetworkList(ctx, types.NetworkListOptions{}) _ = dockerclient.Close() - span := time.Since(start) - log.Debugf("docker(%d) network ls took %s", engine.PID, span) netnsid, _ := ops.NamespacePath(fmt.Sprintf("/proc/%d/ns/net", engine.PID)).ID() docknets.networks = networks docknets.engine = engine diff --git a/decorator/podmannet/libpodclient.go b/decorator/podmannet/libpodclient.go index 9b0ecf7..0f790d4 100644 --- a/decorator/podmannet/libpodclient.go +++ b/decorator/podmannet/libpodclient.go @@ -121,6 +121,13 @@ func ensureReaderClosed(resp *http.Response) { resp.Body.Close() } +// ping the /_ping API endpoint (which is unversioned) and return the value of +// the “Libpod-Api-Version” header that came back from this endpoint, or an +// empty string. The libpod API version is in semver format, without any “v” +// prefix. +// +// Use the returned API version to set Client.libpodVersion so that following +// libpod endpoint calls are properly versioned. func (c *Client) ping(ctx context.Context) (libpodAPIVersion string) { resp, err := c.get(ctx, "/_ping") defer ensureReaderClosed(resp) diff --git a/decorator/podmannet/podmannet.go b/decorator/podmannet/podmannet.go index 54fb30a..f604792 100644 --- a/decorator/podmannet/podmannet.go +++ b/decorator/podmannet/podmannet.go @@ -7,7 +7,6 @@ package podmannet import ( "context" "fmt" - "time" "github.com/siemens/ghostwire/v2/decorator" "github.com/siemens/ghostwire/v2/decorator/dockernet" @@ -48,7 +47,6 @@ type podmanNetworks struct { func makePodmanNetworks(ctx context.Context, engine *model.ContainerEngine, allnetns network.NetworkNamespaces) ( podmannets podmanNetworks, ) { - start := time.Now() libpodclient, err := newLibpodClient(engine.API) if err != nil { log.Warnf("cannot discover podman-managed networks from API %s, reason: %s", @@ -58,8 +56,6 @@ func makePodmanNetworks(ctx context.Context, engine *model.ContainerEngine, alln libpodclient.libpodVersion = libpodclient.ping(ctx) networks, _ := libpodclient.networkList(ctx) _ = libpodclient.Close() - span := time.Since(start) - log.Debugf("podman(%d) network ls took %s", engine.PID, span) netnsid, _ := ops.NamespacePath(fmt.Sprintf("/proc/%d/ns/net", engine.PID)).ID() podmannets.networks = networks podmannets.engine = engine diff --git a/defs_version.go b/defs_version.go index 843e2b9..c6109bc 100644 --- a/defs_version.go +++ b/defs_version.go @@ -4,4 +4,4 @@ package gostwire // SemVersion is the semantic version string of the ghostwire module. -const SemVersion = "2.1.18-12-gede2f48" +const SemVersion = "2.3.0-4-g1927622" From 58049eacdbf3aedf49767a3e30fd720f3fd967f0 Mon Sep 17 00:00:00 2001 From: thediveo Date: Thu, 18 Jan 2024 15:14:58 +0100 Subject: [PATCH 5/5] fix: podmannet test failing on Ubuntu 22.04LTS Signed-off-by: thediveo --- decorator/podmannet/_test/pind/Dockerfile | 13 +++++++++ decorator/podmannet/podmannet_test.go | 35 ++++++++++++----------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/decorator/podmannet/_test/pind/Dockerfile b/decorator/podmannet/_test/pind/Dockerfile index bf0b1a1..f263829 100644 --- a/decorator/podmannet/_test/pind/Dockerfile +++ b/decorator/podmannet/_test/pind/Dockerfile @@ -8,4 +8,17 @@ RUN dnf -y install \ dnf clean all && \ rm -rf /var/cache /var/log/dnf* /var/log/yum.* && \ systemctl enable podman.socket +RUN echo $'[containers]\n\ +netns="host"\n\ +userns="host"\n\ +ipcns="host"\n\ +utsns="host"\n\ +cgroupns="host"\n\ +cgroups="disabled"\n\ +log_driver = "k8s-file"\n\ +[engine]\n\ +cgroup_manager = "cgroupfs"\n\ +events_logger="file"\n\ +runtime="crun"\n\ +' > /etc/containers/containers.conf CMD [ "/usr/sbin/init" ] diff --git a/decorator/podmannet/podmannet_test.go b/decorator/podmannet/podmannet_test.go index 02d7479..e813b72 100644 --- a/decorator/podmannet/podmannet_test.go +++ b/decorator/podmannet/podmannet_test.go @@ -13,13 +13,14 @@ import ( "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "github.com/siemens/ghostwire/v2/internal/discover" + "github.com/siemens/ghostwire/v2/network" "github.com/siemens/turtlefinder" + "github.com/thediveo/lxkns/model" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gleak" . "github.com/thediveo/fdooze" - "github.com/thediveo/lxkns/model" . "github.com/thediveo/success" ) @@ -29,6 +30,9 @@ const ( pindName = "ghostwire-pind" pindImageName = "siemens/ghostwire-pind" + nifDiscoveryTimeout = 5 * time.Second + nifDiscoveryPolling = 250 * time.Millisecond + goroutinesUnwindTimeout = 2 * time.Second goroutinesUnwindPolling = 250 * time.Millisecond ) @@ -89,9 +93,9 @@ var _ = Describe("turtle finder", Ordered, Serial, func() { Repository: pindImageName, Privileged: true, Mounts: []string{ - "/var", // well, this actually is an unnamed volume + "/var/lib/containers", // well, this actually is an unnamed volume }, - Tty: true, + Tty: false, }, func(hc *docker.HostConfig) { hc.Init = false hc.Tmpfs = map[string]string{ @@ -135,7 +139,11 @@ var _ = Describe("turtle finder", Ordered, Serial, func() { By("running a canary container connected to the default 'podman' network") Expect(pindCntr.Exec([]string{ - "podman", "run", "-d", "-it", "--rm", "--name", "canary", "busybox", + "podman", "run", "-d", "--rm", + "--name", "canary", + "--net", "podman", /* WHAT?? otherwise doesn't connect the container??? */ + "busybox", + "/bin/sh", "-c", "while true; do sleep 1; done", }, dockertest.ExecOptions{ StdOut: GinkgoWriter, StdErr: GinkgoWriter, @@ -169,18 +177,12 @@ var _ = Describe("turtle finder", Ordered, Serial, func() { defer cizer.Close() By("running a full Ghostwire discovery that should pick up the podman networks") - allnetns, lxknsdisco := discover.Discover(ctx, cizer, nil) - Expect(lxknsdisco.Processes).To(HaveKey(model.PIDType(pindCntr.Container.State.Pid))) - pindNetnsID := lxknsdisco.Processes[model.PIDType(pindCntr.Container.State.Pid)]. - Namespaces[model.NetNS].ID() - Expect(pindNetnsID).NotTo(BeZero()) - Expect(allnetns).To(HaveKey(pindNetnsID)) - pindNetns := allnetns[pindNetnsID] - // We expect the following network interfaces to be present inside our - // podman-in-docker container: - // - eth0 ... a.k.a. the "mcwielahm" network - // - podman0 ... a.k.a. the "podman" network - Expect(pindNetns.Nifs).To(ContainElements( + Eventually(ctx, func() map[int]network.Interface { + allnetns, lxknsdisco := discover.Discover(ctx, cizer, nil) + pindNetnsID := lxknsdisco.Processes[model.PIDType(pindCntr.Container.State.Pid)]. + Namespaces[model.NetNS].ID() + return allnetns[pindNetnsID].Nifs + }).Within(nifDiscoveryPolling).ProbeEvery(nifDiscoveryPolling).Should(ContainElements( HaveField("Nif()", And( HaveField("Name", "eth0"), HaveField("Alias", "mcwielahm"))), @@ -188,6 +190,7 @@ var _ = Describe("turtle finder", Ordered, Serial, func() { HaveField("Name", "podman0"), HaveField("Alias", "podman"))), )) + }) })