Skip to content

Commit

Permalink
test/functional: Add flag for container layer paths, remove container…
Browse files Browse the repository at this point in the history 
…d dependence (#1536)

* Add flag for container layer paths

Add lcow/wcow layer paths flag to allow providing image layer paths
instead of requiring containerd to pull and unpack it.

The goal is to allow functional tests to be run without requiring
containerd to be installed.

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

* PR: logging, cleanup, naming.

Changed `(LazyImageLayers).ImageLayers` to `(LazyImageLayers).Layers`.
`(LazyImageLayers).Close` now returns an error.
Comment cleanup.

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

* PR: replace LayerFolders and docker

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

* PR: unpack individual LCOW layers, comments

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>
  • Loading branch information
@helsaawy
helsaawy committed Nov 15, 2022
1 parent acd87d5 commit 1d141fa
Show file tree
Hide file tree
Showing 17 changed files with 351 additions and 178 deletions.
7 changes: 2 additions & 5 deletions test/functional/lcow_container_bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package functional

import (
"context"
"testing"

ctrdoci "github.com/containerd/containerd/oci"
Expand All @@ -15,7 +14,6 @@ import (
"github.com/Microsoft/hcsshim/osversion"

"github.com/Microsoft/hcsshim/test/internal/cmd"
"github.com/Microsoft/hcsshim/test/internal/constants"
"github.com/Microsoft/hcsshim/test/internal/container"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/oci"
Expand All @@ -27,9 +25,8 @@ func BenchmarkLCOW_Container(b *testing.B) {
requireFeatures(b, featureLCOW, featureContainer)
require.Build(b, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), b)
ls := layers.FromImage(ctx, b, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)
ctx := namespacedContext()
ls := linuxImageLayers(ctx, b)

// Create a new uvm per benchmark in case any left over state lingers

Expand Down
20 changes: 6 additions & 14 deletions test/functional/lcow_container_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package functional

import (
"context"
"strings"
"testing"

Expand All @@ -13,7 +12,6 @@ import (
"github.com/Microsoft/hcsshim/osversion"

"github.com/Microsoft/hcsshim/test/internal/cmd"
"github.com/Microsoft/hcsshim/test/internal/constants"
"github.com/Microsoft/hcsshim/test/internal/container"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/oci"
Expand All @@ -25,10 +23,8 @@ func TestLCOW_ContainerLifecycle(t *testing.T) {
requireFeatures(t, featureLCOW, featureContainer)
require.Build(t, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), t)
ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)

ctx := namespacedContext()
ls := linuxImageLayers(ctx, t)
opts := defaultLCOWOptions(t)
vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts)

Expand Down Expand Up @@ -81,10 +77,8 @@ func TestLCOW_ContainerIO(t *testing.T) {
requireFeatures(t, featureLCOW, featureContainer)
require.Build(t, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), t)
ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)

ctx := namespacedContext()
ls := linuxImageLayers(ctx, t)
opts := defaultLCOWOptions(t)
cache := layers.CacheFile(ctx, t, "")
vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts)
Expand Down Expand Up @@ -125,10 +119,8 @@ func TestLCOW_ContainerExec(t *testing.T) {
requireFeatures(t, featureLCOW, featureContainer)
require.Build(t, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), t)
ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)

ctx := namespacedContext()
ls := linuxImageLayers(ctx, t)
opts := defaultLCOWOptions(t)
vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts)

Expand Down
8 changes: 2 additions & 6 deletions test/functional/lcow_networking_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package functional

import (
"context"
"fmt"
"strings"
"testing"
Expand All @@ -15,7 +14,6 @@ import (
"github.com/Microsoft/hcsshim/osversion"

"github.com/Microsoft/hcsshim/test/internal/cmd"
"github.com/Microsoft/hcsshim/test/internal/constants"
"github.com/Microsoft/hcsshim/test/internal/container"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/oci"
Expand Down Expand Up @@ -123,10 +121,8 @@ func TestLCOW_IPv6_Assignment(t *testing.T) {
t.Fatalf("network attachment: %v", err)
}

ctx, _, client := newContainerdClient(context.Background(), t)
ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)

ctx := namespacedContext()
ls := linuxImageLayers(ctx, t)
opts := defaultLCOWOptions(t)
vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts)

Expand Down
8 changes: 3 additions & 5 deletions test/functional/lcow_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

testutilities "github.com/Microsoft/hcsshim/test/internal"
testcmd "github.com/Microsoft/hcsshim/test/internal/cmd"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/require"
testuvm "github.com/Microsoft/hcsshim/test/internal/uvm"
)
Expand Down Expand Up @@ -165,8 +164,7 @@ func TestLCOWSimplePodScenario(t *testing.T) {
require.Build(t, osversion.RS5)
requireFeatures(t, featureLCOW, featureContainer)

//nolint:staticcheck // SA1019: TODO: replace `LayerFolders`
alpineLayers := layers.LayerFolders(t, "alpine")
layers := linuxImageLayers(context.Background(), t)

cacheDir := t.TempDir()
cacheFile := filepath.Join(cacheDir, "cache.vhdx")
Expand Down Expand Up @@ -201,7 +199,7 @@ func TestLCOWSimplePodScenario(t *testing.T) {
t.Fatal(err)
}
c1Spec := testutilities.GetDefaultLinuxSpec(t)
c1Folders := append(alpineLayers, c1ScratchDir)
c1Folders := append(layers, c1ScratchDir)
c1Spec.Windows.LayerFolders = c1Folders
c1Spec.Process.Args = []string{"echo", "hello", "lcow", "container", "one"}
c1Opts := &hcsoci.CreateOptions{
Expand All @@ -214,7 +212,7 @@ func TestLCOWSimplePodScenario(t *testing.T) {
t.Fatal(err)
}
c2Spec := testutilities.GetDefaultLinuxSpec(t)
c2Folders := append(alpineLayers, c2ScratchDir)
c2Folders := append(layers, c2ScratchDir)
c2Spec.Windows.LayerFolders = c2Folders
c2Spec.Process.Args = []string{"echo", "hello", "lcow", "container", "two"}
c2Opts := &hcsoci.CreateOptions{
Expand Down
140 changes: 97 additions & 43 deletions test/functional/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,12 @@ import (
"log"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"testing"
"time"

"github.com/containerd/containerd"
"github.com/containerd/containerd/namespaces"
"github.com/sirupsen/logrus"

"github.com/Microsoft/hcsshim/internal/cow"
Expand All @@ -28,15 +27,34 @@ import (
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/winapi"

testctrd "github.com/Microsoft/hcsshim/test/internal/containerd"
"github.com/Microsoft/hcsshim/test/internal/constants"
testflag "github.com/Microsoft/hcsshim/test/internal/flag"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/require"
"github.com/Microsoft/hcsshim/test/internal/util"
testuvm "github.com/Microsoft/hcsshim/test/internal/uvm"
)

// owner field for uVMs.
const hcsOwner = "hcsshim-functional-tests"

var (
alpineImagePaths = &layers.LazyImageLayers{
Image: constants.ImageLinuxAlpineLatest,
Platform: constants.PlatformLinux,
}
//TODO: pick appropriate image based on OS build
nanoserverImagePaths = &layers.LazyImageLayers{
Image: constants.ImageWindowsNanoserverLTSC2022,
Platform: constants.PlatformWindows,
}
// wcow tests originally used busyboxw; cannot find image on docker or mcr
servercoreImagePaths = &layers.LazyImageLayers{
Image: constants.ImageWindowsServercoreLTSC2022,
Platform: constants.PlatformWindows,
}
)

const (
featureLCOW = "LCOW"
featureWCOW = "WCOW"
Expand All @@ -63,16 +81,25 @@ var allFeatures = []string{
featureVPMEM,
}

// todo: use a new containerd namespace and then nuke everything in it

var (
debug bool
pauseDurationOnCreateContainerFailure time.Duration

flagFeatures = testflag.NewFeatureFlag(allFeatures)
flagContainerdAddress = flag.String("ctr-address", "tcp://127.0.0.1:2376", "`address` for containerd's GRPC server")
flagContainerdNamespace = flag.String("ctr-namespace", "k8s.io", "containerd `namespace`")
flagLinuxBootFilesPath = flag.String("linux-bootfiles", "",
flagPauseAfterCreateContainerFailure time.Duration

flagFeatures = testflag.NewFeatureFlag(allFeatures)
flagDebug = flag.Bool("debug",
os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG") != "",
"set logging level to debug [%HCSSHIM_FUNCTIONAL_TESTS_DEBUG%]")
flagContainerdNamespace = flag.String("ctr-namespace", hcsOwner,
"containerd `namespace` to use when creating OCI specs")
flagLCOWLayerPaths = testflag.NewStringSlice("lcow-layer-paths",
"comma separated list of image layer `paths` to use as LCOW container rootfs. "+
"If empty, \""+alpineImagePaths.Image+"\" will be pulled and unpacked.")
//nolint:unused // will be used when WCOW tests are updated
flagWCOWLayerPaths = testflag.NewStringSlice("wcow-layer-paths",
"comma separated list of image layer `paths` to use as WCOW uVM and container rootfs. "+
"If empty, \""+nanoserverImagePaths.Image+"\" will be pulled and unpacked.")
flagLayerTempDir = flag.String("layer-temp-dir", "",
"`directory` to unpack image layers to, if not provided. Leave empty to use os.TempDir.")
flagLinuxBootFilesPath = flag.String("linux-bootfiles", "",
"override default `path` for LCOW uVM boot files (rootfs.vhd, initrd.img, kernel, and vmlinux)")
)

Expand All @@ -81,20 +108,15 @@ func init() {
log.Fatal("tests must be run in an elevated context")
}

if _, ok := os.LookupEnv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG"); ok {
debug = true
}
flag.BoolVar(&debug, "debug", debug, "set logging level to debug [%HCSSHIM_FUNCTIONAL_TESTS_DEBUG%]")

// This allows for debugging a utility VM.
if s := os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES"); s != "" {
if t, err := strconv.Atoi(s); err == nil {
pauseDurationOnCreateContainerFailure = time.Duration(t) * time.Minute
flagPauseAfterCreateContainerFailure = time.Duration(t) * time.Minute
}
}
flag.DurationVar(&pauseDurationOnCreateContainerFailure,
flag.DurationVar(&flagPauseAfterCreateContainerFailure,
"container-creation-failure-pause",
pauseDurationOnCreateContainerFailure,
flagPauseAfterCreateContainerFailure,
"the number of minutes to wait after a container creation failure to try again "+
"[%HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES%]")
}
Expand All @@ -103,18 +125,23 @@ func TestMain(m *testing.M) {
flag.Parse()

lvl := logrus.WarnLevel
if debug {
if *flagDebug {
lvl = logrus.DebugLevel
}
logrus.SetLevel(lvl)
logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true})
logrus.Infof("using features %q", flagFeatures.S.Strings())

images := []*layers.LazyImageLayers{alpineImagePaths, nanoserverImagePaths, servercoreImagePaths}
for _, l := range images {
l.TempPath = *flagLayerTempDir
}

e := m.Run()

// close any uVMs that escaped
cmdStr := `foreach ($vm in Get-ComputeProcess -Owner '` + hcsOwner + `') ` +
`{ Write-Output $vm.Id ; Stop-ComputeProcess -Force -Id $vm.Id }`
cmdStr := ` foreach ($vm in Get-ComputeProcess -Owner '` + hcsOwner +
`') { Write-Output "uVM $($vm.Id) was left running" ; Stop-ComputeProcess -Force -Id $vm.Id } `
cmd := exec.Command("powershell.exe", "-NoLogo", " -NonInteractive", "-Command", cmdStr)
o, err := cmd.CombinedOutput()
s := string(o)
Expand All @@ -124,17 +151,23 @@ func TestMain(m *testing.M) {
logrus.Warningf("cleaned up left over uVMs: %s", strings.Split(s, "\r\n"))
}

// delete downloaded layers; cant use defer, since os.exit does not run them
for _, l := range images {
// just ignore errors: they are logged, and no other cleanup possible
_ = l.Close(context.Background())
}

os.Exit(e)
}

func CreateContainerTestWrapper(ctx context.Context, options *hcsoci.CreateOptions) (cow.Container, *resources.Resources, error) {
if pauseDurationOnCreateContainerFailure != 0 {
if flagPauseAfterCreateContainerFailure != 0 {
options.DoNotReleaseResourcesOnFailure = true
}
s, r, err := hcsoci.CreateContainer(ctx, options)
if err != nil {
logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", pauseDurationOnCreateContainerFailure)
time.Sleep(pauseDurationOnCreateContainerFailure)
logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", flagPauseAfterCreateContainerFailure)
time.Sleep(flagPauseAfterCreateContainerFailure)
_ = resources.ReleaseResources(ctx, r, options.HostingSystem, true)
}

Expand All @@ -146,36 +179,57 @@ func requireFeatures(tb testing.TB, features ...string) {
require.Features(tb, flagFeatures.S, features...)
}

func getContainerdOptions() testctrd.ContainerdClientOptions {
return testctrd.ContainerdClientOptions{
Address: *flagContainerdAddress,
Namespace: *flagContainerdNamespace,
func defaultLCOWOptions(tb testing.TB) *uvm.OptionsLCOW {
tb.Helper()
opts := testuvm.DefaultLCOWOptions(tb, util.CleanName(tb.Name()), hcsOwner)
if p := *flagLinuxBootFilesPath; p != "" {
opts.BootFilesPath = p
}
return opts
}

func newContainerdClient(ctx context.Context, tb testing.TB) (context.Context, context.CancelFunc, *containerd.Client) {
//nolint:deadcode,unused // will be used when WCOW tests are updated
func defaultWCOWOptions(tb testing.TB) *uvm.OptionsWCOW {
tb.Helper()
return getContainerdOptions().NewClient(ctx, tb)
return uvm.NewDefaultOptionsWCOW(util.CleanName(tb.Name()), hcsOwner)
}

func defaultLCOWOptions(tb testing.TB) *uvm.OptionsLCOW {
// linuxImageLayers returns image layer paths appropriate for use as a container rootfs.
// If layer paths were provided on the command line, they are returned.
// Otherwise, it pulls an appropriate image.
func linuxImageLayers(ctx context.Context, tb testing.TB) []string {
tb.Helper()
opts := testuvm.DefaultLCOWOptions(tb, cleanName(tb.Name()), hcsOwner)
if p := *flagLinuxBootFilesPath; p != "" {
opts.BootFilesPath = p
if ss := flagLCOWLayerPaths.S.Strings(); len(ss) > 0 {
return ss
}
return opts
return alpineImagePaths.Layers(ctx, tb)
}

// windowsImageLayers returns image layer paths appropriate for use as a uVM or container rootfs.
// If layer paths were provided on the command line, they are returned.
// Otherwise, it pulls an appropriate image.
//
//nolint:deadcode,unused // will be used when WCOW tests are updated
func defaultWCOWOptions(tb testing.TB) *uvm.OptionsWCOW {
func windowsImageLayers(ctx context.Context, tb testing.TB) []string {
tb.Helper()
opts := uvm.NewDefaultOptionsWCOW(cleanName(tb.Name()), hcsOwner)
return opts
if ss := flagWCOWLayerPaths.S.Strings(); len(ss) > 0 {
return ss
}
return nanoserverImagePaths.Layers(ctx, tb)
}

var _nameRegex = regexp.MustCompile(`[\\\/\s]`)
// windowsServercoreImageLayers returns image layer paths for Windows servercore.
//
// See [windowsImageLayers] for more.
//
//nolint:unused // will be used when WCOW tests are updated
func windowsServercoreImageLayers(ctx context.Context, tb testing.TB) []string {
tb.Helper()
return servercoreImagePaths.Layers(ctx, tb)
}

func cleanName(n string) string {
return _nameRegex.ReplaceAllString(n, "")
// namespacedContext returns a [context.Context] with the provided namespace added via
// [github.com/containerd/containerd/namespaces.WithNamespace].
func namespacedContext() context.Context {
return namespaces.WithNamespace(context.Background(), *flagContainerdNamespace)
}
Loading

0 comments on commit 1d141fa

Please sign in to comment.