Merged PR 8117098: Update hcsshim to 7769a64

This PR updates our ADO fork to commits in hcsshim up to commit hash [7769a64](7769a64). This includes support for partitioned scsi devices and ensuring filesystem format for lcow scsi devices.

Related work items: #1728, #1740, #1741, #1742, #1743, #1744, #1745, #1747, #1748, #1749, #1750, #1752, #1754, #1756, #1757, #1767, #1769, #1771, #1772, #1773, #1779

.github/dependabot.yml

@@ -1,3 +1,7 @@
+# there may be other dependencies manually ignored.
+# search `"@dependabot ignore" in:comments` in the repo to see them.
+#
+# reference:
 # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
 
 version: 2
@@ -7,18 +11,26 @@ updates:
     schedule:
       interval: "weekly"
       day: "sunday"
-    # ignore patch version increment updates (will not affect security updates)
     ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch"]
+      # only do patch updates for containerd and co: don't bump anything else
+      - dependency-name: "github.com/containerd/*"
+        update-types: ["version-update:semver-major", "version-update:semver-minor"]
 
   - package-ecosystem: "gomod"
     directory: "/test"
     schedule:
       interval: "weekly"
       day: "sunday"
-    # ignore patch version increment updates (will not affect security updates)
+    labels:
+      - dependencies
+      - go
+      - tests
     ignore:
+      # updating CRI would break tests
+      - dependency-name: "k8s.io/cri-api"
+      # keep containerd and related dependencies in sync with the shim/the root repo
+      - dependency-name: "github.com/containerd/*"
+      # ignore patch version increment updates in test, since shouldn't be critical
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 

.github/workflows/ci.yml

@@ -53,22 +53,41 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: ${{ env.GO_VERSION }}
+
       - uses: actions/checkout@v3
         with:
           path: "go/src/github.com/Microsoft/hcsshim"
+        name: Checkout hcsshim
+
+      - name: Get containerd ref
+        shell: powershell
+        run: |
+          $v = go list -m -f '{{ .Version }}' 'github.com/containerd/containerd' 2>&1
+          if ( $LASTEXITCODE ) {
+            Write-Output '::error::Could not retrieve containerd version.'
+            exit $LASTEXITCODE
+          }
+
+          Write-Output "containerd ref is: $v"
+          "containerd_ref=$v" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        working-directory: go/src/github.com/Microsoft/hcsshim
+
       # Install protoc-gen-gogoctrd in D:\bin
       - uses: actions/checkout@v3
         with:
           repository: containerd/containerd
-          ref: v1.6.2
           path: "containerd"
+          ref: "${{ env.containerd_ref }}"
+        name: Checkout containerd
+
       - name: Install protoc-gen-gogoctrd
         shell: powershell
         run: |
           cd containerd
           go build ./cmd/protoc-gen-gogoctrd
           mkdir D:\bin
           mv protoc-gen-gogoctrd.exe D:\bin
+
       # Install protoc in D:\bin
       - name: Install protoc
         shell: powershell
@@ -77,6 +96,7 @@ jobs:
           Expand-Archive -Path protoc.zip -DestinationPath .
           mv include go/src/github.com/Microsoft/hcsshim/protobuf
           mv bin\protoc.exe D:\bin
+
       - name: Run Protobuild
         shell: powershell
         run: |
@@ -180,6 +200,9 @@ jobs:
       - name: Test rego policy interpreter
         run: gotestsum --format standard-verbose --debug -- -mod=mod -gcflags=all=-d=checkptr ./internal/regopolicyinterpreter
 
+      - name: Run guest code unit tests
+        run: gotestsum --format standard-verbose --debug -- -mod=mod -gcflags=all=-d=checkptr ./internal/guest/...
+
       - name: Build gcs Testing Binary
         run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./gcs
         working-directory: test
@@ -260,12 +283,28 @@ jobs:
 
       - uses: actions/checkout@v3
         with:
-          path: src/github.com/containerd/containerd
-          repository: "containerd/containerd"
+          path: src/github.com/Microsoft/hcsshim
+        name: Checkout hcsshim
+
+      - name: Get containerd ref
+        shell: powershell
+        run: |
+          $v = go list -m -f '{{ .Version }}' 'github.com/containerd/containerd' 2>&1
+          if ( $LASTEXITCODE ) {
+            Write-Output '::error::Could not retrieve containerd version.'
+            exit $LASTEXITCODE
+          }
+
+          Write-Output "containerd ref is: $v"
+          "containerd_ref=$v" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        working-directory: src/github.com/Microsoft/hcsshim
 
       - uses: actions/checkout@v3
         with:
-          path: src/github.com/Microsoft/hcsshim
+          path: src/github.com/containerd/containerd
+          repository: "containerd/containerd"
+          ref: "${{ env.containerd_ref }}"
+        name: Checkout containerd
 
       - name: Install crictl
         shell: powershell

Makefile

@@ -94,23 +94,9 @@ out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/ho
 	tar -zcf $@ -C rootfs .
 	rm -rf rootfs
 
--include deps/cmd/gcs.gomake
--include deps/cmd/gcstools.gomake
--include deps/cmd/hooks/wait-paths.gomake
--include deps/cmd/tar2ext4.gomake
--include deps/internal/tools/snp-report.gomake
-
-# Implicit rule for includes that define Go targets.
-%.gomake: $(SRCROOT)/Makefile
+bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths bin/cmd/tar2ext4 bin/internal/tools/snp-report:
 	@mkdir -p $(dir $@)
-	@/bin/echo $(@:deps/%.gomake=bin/%): $(SRCROOT)/hack/gomakedeps.sh > $@.new
-	@/bin/echo -e '\t@mkdir -p $$(dir $$@) $(dir $@)' >> $@.new
-	@/bin/echo -e '\t$$(GO_BUILD) -o $$@.new $$(SRCROOT)/$$(@:bin/%=%)' >> $@.new
-	@/bin/echo -e '\tGO="$(GO)" $$(SRCROOT)/hack/gomakedeps.sh $$@ $$(SRCROOT)/$$(@:bin/%=%) $$(GO_FLAGS) $$(GO_FLAGS_EXTRA) > $(@:%.gomake=%.godeps).new' >> $@.new
-	@/bin/echo -e '\tmv $(@:%.gomake=%.godeps).new $(@:%.gomake=%.godeps)' >> $@.new
-	@/bin/echo -e '\tmv $$@.new $$@' >> $@.new
-	@/bin/echo -e '-include $(@:%.gomake=%.godeps)' >> $@.new
-	mv $@.new $@
+	GOOS=linux $(GO_BUILD) -o $@ $(SRCROOT)/$(@:bin/%=%)
 
 bin/vsockexec: vsockexec/vsockexec.o vsockexec/vsock.o
 	@mkdir -p bin

cmd/containerd-shim-runhcs-v1/delete.go

@@ -3,7 +3,7 @@
 package main
 
 import (
-	gcontext "context"
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -50,16 +50,16 @@ This command allows containerd to delete any container resources created, mounte
 The delete command will be executed in the container's bundle as its cwd.
 `,
 	SkipArgReorder: true,
-	Action: func(context *cli.Context) (err error) {
+	Action: func(cCtx *cli.Context) (err error) {
 		// We cant write anything to stdout for this cmd other than the
 		// task.DeleteResponse by protocol. We can write to stderr which will be
 		// logged as a warning in containerd.
 
-		ctx, span := oc.StartSpan(gcontext.Background(), "delete")
+		ctx, span := oc.StartSpan(context.Background(), "delete")
 		defer span.End()
 		defer func() { oc.SetSpanStatus(span, err) }()
 
-		bundleFlag := context.GlobalString("bundle")
+		bundleFlag := cCtx.GlobalString("bundle")
 		if bundleFlag == "" {
 			return errors.New("bundle is required")
 		}
@@ -107,7 +107,10 @@ The delete command will be executed in the container's bundle as its cwd.
 		// be deleted, but if the shim crashed unexpectedly (panic, terminated etc.) then the account may still be around.
 		// The username will be the container ID so try and delete it here. The username character limit is 20, so we need to
 		// slice down the container ID a bit.
-		username := idFlag[:winapi.UserNameCharLimit]
+		username := idFlag
+		if len(username) > winapi.UserNameCharLimit {
+			username = username[:winapi.UserNameCharLimit]
+		}
 
 		// Always try and delete the user, if it doesn't exist we'll get a specific error code that we can use to
 		// not log any warnings.

cmd/containerd-shim-runhcs-v1/exec_hcs.go

@@ -445,8 +445,6 @@ func (he *hcsExec) exitFromCreatedL(ctx context.Context, status int) {
 //
 // 6. Close `he.exited` channel to unblock any waiters who might have called
 // `Create`/`Wait`/`Start` which is a valid pattern.
-//
-// 7. Finally, save the UVM and this container as a template if specified.
 func (he *hcsExec) waitForExit() {
 	var err error // this will only save the last error, since we dont return early on error
 	ctx, span := oc.StartSpan(context.Background(), "hcsExec::waitForExit")

cmd/containerd-shim-runhcs-v1/pod.go

@@ -301,16 +301,6 @@ func (p *pod) ID() string {
 	return p.id
 }
 
-func (p *pod) GetCloneAnnotations(ctx context.Context, s *specs.Spec) (bool, string, error) {
-	isTemplate, templateID, err := oci.ParseCloneAnnotations(ctx, s)
-	if err != nil {
-		return false, "", err
-	} else if (isTemplate || templateID != "") && p.host == nil {
-		return false, "", fmt.Errorf("save as template and creating clones is only supported for hyper-v isolated containers")
-	}
-	return isTemplate, templateID, nil
-}
-
 func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *specs.Spec) (_ shimTask, err error) {
 	if req.ID == p.id {
 		return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "task with id: '%s' already exists", req.ID)
@@ -366,17 +356,7 @@ func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *sp
 			sid)
 	}
 
-	_, templateID, err := p.GetCloneAnnotations(ctx, s)
-	if err != nil {
-		return nil, err
-	}
-
-	var st shimTask
-	if templateID != "" {
-		st, err = newClonedHcsTask(ctx, p.events, p.host, false, req, s, templateID)
-	} else {
-		st, err = newHcsTask(ctx, p.events, p.host, false, req, s)
-	}
+	st, err := newHcsTask(ctx, p.events, p.host, false, req, s)
 	if err != nil {
 		return nil, err
 	}