Skip to content

Commit

Permalink
Merge pull request #1744 from kevpar/new-scsi
Browse files Browse the repository at this point in the history 
Rewrite SCSI support in new package
  • Loading branch information
@kevpar
kevpar committed May 15, 2023
2 parents 497a346 + d98a2ef commit 5c5d85c
Show file tree
Hide file tree
Showing 26 changed files with 1,188 additions and 751 deletions.
14 changes: 7 additions & 7 deletions internal/devices/drivers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/Microsoft/hcsshim/internal/log"
"github.com/Microsoft/hcsshim/internal/resources"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/uvm/scsi"
)

// InstallDriver mounts a share from the host into the UVM, installs any kernel drivers in the share,
Expand Down Expand Up @@ -54,19 +55,18 @@ func InstallDrivers(ctx context.Context, vm *uvm.UtilityVM, share string, gpuDri
}

// first mount driver as scsi in standard mount location
uvmPathForShare := fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter())
mount, err := vm.AddSCSI(ctx,
mount, err := vm.SCSIManager.AddVirtualDisk(
ctx,
share,
uvmPathForShare,
true,
false,
[]string{},
uvm.VMAccessTypeIndividual)
vm.ID(),
&scsi.MountConfig{},
)
if err != nil {
return closer, fmt.Errorf("failed to add SCSI disk to utility VM for path %+v: %s", share, err)
}
closer = mount
uvmPathForShare = mount.UVMPath
uvmPathForShare := mount.GuestPath()

// construct path that the drivers will be remounted as read/write in the UVM

Expand Down
24 changes: 13 additions & 11 deletions internal/hcsoci/resources_lcow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
"github.com/Microsoft/hcsshim/internal/layers"
"github.com/Microsoft/hcsshim/internal/log"
"github.com/Microsoft/hcsshim/internal/resources"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/uvm/scsi"
)

func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error {
Expand Down Expand Up @@ -85,35 +85,37 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
l := log.G(ctx).WithField("mount", fmt.Sprintf("%+v", mount))
if mount.Type == "physical-disk" {
l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI physical disk for OCI mount")
uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter())
scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly, mount.Options)
scsiMount, err := coi.HostingSystem.SCSIManager.AddPhysicalDisk(
ctx,
hostPath,
readOnly,
coi.HostingSystem.ID(),
&scsi.MountConfig{Options: mount.Options},
)
if err != nil {
return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount)
}

uvmPathForFile = scsiMount.UVMPath
uvmPathForFile = scsiMount.GuestPath()
r.Add(scsiMount)
coi.Spec.Mounts[i].Type = "none"
} else if mount.Type == "virtual-disk" {
l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI virtual disk for OCI mount")
uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter())

// if the scsi device is already attached then we take the uvm path that the function below returns
// that is where it was previously mounted in UVM
scsiMount, err := coi.HostingSystem.AddSCSI(
scsiMount, err := coi.HostingSystem.SCSIManager.AddVirtualDisk(
ctx,
hostPath,
uvmPathForShare,
readOnly,
false,
mount.Options,
uvm.VMAccessTypeIndividual,
coi.HostingSystem.ID(),
&scsi.MountConfig{Options: mount.Options},
)
if err != nil {
return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount)
}

uvmPathForFile = scsiMount.UVMPath
uvmPathForFile = scsiMount.GuestPath()
r.Add(scsiMount)
coi.Spec.Mounts[i].Type = "none"
} else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) {
Expand Down
27 changes: 12 additions & 15 deletions internal/hcsoci/resources_wcow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/Microsoft/hcsshim/internal/resources"
"github.com/Microsoft/hcsshim/internal/schemaversion"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/uvm/scsi"
"github.com/Microsoft/hcsshim/internal/wclayer"
)

Expand Down Expand Up @@ -59,8 +60,7 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r

if coi.Spec.Root.Path == "" && (coi.HostingSystem != nil || coi.Spec.Windows.HyperV == nil) {
log.G(ctx).Debug("hcsshim::allocateWindowsResources mounting storage")
containerRootInUVM := r.ContainerRootInUVM()
containerRootPath, closer, err := layers.MountWCOWLayers(ctx, coi.actualID, coi.Spec.Windows.LayerFolders, containerRootInUVM, "", coi.HostingSystem)
containerRootPath, closer, err := layers.MountWCOWLayers(ctx, coi.actualID, coi.Spec.Windows.LayerFolders, "", coi.HostingSystem)
if err != nil {
return errors.Wrap(err, "failed to mount container storage")
}
Expand Down Expand Up @@ -146,7 +146,6 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R
}

if coi.HostingSystem != nil && schemaversion.IsV21(coi.actualSchemaVersion) {
uvmPath := fmt.Sprintf(guestpath.WCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter())
readOnly := false
for _, o := range mount.Options {
if strings.ToLower(o) == "ro" {
Expand All @@ -157,37 +156,35 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R
l := log.G(ctx).WithField("mount", fmt.Sprintf("%+v", mount))
if mount.Type == "physical-disk" || mount.Type == "virtual-disk" || mount.Type == "extensible-virtual-disk" {
var (
scsiMount *uvm.SCSIMount
scsiMount *scsi.Mount
err error
)
switch mount.Type {
case "physical-disk":
l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI physical disk for OCI mount")
scsiMount, err = coi.HostingSystem.AddSCSIPhysicalDisk(
scsiMount, err = coi.HostingSystem.SCSIManager.AddPhysicalDisk(
ctx,
mount.Source,
uvmPath,
readOnly,
mount.Options,
coi.HostingSystem.ID(),
&scsi.MountConfig{},
)
case "virtual-disk":
l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI virtual disk for OCI mount")
scsiMount, err = coi.HostingSystem.AddSCSI(
scsiMount, err = coi.HostingSystem.SCSIManager.AddVirtualDisk(
ctx,
mount.Source,
uvmPath,
readOnly,
false,
mount.Options,
uvm.VMAccessTypeIndividual,
coi.HostingSystem.ID(),
&scsi.MountConfig{},
)
case "extensible-virtual-disk":
l.Debug("hcsshim::allocateWindowsResource Hot-adding ExtensibleVirtualDisk")
scsiMount, err = coi.HostingSystem.AddSCSIExtensibleVirtualDisk(
scsiMount, err = coi.HostingSystem.SCSIManager.AddExtensibleVirtualDisk(
ctx,
mount.Source,
uvmPath,
readOnly,
&scsi.MountConfig{},
)
}
if err != nil {
Expand All @@ -197,7 +194,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R
// Compute guest mounts now, and store them, so they can be added to the container doc later.
// We do this now because we have access to the guest path through the returned mount object.
coi.windowsAdditionalMounts = append(coi.windowsAdditionalMounts, hcsschema.MappedDirectory{
HostPath: scsiMount.UVMPath,
HostPath: scsiMount.GuestPath(),
ContainerPath: mount.Destination,
ReadOnly: readOnly,
})
Expand Down
2 changes: 1 addition & 1 deletion internal/jobcontainers/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func (c *JobContainer) mountLayers(ctx context.Context, containerID string, s *s
if s.Root.Path == "" {
log.G(ctx).Debug("mounting job container storage")
var rootPath string
rootPath, closer, err = layers.MountWCOWLayers(ctx, containerID, s.Windows.LayerFolders, "", volumeMountPath, nil)
rootPath, closer, err = layers.MountWCOWLayers(ctx, containerID, s.Windows.LayerFolders, volumeMountPath, nil)
if err != nil {
return nil, fmt.Errorf("failed to mount job container storage: %w", err)
}
Expand Down
39 changes: 12 additions & 27 deletions internal/layers/layers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/Microsoft/hcsshim/internal/ospath"
"github.com/Microsoft/hcsshim/internal/resources"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/uvm/scsi"
"github.com/Microsoft/hcsshim/internal/wclayer"
)

Expand Down Expand Up @@ -104,22 +105,18 @@ func MountLCOWLayers(ctx context.Context, containerID string, layerFolders []str
lcowUvmLayerPaths = append(lcowUvmLayerPaths, uvmPath)
}

containerScratchPathInUVM := ospath.Join(vm.OS(), guestRoot)
hostPath, err := getScratchVHDPath(layerFolders)
if err != nil {
return "", "", nil, fmt.Errorf("failed to eval symlinks on scratch path: %w", err)
}
log.G(ctx).WithField("hostPath", hostPath).Debug("mounting scratch VHD")

var options []string
scsiMount, err := vm.AddSCSI(
scsiMount, err := vm.SCSIManager.AddVirtualDisk(
ctx,
hostPath,
containerScratchPathInUVM,
false,
vm.ScratchEncryptionEnabled(),
options,
uvm.VMAccessTypeIndividual,
vm.ID(),
&scsi.MountConfig{Encrypted: vm.ScratchEncryptionEnabled()},
)
if err != nil {
return "", "", nil, fmt.Errorf("failed to add SCSI scratch VHD: %s", err)
Expand All @@ -128,7 +125,7 @@ func MountLCOWLayers(ctx context.Context, containerID string, layerFolders []str
// handles the case where we want to share a scratch disk for multiple containers instead
// of mounting a new one. Pass a unique value for `ScratchPath` to avoid container upper and
// work directories colliding in the UVM.
containerScratchPathInUVM = ospath.Join("linux", scsiMount.UVMPath, "scratch", containerID)
containerScratchPathInUVM := ospath.Join("linux", scsiMount.GuestPath(), "scratch", containerID)

defer func() {
if err != nil {
Expand Down Expand Up @@ -164,7 +161,7 @@ func MountLCOWLayers(ctx context.Context, containerID string, layerFolders []str
//
// Job container: Returns the mount path on the host as a volume guid, with the volume mounted on
// the host at `volumeMountPath`.
func MountWCOWLayers(ctx context.Context, containerID string, layerFolders []string, guestRoot, volumeMountPath string, vm *uvm.UtilityVM) (_ string, _ resources.ResourceCloser, err error) {
func MountWCOWLayers(ctx context.Context, containerID string, layerFolders []string, volumeMountPath string, vm *uvm.UtilityVM) (_ string, _ resources.ResourceCloser, err error) {
if vm == nil {
return mountWCOWHostLayers(ctx, layerFolders, volumeMountPath)
}
Expand All @@ -173,7 +170,7 @@ func MountWCOWLayers(ctx context.Context, containerID string, layerFolders []str
return "", nil, errors.New("MountWCOWLayers should only be called for WCOW")
}

return mountWCOWIsolatedLayers(ctx, containerID, layerFolders, guestRoot, volumeMountPath, vm)
return mountWCOWIsolatedLayers(ctx, containerID, layerFolders, volumeMountPath, vm)
}

type wcowHostLayersCloser struct {
Expand Down Expand Up @@ -310,7 +307,7 @@ func (lc *wcowIsolatedLayersCloser) Release(ctx context.Context) (retErr error)
return
}

func mountWCOWIsolatedLayers(ctx context.Context, containerID string, layerFolders []string, guestRoot, volumeMountPath string, vm *uvm.UtilityVM) (_ string, _ resources.ResourceCloser, err error) {
func mountWCOWIsolatedLayers(ctx context.Context, containerID string, layerFolders []string, volumeMountPath string, vm *uvm.UtilityVM) (_ string, _ resources.ResourceCloser, err error) {
log.G(ctx).WithField("os", vm.OS()).Debug("hcsshim::MountWCOWLayers V2 UVM")

var (
Expand Down Expand Up @@ -339,27 +336,17 @@ func mountWCOWIsolatedLayers(ctx context.Context, containerID string, layerFolde
layerClosers = append(layerClosers, mount)
}

containerScratchPathInUVM := ospath.Join(vm.OS(), guestRoot)
hostPath, err := getScratchVHDPath(layerFolders)
if err != nil {
return "", nil, fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err)
}
log.G(ctx).WithField("hostPath", hostPath).Debug("mounting scratch VHD")

var options []string
scsiMount, err := vm.AddSCSI(
ctx,
hostPath,
containerScratchPathInUVM,
false,
vm.ScratchEncryptionEnabled(),
options,
uvm.VMAccessTypeIndividual,
)
scsiMount, err := vm.SCSIManager.AddVirtualDisk(ctx, hostPath, false, vm.ID(), &scsi.MountConfig{})
if err != nil {
return "", nil, fmt.Errorf("failed to add SCSI scratch VHD: %s", err)
}
containerScratchPathInUVM = scsiMount.UVMPath
containerScratchPathInUVM := scsiMount.GuestPath()

defer func() {
if err != nil {
Expand Down Expand Up @@ -407,17 +394,15 @@ func addLCOWLayer(ctx context.Context, vm *uvm.UtilityVM, layerPath string) (uvm
}
}

options := []string{"ro"}
uvmPath = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter())
sm, err := vm.AddSCSI(ctx, layerPath, uvmPath, true, false, options, uvm.VMAccessTypeNoop)
sm, err := vm.SCSIManager.AddVirtualDisk(ctx, layerPath, true, "", &scsi.MountConfig{Options: []string{"ro"}})
if err != nil {
return "", nil, fmt.Errorf("failed to add SCSI layer: %s", err)
}
log.G(ctx).WithFields(logrus.Fields{
"layerPath": layerPath,
"layerType": "scsi",
}).Debug("Added LCOW layer")
return sm.UVMPath, sm, nil
return sm.GuestPath(), sm, nil
}

// GetHCSLayers converts host paths corresponding to container layers into HCS schema V2 layers
Expand Down
6 changes: 3 additions & 3 deletions internal/lcow/disk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ func FormatDisk(ctx context.Context, lcowUVM *uvm.UtilityVM, destPath string) er
"dest": destPath,
}).Debug("lcow::FormatDisk opts")

var options []string
scsi, err := lcowUVM.AddSCSIPhysicalDisk(ctx, destPath, "", false, options) // No destination as not formatted
// Attach without mounting.
scsi, err := lcowUVM.SCSIManager.AddPhysicalDisk(ctx, destPath, false, lcowUVM.ID(), nil)
if err != nil {
return err
}
Expand All @@ -46,7 +46,7 @@ func FormatDisk(ctx context.Context, lcowUVM *uvm.UtilityVM, destPath string) er
"lun": scsi.LUN,
}).Debug("lcow::FormatDisk device attached")

if err := formatDiskUvm(ctx, lcowUVM, scsi.Controller, scsi.LUN, destPath); err != nil {
if err := formatDiskUvm(ctx, lcowUVM, int(scsi.Controller()), int32(scsi.LUN()), destPath); err != nil {
return err
}
log.G(ctx).WithField("dest", destPath).Debug("lcow::FormatDisk complete")
Expand Down
19 changes: 8 additions & 11 deletions internal/lcow/scratch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,34 +68,31 @@ func CreateScratch(ctx context.Context, lcowUVM *uvm.UtilityVM, destFile string,
return fmt.Errorf("failed to create VHDx %s: %s", destFile, err)
}

var options []string
scsi, err := lcowUVM.AddSCSI(
scsi, err := lcowUVM.SCSIManager.AddVirtualDisk(
ctx,
destFile,
"", // No destination as not formatted
false,
lcowUVM.ScratchEncryptionEnabled(),
options,
uvm.VMAccessTypeIndividual,
lcowUVM.ID(),
nil, // Attach without mounting.
)
if err != nil {
return err
}
removeSCSI := true
defer func() {
if removeSCSI {
_ = lcowUVM.RemoveSCSI(ctx, destFile)
_ = scsi.Release(ctx)
}
}()

log.G(ctx).WithFields(logrus.Fields{
"dest": destFile,
"controller": scsi.Controller,
"lun": scsi.LUN,
"controller": scsi.Controller(),
"lun": scsi.LUN(),
}).Debug("lcow::CreateScratch device attached")

// Validate /sys/bus/scsi/devices/C:0:0:L exists as a directory
devicePath := fmt.Sprintf("/sys/bus/scsi/devices/%d:0:0:%d/block", scsi.Controller, scsi.LUN)
devicePath := fmt.Sprintf("/sys/bus/scsi/devices/%d:0:0:%d/block", scsi.Controller(), scsi.LUN())
testdCtx, cancel := context.WithTimeout(ctx, timeout.TestDRetryLoop)
defer cancel()
for {
Expand Down Expand Up @@ -138,7 +135,7 @@ func CreateScratch(ctx context.Context, lcowUVM *uvm.UtilityVM, destFile string,

// Hot-Remove before we copy it
removeSCSI = false
if err := lcowUVM.RemoveSCSI(ctx, destFile); err != nil {
if err := scsi.Release(ctx); err != nil {
return fmt.Errorf("failed to hot-remove: %s", err)
}

Expand Down
Loading

0 comments on commit 5c5d85c

Please sign in to comment.