Skip to content

Commit

Permalink
fix: Allow disks to be resized/deleted on instances that have been ru…
Browse files Browse the repository at this point in the history 
…nning for > 90 days (#1581)

* Add WaitForInstanceNonTransientStatus helper; consume in InstanceDisk

* Apply to deletion logic

* Consolidate disk boot logic
  • Loading branch information
@lgarber-akamai
lgarber-akamai committed Sep 24, 2024
1 parent 36cec5c commit 86e3962
Show file tree
Hide file tree
Showing 4 changed files with 285 additions and 324 deletions.
118 changes: 118 additions & 0 deletions linode/helper/instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,3 +305,121 @@ func FlattenInterfaces(interfaces []linodego.InstanceConfigInterface) []map[stri
}
return result
}

// BootInstanceSync boots the instance with the given ID and waits for the operation to
// complete before returning.
func BootInstanceSync(
ctx context.Context,
client *linodego.Client,
instanceID,
configID,
deadlineSeconds int,
) error {
ctx = SetLogFieldBulk(
ctx,
map[string]any{
"instance_id": instanceID,
"config_id": configID,
},
)

tflog.Info(ctx, "Booting instance")

p, err := client.NewEventPoller(ctx, instanceID, linodego.EntityLinode, linodego.ActionLinodeBoot)
if err != nil {
return fmt.Errorf("failed to initialize event poller: %s", err)
}

tflog.Debug(ctx, "client.BootInstance(...)")

if err := client.BootInstance(ctx, instanceID, configID); err != nil {
return fmt.Errorf("failed to boot instance: %s", err)
}

tflog.Debug(ctx, "Waiting for instance boot to finish")

if _, err := p.WaitForFinished(ctx, deadlineSeconds); err != nil {
return fmt.Errorf("failed to wait for instance boot: %s", err)
}

tflog.Debug(ctx, "Instance has finished booting")

return nil
}

// ShutDownInstanceSync shuts down the instance with the given ID and waits for the operation to
// complete before returning.
func ShutDownInstanceSync(
ctx context.Context,
client *linodego.Client,
instanceID,
deadlineSeconds int,
) error {
ctx = tflog.SetField(ctx, "instance_id", instanceID)

tflog.Info(ctx, "Shutting down instance")

p, err := client.NewEventPoller(ctx, instanceID, linodego.EntityLinode, linodego.ActionLinodeShutdown)
if err != nil {
return fmt.Errorf("failed to initialize event poller: %s", err)
}

tflog.Debug(ctx, "client.ShutdownInstance(...)")

if err := client.ShutdownInstance(ctx, instanceID); err != nil {
return fmt.Errorf("failed to shutdown instance: %s", err)
}

tflog.Debug(ctx, "Waiting for instance shutdown to finish")

if _, err := p.WaitForFinished(ctx, deadlineSeconds); err != nil {
return fmt.Errorf("failed to wait for instance shutdown: %s", err)
}

tflog.Debug(ctx, "Instance has finished shutting down")

return nil
}

// WaitForInstanceNonTransientStatus waits for the instance with the given ID to enter
// a non-transient status (e.g. running, offline), and returns the final status of the instance.
func WaitForInstanceNonTransientStatus(
ctx context.Context,
client *linodego.Client,
linodeID int,
timeoutSeconds int,
) (linodego.InstanceStatus, error) {
instance, err := client.GetInstance(ctx, linodeID)
if err != nil {
return "", fmt.Errorf("failed to get instance: %w", err)
}

var targetStatus linodego.InstanceStatus

switch instance.Status {
case linodego.InstanceBooting, linodego.InstanceRebooting:
targetStatus = linodego.InstanceRunning

case linodego.InstanceShuttingDown:
targetStatus = linodego.InstanceOffline

case linodego.InstanceRunning, linodego.InstanceOffline:
// Instance is offline, nothing to do here
return instance.Status, nil

default:
return "", fmt.Errorf("cannot wait for instance to exit transient status %s", instance.Status)
}

instance, err = client.WaitForInstanceStatus(
ctx,
instance.ID,
targetStatus,
timeoutSeconds,
)
if err != nil {
return "", fmt.Errorf("failed to wait for instance to reach status %s: %w", targetStatus, err)
}

return instance.Status, nil
}
128 changes: 12 additions & 116 deletions linode/instance/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1180,127 +1180,27 @@ func handleBootedUpdate(
return nil
}

inst, err := client.GetInstance(ctx, instanceID)
if err != nil {
return err
}

instStatus, err := waitForRunningOrOfflineState(ctx, inst.Status, &client, instanceID)
instStatus, err := helper.WaitForInstanceNonTransientStatus(ctx, &client, instanceID, 120)
if err != nil {
return err
}

// Boot or shutdown the instance if necessary
if instStatus != linodego.InstanceRunning && booted.(bool) {
if err := BootInstanceSync(ctx, &client, instanceID, configID, deadlineSeconds); err != nil {
if err := helper.BootInstanceSync(ctx, &client, instanceID, configID, deadlineSeconds); err != nil {
return err
}
}

if instStatus != linodego.InstanceOffline && !booted.(bool) {
if err := shutDownInstanceSync(ctx, client, instanceID, deadlineSeconds); err != nil {
if err := helper.ShutDownInstanceSync(ctx, &client, instanceID, deadlineSeconds); err != nil {
return err
}
}

return nil
}

func waitForRunningOrOfflineState(
ctx context.Context, status linodego.InstanceStatus, client *linodego.Client, instanceID int,
) (instStatus linodego.InstanceStatus, err error) {
// Ensure the Linode reaches a running or offline state
// if in a transition state (shutting down, booting, rebooting)
// TODO: clean up this logic
switch status {

// These cases can be ignored
case linodego.InstanceRunning:
case linodego.InstanceOffline:

case linodego.InstanceShuttingDown:
tflog.Info(ctx, "Awaiting instance shutdown before continuing")

tflog.Trace(ctx, "client.WaitForInstanceStatus(...)", map[string]any{
"status": linodego.InstanceOffline,
})
_, err = client.WaitForInstanceStatus(ctx, instanceID, linodego.InstanceOffline, 120)
if err != nil {
return "", fmt.Errorf("failed to wait for instance offline: %s", err)
}

instStatus = linodego.InstanceOffline

case linodego.InstanceBooting, linodego.InstanceRebooting:
tflog.Info(ctx, "Awaiting instance boot before continuing")

tflog.Trace(ctx, "client.WaitForInstanceStatus(...)", map[string]any{
"status": linodego.InstanceRunning,
})
_, err = client.WaitForInstanceStatus(ctx, instanceID, linodego.InstanceRunning, 120)
if err != nil {
return "", fmt.Errorf("failed to wait for instance running: %s", err)
}

instStatus = linodego.InstanceRunning

default:
return "", fmt.Errorf("instance is in unhandled state %s", status)
}
return instStatus, nil
}

func shutDownInstanceSync(ctx context.Context, client linodego.Client, instanceID, deadlineSeconds int) error {
tflog.Info(ctx, "Shutting down instance")

p, err := client.NewEventPoller(ctx, instanceID, linodego.EntityLinode, linodego.ActionLinodeShutdown)
if err != nil {
return fmt.Errorf("failed to initialize event poller: %s", err)
}

tflog.Debug(ctx, "client.ShutdownInstance(...)")

if err := client.ShutdownInstance(ctx, instanceID); err != nil {
return fmt.Errorf("failed to shutdown instance: %s", err)
}

tflog.Debug(ctx, "Waiting for instance shutdown to finish")

if _, err := p.WaitForFinished(ctx, deadlineSeconds); err != nil {
return fmt.Errorf("failed to wait for instance shutdown: %s", err)
}

tflog.Debug(ctx, "Instance has finished shutting down")

return nil
}

func BootInstanceSync(ctx context.Context, client *linodego.Client, instanceID, configID, deadlineSeconds int) error {
ctx = tflog.SetField(ctx, "config_id", configID)

tflog.Info(ctx, "Booting instance")

p, err := client.NewEventPoller(ctx, instanceID, linodego.EntityLinode, linodego.ActionLinodeBoot)
if err != nil {
return fmt.Errorf("failed to initialize event poller: %s", err)
}

tflog.Debug(ctx, "client.BootInstance(...)")
if err := client.BootInstance(ctx, instanceID, configID); err != nil {
return fmt.Errorf("failed to boot instance: %s", err)
}

tflog.Debug(ctx, "Waiting for instance boot to finish")

if _, err := p.WaitForFinished(ctx, deadlineSeconds); err != nil {
return fmt.Errorf("failed to wait for instance boot: %s", err)
}

tflog.Debug(ctx, "Instance has finished booting")

return nil
}

func getDiskSizeSum(ctx context.Context, d *schema.ResourceData,
client *linodego.Client, instanceID int,
) (int, error) {
Expand Down Expand Up @@ -1431,7 +1331,7 @@ func VPCInterfaceIncluded(

func BootInstanceAfterVPCInterfaceUpdate(ctx context.Context, meta *helper.ProviderMeta, instanceID, targetConfigID, deadlineSeconds int) diag.Diagnostics {
tflog.Debug(ctx, "Booting instance after VPC interface change applied")
if err := BootInstanceSync(
if err := helper.BootInstanceSync(
ctx, &meta.Client, instanceID, targetConfigID, deadlineSeconds,
); err != nil {
return diag.Errorf("failed to boot instance after VPC interface change applied: %s", err)
Expand All @@ -1452,23 +1352,19 @@ func ShutdownInstanceForVPCInterfaceUpdate(ctx context.Context, client *linodego
}

func SafeShutdownInstance(ctx context.Context, client *linodego.Client, instanceID, deadlineSeconds int) error {
instance, err := client.GetInstance(ctx, instanceID)
if err != nil {
return fmt.Errorf("failed to get instance %d: %s", instanceID, err)
}

tflog.Debug(ctx, "Shutting down Linode instance")

if _, err := waitForRunningOrOfflineState(
ctx, instance.Status, client, instance.ID,
); err != nil {
instanceStatus, err := helper.WaitForInstanceNonTransientStatus(
ctx, client, instanceID, 120,
)
if err != nil {
return fmt.Errorf(
"failed waiting for instance %d to be in running or offline state: %s", instance.ID, err,
"failed waiting for instance %d to be in running or offline state: %s", instanceID, err,
)
}
if instance.Status != linodego.InstanceOffline {
if err := shutDownInstanceSync(
ctx, *client, instance.ID, deadlineSeconds,
if instanceStatus != linodego.InstanceOffline {
if err := helper.ShutDownInstanceSync(
ctx, client, instanceID, deadlineSeconds,
); err != nil {
return fmt.Errorf("failed to shutdown instance: %s", err)
}
Expand Down
Loading

0 comments on commit 86e3962

Please sign in to comment.