fix: handle NATS connection closing itself

If the NATS server restarts then the connection will close itself due to
authorization failures. Previously the connection closing itself instead
of being instigated by service-api could lead to a deadlock by blocking
on <-ctx.Done().

This change cancels the context in the NATS connection closed handler to
avoid the deadlock.

cmd/service-api/serve.go

@@ -57,6 +57,6 @@ func (cmd *ServeCmd) Run(log *zap.Logger) error {
 		return fmt.Errorf("couldn't init keycloak Client: %v", err)
 	}
 	// start serving NATS requests
-	return serviceapi.ServeNATS(ctx, log, l, k, cmd.NATSURL, cmd.NATSUsername,
-		cmd.NATSPassword)
+	return serviceapi.ServeNATS(ctx, stop, log, l, k, cmd.NATSURL,
+		cmd.NATSUsername, cmd.NATSPassword)
 }

internal/serviceapi/server.go

@@ -28,15 +28,17 @@ type KeycloakService interface {
 }
 
 // ServeNATS serviceapi NATS requests.
-func ServeNATS(ctx context.Context, log *zap.Logger, l LagoonDBService,
-	k KeycloakService, natsURL, natsUser, natsPass string) error {
+func ServeNATS(ctx context.Context, stop context.CancelFunc, log *zap.Logger,
+	l LagoonDBService, k KeycloakService, natsURL, natsUser,
+	natsPass string) error {
 	// setup synchronisation
 	wg := sync.WaitGroup{}
 	wg.Add(1)
 	// connect to NATS server
 	nc, err := nats.Connect(natsURL,
 		// synchronise exiting ServeNATS()
 		nats.ClosedHandler(func(_ *nats.Conn) {
+			stop()
 			wg.Done()
 		}),
 		// pass credentials