Plugin schema validation errors should be displayed as warnings rather than cause steampipe to exit. Closes #644

constants/db.go

@@ -19,7 +19,7 @@ const (
 // constants for installing db and fdw images
 const (
 	DatabaseVersion = "12.1.0"
-	FdwVersion      = "0.1.0-rc.3"
+	FdwVersion      = "0.1.0-rc.4"
 
 	// DefaultEmbeddedPostgresImage :: The 12.1.0 image uses the older jar format 12.1.0-v2 is the same version of postgres,
 	// just packaged as gzipped tar files (consistent with oras, faster to unzip).  Once everyone is

db/client_connections.go

@@ -46,12 +46,13 @@ func (c *Client) RefreshConnections() *RefreshConnectionResult {
 	numUpdates := len(updates.Update)
 	if numUpdates > 0 {
 
-		// first instantiate connection plugins for all updates
-		connectionPlugins, err := getConnectionPlugins(updates.Update)
-		if err != nil {
-			res.Error = err
+		// first instantiate connection plugins for all updates (reuse 'res' defined above)
+		var connectionPlugins []*steampipeconfig.ConnectionPlugin
+		connectionPlugins, res = getConnectionPlugins(updates.Update)
+		if res.Error != nil {
 			return res
 		}
+
 		// find any plugins which use a newer sdk version than steampipe.
 		validationFailures, validatedUpdates, validatedPlugins := steampipeconfig.ValidatePlugins(updates.Update, connectionPlugins)
 		res.Warnings = append(res.Warnings, steampipeconfig.BuildValidationWarningString(validationFailures))
@@ -102,33 +103,36 @@ func (c *Client) updateConnectionMap() error {
 	return nil
 }
 
-func getConnectionPlugins(updates steampipeconfig.ConnectionMap) ([]*steampipeconfig.ConnectionPlugin, error) {
+func getConnectionPlugins(updates steampipeconfig.ConnectionMap) ([]*steampipeconfig.ConnectionPlugin, *RefreshConnectionResult) {
+	res := &RefreshConnectionResult{}
 	var connectionPlugins []*steampipeconfig.ConnectionPlugin
+
+	// create channels buffered to hold all updates
 	numUpdates := len(updates)
 	var pluginChan = make(chan *steampipeconfig.ConnectionPlugin, numUpdates)
 	var errorChan = make(chan error, numUpdates)
-	for connectionName, connectionData := range updates {
 
+	for connectionName, connectionData := range updates {
 		// instantiate the connection plugin, and retrieve schema
-		go getConnectionPluginsAsync(connectionName, connectionData, pluginChan, errorChan)
+		go getConnectionPluginAsync(connectionName, connectionData, pluginChan, errorChan)
 	}
 
 	for i := 0; i < numUpdates; i++ {
 		select {
 		case err := <-errorChan:
-			log.Println("[TRACE] get connections err chan select", "error", err)
-			return nil, err
-		case <-time.After(10 * time.Second):
-			return nil, fmt.Errorf("timed out retrieving schema from plugins")
+			log.Println("[TRACE] get connections err chan select - adding warning", "error", err)
+			res.Warnings = append(res.Warnings, err.Error())
 		case p := <-pluginChan:
 			connectionPlugins = append(connectionPlugins, p)
+		case <-time.After(10 * time.Second):
+			res.Error = fmt.Errorf("timed out retrieving schema from plugins")
+			return nil, res
 		}
 	}
-
-	return connectionPlugins, nil
+	return connectionPlugins, res
 }
 
-func getConnectionPluginsAsync(connectionName string, connectionData *steampipeconfig.ConnectionData, pluginChan chan *steampipeconfig.ConnectionPlugin, errorChan chan error) {
+func getConnectionPluginAsync(connectionName string, connectionData *steampipeconfig.ConnectionData, pluginChan chan *steampipeconfig.ConnectionPlugin, errorChan chan error) {
 	opts := &steampipeconfig.ConnectionPluginInput{
 		ConnectionName:    connectionName,
 		PluginName:        connectionData.Plugin,

db/init_result.go

@@ -21,6 +21,7 @@ func (r *InitResult) HasMessages() bool {
 }
 
 func (r *InitResult) DisplayMessages() {
+	fmt.Println()
 	for _, w := range r.Warnings {
 		fmt.Println(w)
 	}

db/interactive_client.go

@@ -140,8 +140,10 @@ func (c *InteractiveClient) handleInitResult(ctx context.Context, initResult *In
 
 	// if there is an error, shutdown
 	if initResult.Error != nil {
-		utils.ShowError(initResult.Error)
 		c.ClosePrompt(AfterPromptCloseExit)
+		// add newline to ensure error is not printed at end of current prompt line
+		fmt.Println()
+		utils.ShowError(initResult.Error)
 		return initResult.Error
 	}
 	if initResult.HasMessages() {

steampipeconfig/test_data/mods/many controls/c1/control.sp

@@ -0,0 +1,101 @@
+
+control "cis_v130_1_1" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q1.sql
+}
+control "cis_v130_1_2" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q2.sql
+}
+control "cis_v130_1_3" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q3.sql
+}
+control "cis_v130_1_4" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q4.sql
+}
+control "cis_v130_1_5" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q5.sql
+}
+control "cis_v130_1_6" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q6.sql
+}
+control "cis_v130_1_7" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q7.sql
+}
+control "cis_v130_1_8" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q8.sql
+}
+control "cis_v130_1_9" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q9.sql
+}
+control "cis_v130_1_10" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q10.sql
+}
+control "cis_v130_1_11" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q11.sql
+}
+control "cis_v130_1_12" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q12.sql
+}
+control "cis_v130_1_13" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q13.sql
+}
+control "cis_v130_1_14" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q14.sql
+}
+control "cis_v130_1_15" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q15.sql
+}
+control "cis_v130_1_16" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q16.sql
+}
+control "cis_v130_1_17" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q17.sql
+}
+control "cis_v130_1_18" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q18.sql
+}
+control "cis_v130_1_19" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q19.sql
+}
+control "cis_v130_1_20" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q20.sql
+}

steampipeconfig/test_data/mods/many controls/c1/control10.sp

@@ -0,0 +1,101 @@
+
+control "cis_v1310_1_1" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q1.sql
+}
+control "cis_v1310_1_2" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q2.sql
+}
+control "cis_v1310_1_3" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q3.sql
+}
+control "cis_v1310_1_4" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q4.sql
+}
+control "cis_v1310_1_5" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q5.sql
+}
+control "cis_v1310_1_6" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q6.sql
+}
+control "cis_v1310_1_7" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q7.sql
+}
+control "cis_v1310_1_8" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q8.sql
+}
+control "cis_v1310_1_9" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q9.sql
+}
+control "cis_v1310_1_10" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q10.sql
+}
+control "cis_v1310_1_11" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q11.sql
+}
+control "cis_v1310_1_12" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q12.sql
+}
+control "cis_v1310_1_13" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q13.sql
+}
+control "cis_v1310_1_14" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q14.sql
+}
+control "cis_v1310_1_15" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q15.sql
+}
+control "cis_v1310_1_16" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q16.sql
+}
+control "cis_v1310_1_17" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q17.sql
+}
+control "cis_v1310_1_18" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q18.sql
+}
+control "cis_v1310_1_19" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q19.sql
+}
+control "cis_v1310_1_20" {
+    title         = "1.1 Maintain current contact details"
+    description   = "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization."
+    sql           = query.q20.sql
+}