Creates 'version.json' in each plugin directory. Recompose the global…

… plugin versions.json if it is missing or corrupt. Closes #3492

.github/workflows/test.yml

@@ -25,13 +25,6 @@ jobs:
         with:
           go-version: 1.19
 
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        continue-on-error: true # we dont want to enforce just yet
-        with:
-          version: v1.52.2
-          args: --timeout=15m --config=.golangci.yml
-
       - name: Fetching Go Cache Paths
         id: go-cache-paths
         run: |
@@ -46,13 +39,12 @@ jobs:
           path: ${{ steps.go-cache-paths.outputs.go-build }}
           key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
 
-      # Cache go mod cache, used to speedup builds
-      - name: Go Mod Cache
-        id: mod-cache
-        uses: actions/cache@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        continue-on-error: true # we dont want to enforce just yet
         with:
-          path: ${{ steps.go-cache-paths.outputs.go-mod }}
-          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+          version: v1.52.2
+          args: --timeout=15m --config=.golangci.yml
 
       - name: Run CLI Unit Tests
         run: |

cmd/plugin.go

@@ -239,7 +239,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
 		fmt.Println()
 		error_helpers.ShowError(ctx, fmt.Errorf("you need to provide at least one plugin to install"))
 		fmt.Println()
-		cmd.Help()
+		_ = cmd.Help()
 		fmt.Println()
 		exitCode = constants.ExitCodeInsufficientOrWrongInputs
 		return
@@ -584,7 +584,7 @@ func resolveUpdatePluginsFromArgs(args []string) ([]string, error) {
 	return plugins, nil
 }
 
-func runPluginListCmd(cmd *cobra.Command, args []string) {
+func runPluginListCmd(cmd *cobra.Command, _ []string) {
 	// setup a cancel context and start cancel handler
 	ctx, cancel := context.WithCancel(cmd.Context())
 	contexthelpers.StartCancelHandler(cancel)
@@ -805,6 +805,9 @@ func getPluginList(ctx context.Context) (pluginList []plugin.PluginListItem, fai
 }
 
 func getPluginConnectionMap(ctx context.Context) (pluginConnectionMap, failedPluginMap, missingPluginMap map[string][]*modconfig.Connection, res *modconfig.ErrorAndWarnings) {
+	utils.LogTime("cmd.getPluginConnectionMap start")
+	defer utils.LogTime("cmd.getPluginConnectionMap end")
+
 	statushooks.SetStatus(ctx, "Fetching connection map")
 
 	res = &modconfig.ErrorAndWarnings{}
@@ -840,6 +843,9 @@ func getPluginConnectionMap(ctx context.Context) (pluginConnectionMap, failedPlu
 
 // load the connection state, waiting until all connections are loaded
 func getConnectionState(ctx context.Context) (steampipeconfig.ConnectionStateMap, *modconfig.ErrorAndWarnings) {
+	utils.LogTime("cmd.getConnectionState start")
+	defer utils.LogTime("cmd.getConnectionState end")
+
 	// start service
 	client, res := db_local.GetLocalClient(ctx, constants.InvokerPlugin, nil)
 	if res.Error != nil {

cmd/root.go

@@ -96,6 +96,13 @@ var rootCmd = &cobra.Command{
 		// runScheduledTasks skips running tasks if this instance is the plugin manager
 		waitForTasksChannel = runScheduledTasks(cmd.Context(), cmd, args, ew)
 
+		// ensure all plugin installation directories have a version.json file
+		// (this is to handle the case of migrating an existing installation from v0.20.x)
+		// no point doing this for the plugin-manager since that would have been done by the initiating CLI process
+		if !task.IsPluginManagerCmd(cmd) {
+			versionfile.EnsureVersionFilesInPluginDirectories()
+		}
+
 		// set the max memory
 		debug.SetMemoryLimit(plugin.GetMaxMemoryBytes())
 	},

pkg/db/db_client/db_client.go

@@ -121,7 +121,7 @@ func (c *DbClient) loadServerSettings(ctx context.Context) error {
 			// when connecting to pre-0.21.0 services, the server_settings table will not be available.
 			// this is expected and not an error
 			// code which uses server_settings should handle this
-			log.Printf("[INFO] could not find %s.%s table.", constants.InternalSchema, constants.ServerSettingsTable)
+			log.Printf("[TRACE] could not find %s.%s table. skipping\n", constants.InternalSchema, constants.ServerSettingsTable)
 			return nil
 		}
 		return err

pkg/ociinstaller/plugin.go

@@ -19,6 +19,8 @@ import (
 	"github.com/turbot/steampipe/pkg/utils"
 )
 
+var versionFileUpdateLock = &sync.Mutex{}
+
 // InstallPlugin installs a plugin from an OCI Image
 func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}) (*SteampipeImage, error) {
 	tempDir := NewTempDir(filepaths.EnsurePluginDir())
@@ -51,17 +53,16 @@ func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}) (*St
 	if err = installPluginConfigFiles(image, tempDir.Path); err != nil {
 		return nil, fmt.Errorf("plugin installation failed: %s", err)
 	}
-
 	sub <- struct{}{}
-	if err := updateVersionFilePlugin(image); err != nil {
+	if err := updatePluginVersionFiles(image); err != nil {
 		return nil, err
 	}
 	return image, nil
 }
 
-var versionFileUpdateLock = &sync.Mutex{}
-
-func updateVersionFilePlugin(image *SteampipeImage) error {
+// updatePluginVersionFiles updates the global versions.json to add installation of the plugin
+// also adds a version file in the plugin installation directory with the information
+func updatePluginVersionFiles(image *SteampipeImage) error {
 	versionFileUpdateLock.Lock()
 	defer versionFileUpdateLock.Unlock()
 
@@ -73,23 +74,31 @@ func updateVersionFilePlugin(image *SteampipeImage) error {
 
 	pluginFullName := image.ImageRef.DisplayImageRef()
 
-	plugin, ok := v.Plugins[pluginFullName]
+	installedVersion, ok := v.Plugins[pluginFullName]
 	if !ok {
-		plugin = &versionfile.InstalledVersion{}
+		installedVersion = versionfile.EmptyInstalledVersion()
+	}
+
+	installedVersion.Name = pluginFullName
+	installedVersion.Version = image.Config.Plugin.Version
+	installedVersion.ImageDigest = string(image.OCIDescriptor.Digest)
+	installedVersion.BinaryDigest = image.Plugin.BinaryDigest
+	installedVersion.BinaryArchitecture = image.Plugin.BinaryArchitecture
+	installedVersion.InstalledFrom = image.ImageRef.ActualImageRef()
+	installedVersion.LastCheckedDate = timeNow
+	installedVersion.InstallDate = timeNow
+
+	v.Plugins[pluginFullName] = installedVersion
+
+	// Ensure that the version file is written to the plugin installation folder
+	// Having this file is important, since this can be used
+	// to compose the global version file if it is unavailable or unparseable
+	// This makes sure that in the event of corruption (global/individual) we don't end up
+	// losing all the plugin install data
+	if err := v.EnsurePluginVersionFile(installedVersion); err != nil {
+		return err
 	}
 
-	//change this to the path????
-	plugin.Name = pluginFullName
-	plugin.Version = image.Config.Plugin.Version
-	plugin.ImageDigest = string(image.OCIDescriptor.Digest)
-	plugin.BinaryDigest = image.Plugin.BinaryDigest
-	plugin.BinaryArchitecture = image.Plugin.BinaryArchitecture
-	plugin.InstalledFrom = image.ImageRef.ActualImageRef()
-	plugin.LastCheckedDate = timeNow
-	plugin.InstallDate = timeNow
-
-	v.Plugins[pluginFullName] = plugin
-
 	return v.Save()
 }
 

pkg/ociinstaller/versionfile/db_version_file.go

@@ -47,13 +47,6 @@ func (s *DatabaseVersionFile) MigrateFrom() migrate.Migrateable {
 	return s
 }
 
-func databaseVersionFileFromLegacy(legacyFile *LegacyCompositeVersionFile) *DatabaseVersionFile {
-	return &DatabaseVersionFile{
-		FdwExtension: legacyFile.FdwExtension,
-		EmbeddedDB:   legacyFile.EmbeddedDB,
-	}
-}
-
 // LoadDatabaseVersionFile migrates from the old version file format if necessary and loads the database version data
 func LoadDatabaseVersionFile() (*DatabaseVersionFile, error) {
 	versionFilePath := filepaths.DatabaseVersionFilePath()

pkg/ociinstaller/versionfile/installed_version.go

@@ -1,5 +1,7 @@
 package versionfile
 
+const InstalledVersionStructVersion = 20230502
+
 type InstalledVersion struct {
 	Name               string `json:"name"`
 	Version            string `json:"version"`
@@ -15,6 +17,19 @@ type InstalledVersion struct {
 	LegacyInstalledFrom   string `json:"installedFrom,omitempty"`
 	LegacyLastCheckedDate string `json:"lastCheckedDate,omitempty"`
 	LegacyInstallDate     string `json:"installDate,omitempty"`
+
+	StructVersion int64 `json:"struct_version"`
+}
+
+func EmptyInstalledVersion() *InstalledVersion {
+	i := new(InstalledVersion)
+	i.StructVersion = InstalledVersionStructVersion
+	return i
+}
+
+// Equal compares the `Name` and `BinaryDigest`
+func (f *InstalledVersion) Equal(other *InstalledVersion) bool {
+	return f.Name == other.Name && f.BinaryDigest == other.BinaryDigest
 }
 
 // MigrateLegacy migrates the legacy properties into new properties