From 9754ed0c1a84e996d820dfe75199131046d83727 Mon Sep 17 00:00:00 2001 From: Binaek Sarkar Date: Fri, 30 Jun 2023 13:26:59 +0530 Subject: [PATCH] Creates 'version.json' in each plugin directory. Recompose the global plugin versions.json if it is missing or corrupt. Closes #3492 --- .github/workflows/test.yml | 18 +- cmd/plugin.go | 10 +- cmd/root.go | 7 + pkg/db/db_client/db_client.go | 2 +- pkg/ociinstaller/plugin.go | 47 +++-- .../versionfile/db_version_file.go | 7 - .../versionfile/installed_version.go | 15 ++ .../versionfile/legacy_version_file.go | 87 --------- .../versionfile/plugin_version_file.go | 177 +++++++++++++++--- .../versionfile/plugin_version_file_test.go | 2 +- pkg/plugin/actions.go | 105 +++++++---- tests/acceptance/test_files/cloud.bats | 3 + .../test_files/service_and_plugin.bats | 157 +++++++++++++++- 13 files changed, 445 insertions(+), 192 deletions(-) delete mode 100644 pkg/ociinstaller/versionfile/legacy_version_file.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 878c07ffa0..93081466be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,13 +25,6 @@ jobs: with: go-version: 1.19 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - continue-on-error: true # we dont want to enforce just yet - with: - version: v1.52.2 - args: --timeout=15m --config=.golangci.yml - - name: Fetching Go Cache Paths id: go-cache-paths run: | @@ -46,13 +39,12 @@ jobs: path: ${{ steps.go-cache-paths.outputs.go-build }} key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} - # Cache go mod cache, used to speedup builds - - name: Go Mod Cache - id: mod-cache - uses: actions/cache@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + continue-on-error: true # we dont want to enforce just yet with: - path: ${{ steps.go-cache-paths.outputs.go-mod }} - key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + version: v1.52.2 + args: --timeout=15m --config=.golangci.yml - name: Run CLI Unit Tests run: | diff --git a/cmd/plugin.go b/cmd/plugin.go index 033af59a0f..1863d0d3f2 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -239,7 +239,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) { fmt.Println() error_helpers.ShowError(ctx, fmt.Errorf("you need to provide at least one plugin to install")) fmt.Println() - cmd.Help() + _ = cmd.Help() fmt.Println() exitCode = constants.ExitCodeInsufficientOrWrongInputs return @@ -584,7 +584,7 @@ func resolveUpdatePluginsFromArgs(args []string) ([]string, error) { return plugins, nil } -func runPluginListCmd(cmd *cobra.Command, args []string) { +func runPluginListCmd(cmd *cobra.Command, _ []string) { // setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) @@ -805,6 +805,9 @@ func getPluginList(ctx context.Context) (pluginList []plugin.PluginListItem, fai } func getPluginConnectionMap(ctx context.Context) (pluginConnectionMap, failedPluginMap, missingPluginMap map[string][]*modconfig.Connection, res *modconfig.ErrorAndWarnings) { + utils.LogTime("cmd.getPluginConnectionMap start") + defer utils.LogTime("cmd.getPluginConnectionMap end") + statushooks.SetStatus(ctx, "Fetching connection map") res = &modconfig.ErrorAndWarnings{} @@ -840,6 +843,9 @@ func getPluginConnectionMap(ctx context.Context) (pluginConnectionMap, failedPlu // load the connection state, waiting until all connections are loaded func getConnectionState(ctx context.Context) (steampipeconfig.ConnectionStateMap, *modconfig.ErrorAndWarnings) { + utils.LogTime("cmd.getConnectionState start") + defer utils.LogTime("cmd.getConnectionState end") + // start service client, res := db_local.GetLocalClient(ctx, constants.InvokerPlugin, nil) if res.Error != nil { diff --git a/cmd/root.go b/cmd/root.go index f4561238e6..b8f4b74d47 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -96,6 +96,13 @@ var rootCmd = &cobra.Command{ // runScheduledTasks skips running tasks if this instance is the plugin manager waitForTasksChannel = runScheduledTasks(cmd.Context(), cmd, args, ew) + // ensure all plugin installation directories have a version.json file + // (this is to handle the case of migrating an existing installation from v0.20.x) + // no point doing this for the plugin-manager since that would have been done by the initiating CLI process + if !task.IsPluginManagerCmd(cmd) { + versionfile.EnsureVersionFilesInPluginDirectories() + } + // set the max memory debug.SetMemoryLimit(plugin.GetMaxMemoryBytes()) }, diff --git a/pkg/db/db_client/db_client.go b/pkg/db/db_client/db_client.go index 8289f1219c..5069c88bf9 100644 --- a/pkg/db/db_client/db_client.go +++ b/pkg/db/db_client/db_client.go @@ -121,7 +121,7 @@ func (c *DbClient) loadServerSettings(ctx context.Context) error { // when connecting to pre-0.21.0 services, the server_settings table will not be available. // this is expected and not an error // code which uses server_settings should handle this - log.Printf("[INFO] could not find %s.%s table.", constants.InternalSchema, constants.ServerSettingsTable) + log.Printf("[TRACE] could not find %s.%s table. skipping\n", constants.InternalSchema, constants.ServerSettingsTable) return nil } return err diff --git a/pkg/ociinstaller/plugin.go b/pkg/ociinstaller/plugin.go index b4e81668e5..5e62a51da3 100644 --- a/pkg/ociinstaller/plugin.go +++ b/pkg/ociinstaller/plugin.go @@ -19,6 +19,8 @@ import ( "github.com/turbot/steampipe/pkg/utils" ) +var versionFileUpdateLock = &sync.Mutex{} + // InstallPlugin installs a plugin from an OCI Image func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}) (*SteampipeImage, error) { tempDir := NewTempDir(filepaths.EnsurePluginDir()) @@ -51,17 +53,16 @@ func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}) (*St if err = installPluginConfigFiles(image, tempDir.Path); err != nil { return nil, fmt.Errorf("plugin installation failed: %s", err) } - sub <- struct{}{} - if err := updateVersionFilePlugin(image); err != nil { + if err := updatePluginVersionFiles(image); err != nil { return nil, err } return image, nil } -var versionFileUpdateLock = &sync.Mutex{} - -func updateVersionFilePlugin(image *SteampipeImage) error { +// updatePluginVersionFiles updates the global versions.json to add installation of the plugin +// also adds a version file in the plugin installation directory with the information +func updatePluginVersionFiles(image *SteampipeImage) error { versionFileUpdateLock.Lock() defer versionFileUpdateLock.Unlock() @@ -73,23 +74,31 @@ func updateVersionFilePlugin(image *SteampipeImage) error { pluginFullName := image.ImageRef.DisplayImageRef() - plugin, ok := v.Plugins[pluginFullName] + installedVersion, ok := v.Plugins[pluginFullName] if !ok { - plugin = &versionfile.InstalledVersion{} + installedVersion = versionfile.EmptyInstalledVersion() + } + + installedVersion.Name = pluginFullName + installedVersion.Version = image.Config.Plugin.Version + installedVersion.ImageDigest = string(image.OCIDescriptor.Digest) + installedVersion.BinaryDigest = image.Plugin.BinaryDigest + installedVersion.BinaryArchitecture = image.Plugin.BinaryArchitecture + installedVersion.InstalledFrom = image.ImageRef.ActualImageRef() + installedVersion.LastCheckedDate = timeNow + installedVersion.InstallDate = timeNow + + v.Plugins[pluginFullName] = installedVersion + + // Ensure that the version file is written to the plugin installation folder + // Having this file is important, since this can be used + // to compose the global version file if it is unavailable or unparseable + // This makes sure that in the event of corruption (global/individual) we don't end up + // losing all the plugin install data + if err := v.EnsurePluginVersionFile(installedVersion); err != nil { + return err } - //change this to the path???? - plugin.Name = pluginFullName - plugin.Version = image.Config.Plugin.Version - plugin.ImageDigest = string(image.OCIDescriptor.Digest) - plugin.BinaryDigest = image.Plugin.BinaryDigest - plugin.BinaryArchitecture = image.Plugin.BinaryArchitecture - plugin.InstalledFrom = image.ImageRef.ActualImageRef() - plugin.LastCheckedDate = timeNow - plugin.InstallDate = timeNow - - v.Plugins[pluginFullName] = plugin - return v.Save() } diff --git a/pkg/ociinstaller/versionfile/db_version_file.go b/pkg/ociinstaller/versionfile/db_version_file.go index b54df89bd9..30c0293ade 100644 --- a/pkg/ociinstaller/versionfile/db_version_file.go +++ b/pkg/ociinstaller/versionfile/db_version_file.go @@ -47,13 +47,6 @@ func (s *DatabaseVersionFile) MigrateFrom() migrate.Migrateable { return s } -func databaseVersionFileFromLegacy(legacyFile *LegacyCompositeVersionFile) *DatabaseVersionFile { - return &DatabaseVersionFile{ - FdwExtension: legacyFile.FdwExtension, - EmbeddedDB: legacyFile.EmbeddedDB, - } -} - // LoadDatabaseVersionFile migrates from the old version file format if necessary and loads the database version data func LoadDatabaseVersionFile() (*DatabaseVersionFile, error) { versionFilePath := filepaths.DatabaseVersionFilePath() diff --git a/pkg/ociinstaller/versionfile/installed_version.go b/pkg/ociinstaller/versionfile/installed_version.go index 69e25b7949..9809455367 100644 --- a/pkg/ociinstaller/versionfile/installed_version.go +++ b/pkg/ociinstaller/versionfile/installed_version.go @@ -1,5 +1,7 @@ package versionfile +const InstalledVersionStructVersion = 20230502 + type InstalledVersion struct { Name string `json:"name"` Version string `json:"version"` @@ -15,6 +17,19 @@ type InstalledVersion struct { LegacyInstalledFrom string `json:"installedFrom,omitempty"` LegacyLastCheckedDate string `json:"lastCheckedDate,omitempty"` LegacyInstallDate string `json:"installDate,omitempty"` + + StructVersion int64 `json:"struct_version"` +} + +func EmptyInstalledVersion() *InstalledVersion { + i := new(InstalledVersion) + i.StructVersion = InstalledVersionStructVersion + return i +} + +// Equal compares the `Name` and `BinaryDigest` +func (f *InstalledVersion) Equal(other *InstalledVersion) bool { + return f.Name == other.Name && f.BinaryDigest == other.BinaryDigest } // MigrateLegacy migrates the legacy properties into new properties diff --git a/pkg/ociinstaller/versionfile/legacy_version_file.go b/pkg/ociinstaller/versionfile/legacy_version_file.go deleted file mode 100644 index 2bc780b964..0000000000 --- a/pkg/ociinstaller/versionfile/legacy_version_file.go +++ /dev/null @@ -1,87 +0,0 @@ -package versionfile - -import ( - "encoding/json" - "log" - "os" - - filehelpers "github.com/turbot/go-kit/files" - "github.com/turbot/steampipe/pkg/filepaths" -) - -// LegacyCompositeVersionFile is the composite version file used before v0.7.0, which contained -// both db and plugin properties, now split into two different files -type LegacyCompositeVersionFile struct { - Plugins map[string]*InstalledVersion `json:"plugins"` - FdwExtension InstalledVersion `json:"fdwExtension"` - EmbeddedDB InstalledVersion `json:"embeddedDB"` -} - -// LoadLegacyVersionFile loads the legacy version file, or returns nil if it does not exist -func LoadLegacyVersionFile() (*LegacyCompositeVersionFile, error) { - versionFilePath := filepaths.LegacyVersionFilePath() - if filehelpers.FileExists(versionFilePath) { - return readLegacyVersionFile(versionFilePath) - } - return nil, nil -} - -func readLegacyVersionFile(path string) (*LegacyCompositeVersionFile, error) { - file, _ := os.ReadFile(path) - - var data LegacyCompositeVersionFile - - if err := json.Unmarshal([]byte(file), &data); err != nil { - log.Println("[ERROR]", "Error while reading version file", err) - return nil, err - } - - if data.Plugins == nil { - data.Plugins = map[string]*InstalledVersion{} - } - - for key := range data.Plugins { - // hard code the name to the key - data.Plugins[key].Name = key - } - - return &data, nil -} - -func migrateVersionFiles() (*PluginVersionFile, *DatabaseVersionFile, error) { - legacyVersionFile, err := LoadLegacyVersionFile() - if err != nil { - return nil, nil, err - } - if legacyVersionFile == nil { - return nil, nil, nil - } - - log.Printf("[TRACE] migrating version file from '%s' to '%s' and '%s'\n", - filepaths.LegacyVersionFilePath(), - filepaths.DatabaseVersionFilePath(), - filepaths.PluginVersionFilePath()) - - pluginVersionFile := pluginVersionFileFromLegacy(legacyVersionFile) - databaseVersionFile := databaseVersionFileFromLegacy(legacyVersionFile) - - // save the new files and remove the old one - if err := pluginVersionFile.Save(); err != nil { - return nil, nil, err - } - if err := databaseVersionFile.Save(); err != nil { - // delete the plugin version file which we have already saved - pluginVersionFile.delete() - return nil, nil, err - } - legacyVersionFile.delete() - return pluginVersionFile, databaseVersionFile, nil -} - -// delete the file on disk if it exists -func (f *LegacyCompositeVersionFile) delete() { - versionFilePath := filepaths.LegacyVersionFilePath() - if filehelpers.FileExists(versionFilePath) { - os.Remove(versionFilePath) - } -} diff --git a/pkg/ociinstaller/versionfile/plugin_version_file.go b/pkg/ociinstaller/versionfile/plugin_version_file.go index 0c2d1c51fa..0c9469f9b7 100644 --- a/pkg/ociinstaller/versionfile/plugin_version_file.go +++ b/pkg/ociinstaller/versionfile/plugin_version_file.go @@ -3,8 +3,11 @@ package versionfile import ( "encoding/json" "errors" + "fmt" "log" "os" + "path/filepath" + "sync" filehelpers "github.com/turbot/go-kit/files" "github.com/turbot/steampipe/pkg/filepaths" @@ -16,13 +19,24 @@ var ( ErrNoContent = errors.New("no content") ) -const PluginStructVersion = 20220411 +const ( + PluginStructVersion = 20220411 + // the name of the version files that are put in the plugin installation directories + pluginVersionFileName = "version.json" +) type PluginVersionFile struct { Plugins map[string]*InstalledVersion `json:"plugins"` StructVersion int64 `json:"struct_version"` } +func newPluginVersionFile() *PluginVersionFile { + return &PluginVersionFile{ + Plugins: map[string]*InstalledVersion{}, + StructVersion: PluginStructVersion, + } +} + // IsValid checks whether the struct was correctly deserialized, // by checking if the StructVersion is populated func (f PluginVersionFile) IsValid() bool { @@ -37,26 +51,34 @@ func (f *PluginVersionFile) MigrateFrom() migrate.Migrateable { return f } -func NewPluginVersionFile() *PluginVersionFile { - return &PluginVersionFile{ - Plugins: map[string]*InstalledVersion{}, - StructVersion: PluginStructVersion, +// EnsurePluginVersionFile reads the version file in the plugin directory (if exists) and overwrites it if the data in the +// argument is different. The comparison is done using the `Name` and `BinaryDigest` properties. +// If the file doesn't exist, or cannot be read/parsed, EnsurePluginVersionFile fails over to overwriting the data +func (f *PluginVersionFile) EnsurePluginVersionFile(installData *InstalledVersion) error { + pluginFolder, err := filepaths.FindPluginFolder(installData.Name) + if err != nil { + return err } -} - -func pluginVersionFileFromLegacy(legacyFile *LegacyCompositeVersionFile) *PluginVersionFile { - return &PluginVersionFile{ - Plugins: legacyFile.Plugins, + versionFile := filepath.Join(pluginFolder, pluginVersionFileName) + + // If the version file already exists, we only write to it if the incoming data is newer + if filehelpers.FileExists(versionFile) { + installation, err := readPluginVersionFile(versionFile) + if err == nil && installation.Equal(installData) { + // the new and old data match - no need to overwrite + return nil + } + // in case of error, just failover to a overwrite } -} -// LoadPluginVersionFile migrates from the old version file format if necessary and loads the plugin version data -func LoadPluginVersionFile() (*PluginVersionFile, error) { - versionFilePath := filepaths.PluginVersionFilePath() - if filehelpers.FileExists(versionFilePath) { - return readPluginVersionFile(versionFilePath) + // make sure that the legacy fields are also filled in + installData.MaintainLegacy() + + theBytes, err := json.MarshalIndent(installData, "", " ") + if err != nil { + return err } - return NewPluginVersionFile(), nil + return os.WriteFile(versionFile, theBytes, 0644) } // Save writes the config file to disk @@ -84,15 +106,115 @@ func (f *PluginVersionFile) write(path string) error { return os.WriteFile(path, versionFileJSON, 0644) } -// delete the file on disk if it exists -func (f *PluginVersionFile) delete() { +func (f *PluginVersionFile) ensureVersionFilesInPluginDirectories() error { + for _, installation := range f.Plugins { + if err := f.EnsurePluginVersionFile(installation); err != nil { + return err + } + } + return nil +} + +// to lock plugin version file loads +var pluginLoadLock = sync.Mutex{} + +// LoadPluginVersionFile migrates from the old version file format if necessary and loads the plugin version data +func LoadPluginVersionFile() (*PluginVersionFile, error) { + // we need a lock here so that we don't hit a race condition where + // the plugin file needs to be composed + // if recomposition is not required, this has (almost) zero penalty + pluginLoadLock.Lock() + defer pluginLoadLock.Unlock() + versionFilePath := filepaths.PluginVersionFilePath() if filehelpers.FileExists(versionFilePath) { - os.Remove(versionFilePath) + pluginVersions, err := readGlobalPluginVersionsFile(versionFilePath) + + // we could read and parse out the file - all is well + if err == nil { + return pluginVersions, nil + } + + // check if this was a syntax error during parsing + var syntaxError *json.SyntaxError + isSyntaxError := errors.As(err, &syntaxError) + if !isSyntaxError { + // not a syntax error - return the error + return nil, err + } + + // it was a syntax error, either the file is corrupted or empty - try to regenerate it + // if it was empty, and there are no plugin version files, regeneration will detect it and + // return and Empty structure + } + + // we don't have a global plugin/versions.json or it is not parseable + // generate the version file from the individual version files by walking the plugin directories + // this will return an Empty Version file if there are no version files in the plugin directories + pluginVersions := recomposePluginVersionFile() + + // save the recomposed file + err := pluginVersions.Save() + if err != nil { + return nil, err + } + return pluginVersions, err +} + +// EnsureVersionFilesInPluginDirectories attempts a backfill of the individual version.json for plugins +// this is required only once when upgrading from 0.20.x +func EnsureVersionFilesInPluginDirectories() error { + versions, err := LoadPluginVersionFile() + if err != nil { + return err + } + return versions.ensureVersionFilesInPluginDirectories() +} + +// recomposePluginVersionFile recursively traverses down the plugin direcory and tries to +// recompose the global version file from the plugin version files +// if there are no plugin version files, this returns a ready to use empty global version file +func recomposePluginVersionFile() *PluginVersionFile { + pvf := newPluginVersionFile() + + versionFiles, err := filehelpers.ListFiles(filepaths.EnsurePluginDir(), &filehelpers.ListOptions{ + Include: []string{fmt.Sprintf("**/%s", pluginVersionFileName)}, + Flags: filehelpers.FilesRecursive, + }) + + if err != nil { + log.Println("[TRACE] recomposePluginVersionFile failed - error while walking plugin directory for version files", err) + return pvf + } + + for _, versionFile := range versionFiles { + install, err := readPluginVersionFile(versionFile) + if err != nil { + log.Println("[TRACE] could not read file", versionFile) + continue + } + pvf.Plugins[install.Name] = install + } + + return pvf +} + +func readPluginVersionFile(versionFile string) (*InstalledVersion, error) { + data, err := os.ReadFile(versionFile) + if err != nil { + log.Println("[TRACE] could not read file", versionFile) + return nil, err + } + install := EmptyInstalledVersion() + if err := json.Unmarshal(data, &install); err != nil { + // this wasn't the version file (probably) - keep going + log.Println("[TRACE] unmarshal failed for file:", versionFile) + return nil, err } + return install, nil } -func readPluginVersionFile(path string) (*PluginVersionFile, error) { +func readGlobalPluginVersionsFile(path string) (*PluginVersionFile, error) { file, err := os.ReadFile(path) if err != nil { return nil, err @@ -100,13 +222,12 @@ func readPluginVersionFile(path string) (*PluginVersionFile, error) { if len(file) == 0 { // the file exists, but is empty // start from scratch - return NewPluginVersionFile(), nil + return newPluginVersionFile(), nil } var data PluginVersionFile - if err := json.Unmarshal([]byte(file), &data); err != nil { - log.Println("[WARN]", "Error while parsing plugin version file", err) + if err := json.Unmarshal(file, &data); err != nil { return nil, err } @@ -114,9 +235,13 @@ func readPluginVersionFile(path string) (*PluginVersionFile, error) { data.Plugins = map[string]*InstalledVersion{} } - for key := range data.Plugins { + for key, installedPlugin := range data.Plugins { // hard code the name to the key - data.Plugins[key].Name = key + installedPlugin.Name = key + if installedPlugin.StructVersion == 0 { + // also backfill the StructVersion in map values + installedPlugin.StructVersion = InstalledVersionStructVersion + } } return &data, nil diff --git a/pkg/ociinstaller/versionfile/plugin_version_file_test.go b/pkg/ociinstaller/versionfile/plugin_version_file_test.go index 82a2f9c934..a7343804ff 100644 --- a/pkg/ociinstaller/versionfile/plugin_version_file_test.go +++ b/pkg/ociinstaller/versionfile/plugin_version_file_test.go @@ -38,7 +38,7 @@ func TestWrite(t *testing.T) { if err := v.write(fileName); err != nil { t.Errorf("\nError writing file: %s", err.Error()) } - v2, err := readPluginVersionFile(fileName) + v2, err := readGlobalPluginVersionsFile(fileName) if err != nil { t.Errorf("\nError reading file: %s", err.Error()) } diff --git a/pkg/plugin/actions.go b/pkg/plugin/actions.go index f575f29dbb..e92fcb4008 100644 --- a/pkg/plugin/actions.go +++ b/pkg/plugin/actions.go @@ -3,17 +3,18 @@ package plugin import ( "context" "fmt" + "log" "os" "path/filepath" - "strings" + "time" + "github.com/turbot/go-kit/files" "github.com/turbot/steampipe/pkg/display" "github.com/turbot/steampipe/pkg/filepaths" "github.com/turbot/steampipe/pkg/ociinstaller" "github.com/turbot/steampipe/pkg/ociinstaller/versionfile" "github.com/turbot/steampipe/pkg/statushooks" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" - "github.com/turbot/steampipe/sperr" ) const ( @@ -85,22 +86,6 @@ type PluginListItem struct { func List(pluginConnectionMap map[string][]*modconfig.Connection) ([]PluginListItem, error) { var items []PluginListItem - var installedPlugins []string - - err := filepath.Walk(filepaths.EnsurePluginDir(), func(path string, info os.FileInfo, err error) error { - if !info.IsDir() && strings.HasSuffix(info.Name(), ".plugin") { - rel, err := filepath.Rel(filepaths.EnsurePluginDir(), filepath.Dir(path)) - if err != nil { - return err - } - installedPlugins = append(installedPlugins, rel) - } - return nil - }) - if err != nil { - return nil, sperr.WrapWithMessage(err, "could not traverse plugins directory") - } - v, err := versionfile.LoadPluginVersionFile() if err != nil { return nil, err @@ -108,32 +93,82 @@ func List(pluginConnectionMap map[string][]*modconfig.Connection) ([]PluginListI pluginVersions := v.Plugins - for _, plugin := range installedPlugins { - version := "local" - pluginDetails, found := pluginVersions[plugin] - if found { - version = pluginDetails.Version + pluginBinaries, err := files.ListFiles(filepaths.EnsurePluginDir(), &files.ListOptions{ + Include: []string{"**/*.plugin"}, + Flags: files.AllRecursive, + }) + if err != nil { + return nil, err + } + + // we have the plugin binary paths + for _, pluginBinary := range pluginBinaries { + parent := filepath.Dir(pluginBinary) + fullPluginName, err := filepath.Rel(filepaths.EnsurePluginDir(), parent) + if err != nil { + return nil, err } item := PluginListItem{ - Name: plugin, - Version: version, + Name: fullPluginName, + Version: "local", } + // check if this plugin is recorded in plugin versions + installation, found := pluginVersions[fullPluginName] + if found { + // use the version as recorded + item.Version = installation.Version + // but if the modtime of the binary is after the installation date, + // this is "local" + + if detectLocalPlugin(installation, pluginBinary) { + item.Version = "local" + } - if pluginConnectionMap != nil { - // extract only the connection names - var connectionNames []string - for _, connection := range pluginConnectionMap[plugin] { - connectionName := connection.Name - if connection.ImportDisabled() { - connectionName = fmt.Sprintf("%s(disabled)", connectionName) + if pluginConnectionMap != nil { + // extract only the connection names + var connectionNames []string + for _, connection := range pluginConnectionMap[fullPluginName] { + connectionName := connection.Name + if connection.ImportDisabled() { + connectionName = fmt.Sprintf("%s(disabled)", connectionName) + } + connectionNames = append(connectionNames, connectionName) } - connectionNames = append(connectionNames, connectionName) + item.Connections = connectionNames } - item.Connections = connectionNames + items = append(items, item) } - items = append(items, item) } return items, nil } + +// detectLocalPlugin returns true if the modTime of the `pluginBinary` is after the installation date as recorded in the installation data +// this may happen when a plugin is installed from the registry, but is then compiled from source +func detectLocalPlugin(installation *versionfile.InstalledVersion, pluginBinary string) bool { + installDate, err := time.Parse(time.RFC3339, installation.InstallDate) + if err != nil { + log.Printf("[WARN] could not parse install date for %s: %s", installation.Name, installation.InstallDate) + return false + } + + // truncate to second + // otherwise, comparisons may get skewed because of the + // underlying monotonic clock + installDate = installDate.Truncate(time.Second) + + // get the modtime of the plugin binary + stat, err := os.Lstat(pluginBinary) + if err != nil { + log.Printf("[WARN] could not parse install date for %s: %s", installation.Name, installation.InstallDate) + return false + } + modTime := stat.ModTime(). + // truncate to second + // otherwise, comparisons may get skewed because of the + // underlying monotonic clock + Truncate(time.Second) + + return installDate.Before(modTime) +} diff --git a/tests/acceptance/test_files/cloud.bats b/tests/acceptance/test_files/cloud.bats index 632f002e48..66955ccdf4 100644 --- a/tests/acceptance/test_files/cloud.bats +++ b/tests/acceptance/test_files/cloud.bats @@ -8,6 +8,7 @@ load "$LIB_BATS_SUPPORT/load.bash" @test "connect to cloud workspace - passing the postgres connection string to workspace-database arg" { # run steampipe query and fetch an account from the cloud workspace run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --workspace-database $SPIPETOOLS_PG_CONN_STRING --output json + echo $output # fetch the value of account_alias to compare op=$(echo $output | jq '.[0].account_aliases[0]') @@ -20,6 +21,7 @@ load "$LIB_BATS_SUPPORT/load.bash" @test "connect to cloud workspace - passing the cloud-token arg and the workspace name to workspace-database arg" { # run steampipe query and fetch an account from the cloud workspace run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --cloud-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json + echo $output # fetch the value of account_alias to compare op=$(echo $output | jq '.[0].account_aliases[0]') @@ -32,6 +34,7 @@ load "$LIB_BATS_SUPPORT/load.bash" @test "connect to cloud workspace - passing the cloud-host arg, the cloud-token arg and the workspace name to workspace-database arg" { # run steampipe query and fetch an account from the cloud workspace run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --cloud-host "cloud.steampipe.io" --cloud-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json + echo $output # fetch the value of account_alias to compare op=$(echo $output | jq '.[0].account_aliases[0]') diff --git a/tests/acceptance/test_files/service_and_plugin.bats b/tests/acceptance/test_files/service_and_plugin.bats index 6c58727ea2..553a331d2f 100644 --- a/tests/acceptance/test_files/service_and_plugin.bats +++ b/tests/acceptance/test_files/service_and_plugin.bats @@ -536,6 +536,161 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_equal "$output" "$(cat $TEST_DATA_DIR/expected_plugin_list_json_with_failed_plugins.json)" } +@test "verify that installing plugins creates individual version.json files" { + tmpdir=$(mktemp -d) + run steampipe plugin install net chaos --install-dir $tmpdir + assert_success + + vFile1="$tmpdir/plugins/hub.steampipe.io/plugins/turbot/net@latest/version.json" + vFile2="$tmpdir/plugins/hub.steampipe.io/plugins/turbot/chaos@latest/version.json" + + [ ! -f $vFile1 ] && fail "could not find $vFile1" + [ ! -f $vFile2 ] && fail "could not find $vFile2" + + rm -rf $tmpdir +} + +@test "verify that backfilling of individual plugin version.json works" { + tmpdir=$(mktemp -d) + run steampipe plugin install net chaos --install-dir $tmpdir + assert_success + + vFile1="$tmpdir/plugins/hub.steampipe.io/plugins/turbot/net@latest/version.json" + vFile2="$tmpdir/plugins/hub.steampipe.io/plugins/turbot/chaos@latest/version.json" + + file1Content=$(cat $vFile1) + file2Content=$(cat $vFile2) + + # remove the individual version files + rm -f $vFile1 + rm -f $vFile2 + + # run steampipe again so that the plugin version files get backfilled + run steampipe plugin list --install-dir $tmpdir + + [ ! -f $vFile1 ] && fail "could not find $vFile1" + [ ! -f $vFile2 ] && fail "could not find $vFile2" + + assert_equal "$(cat $vFile1)" "$file1Content" + assert_equal "$(cat $vFile2)" "$file2Content" + + rm -rf $tmpdir +} + +@test "verify that backfilling of individual plugin version.json works where it is only partially backfilled" { + tmpdir=$(mktemp -d) + run steampipe plugin install net chaos --install-dir $tmpdir + assert_success + + vFile1="$tmpdir/plugins/hub.steampipe.io/plugins/turbot/net@latest/version.json" + vFile2="$tmpdir/plugins/hub.steampipe.io/plugins/turbot/chaos@latest/version.json" + + file1Content=$(cat $vFile1) + file2Content=$(cat $vFile2) + + # remove one individual version file + rm -f $vFile1 + + # run steampipe again so that the plugin version files get backfilled + run steampipe plugin list --install-dir $tmpdir + + [ ! -f $vFile1 ] && fail "could not find $vFile1" + [ ! -f $vFile2 ] && fail "could not find $vFile2" + + assert_equal "$(cat $vFile1)" "$file1Content" + assert_equal "$(cat $vFile2)" "$file2Content" + + rm -rf $tmpdir +} + +@test "verify that global plugin/versions.json is composed from individual version.json files when it is absent" { + tmpdir=$(mktemp -d) + run steampipe plugin install net chaos --install-dir $tmpdir + assert_success + + vFile="$tmpdir/plugins/versions.json" + + fileContent=$(cat $vFile) + + # remove global version file + rm -f $vFile + + # run steampipe again so that the plugin version files get backfilled + run steampipe plugin list --install-dir $tmpdir + + ls -la $vFile + + [ ! -f $vFile ] && fail "could not find $vFile" + + assert_equal "$(cat $vFile)" "$fileContent" + + rm -rf $tmpdir +} + +@test "verify that global plugin/versions.json is composed from individual version.json files when it is corrupt" { + tmpdir=$(mktemp -d) + run steampipe plugin install net chaos --install-dir $tmpdir + assert_success + + vFile="$tmpdir/plugins/versions.json" + fileContent=$(cat $vFile) + + # remove global version file + echo "badline to corrupt versions.json" >> $vFile + + # run steampipe again so that the plugin version files get backfilled + run steampipe plugin list --install-dir $tmpdir + + [ ! -f $vFile ] && fail "could not find $vFile" + + assert_equal "$(cat $vFile)" "$fileContent" + + rm -rf $tmpdir +} + +@test "verify that composition of global plugin/versions.json works when an individual version.json file is corrupt" { + tmpdir=$(mktemp -d) + run steampipe plugin install net chaos --install-dir $tmpdir + assert_success + + vFile="$tmpdir/plugins/versions.json" + vFile1="$tmpdir/plugins/hub.steampipe.io/plugins/turbot/net@latest/version.json" + + # corrupt a version file + echo "bad line to corrupt" >> $vFile1 + + # remove global file + rm -f $vFile + + # run steampipe again so that the plugin version files get backfilled + run steampipe plugin list --install-dir $tmpdir + + # verify that global file got created + [ ! -f $vFile ] && fail "could not find $vFile" + + rm -rf $tmpdir +} + +@test "verify that plugin installed from registry are marked as 'local' when the modtime of the binary is after the install time" { + tmpdir=$(mktemp -d) + run steampipe plugin install net chaos --install-dir $tmpdir + assert_success + + # wait for a couple of seconds + sleep 2 + + # touch one of the plugin binaries + touch $tmpdir/plugins/hub.steampipe.io/plugins/turbot/net@latest/steampipe-plugin-net.plugin + + # run steampipe again so that the plugin version files get backfilled + version=$(steampipe plugin list --install-dir $tmpdir --output json | jq '.installed' | jq '. | map(select(.name | contains("net@latest")))' | jq '.[0].version') + + # assert + assert_equal "$version" '"local"' + + rm -rf $tmpdir +} + @test "cleanup" { rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_agg.spc run steampipe plugin uninstall steampipe @@ -543,7 +698,7 @@ load "$LIB_BATS_SUPPORT/load.bash" } function setup_file() { - export BATS_TEST_TIMEOUT=60 + export BATS_TEST_TIMEOUT=120 echo "# setup_file()">&3 }