SBOM ingestion

cli/docs/enrich/help.go

@@ -0,0 +1,13 @@
+package enrich
+
+import (
+	"github.com/jfrog/jfrog-cli-core/v2/plugins/components"
+)
+
+func GetDescription() string {
+	return "Enrich sbom format JSON located on the local file-system with Xray."
+}
+
+func GetArguments() []components.Argument {
+	return []components.Argument{{Name: "Specific file path", Description: `Specifies the local file system path of the JSON to be scanned.`}}
+}

cli/docs/flags.go

@@ -22,6 +22,7 @@ const (
 	Audit                = "audit"
 	CurationAudit        = "curation-audit"
 	GitCountContributors = "count-contributors"
+	Enrich               = "sbom-enrich"
 
 	// TODO: Deprecated commands (remove at next CLI major version)
 	AuditMvn    = "audit-maven"
@@ -134,6 +135,9 @@ var commandFlags = map[string][]string{
 		url, user, password, accessToken, ServerId, SpecFlag, Threads, scanRecursive, scanRegexp, scanAnt,
 		Project, Watches, RepoPath, Licenses, OutputFormat, Fail, ExtendedTable, BypassArchiveLimits, MinSeverity, FixableOnly,
 	},
+	Enrich: {
+		url, user, password, accessToken, ServerId, SpecFlag, Threads,
+	},
 	BuildScan: {
 		url, user, password, accessToken, ServerId, Project, Vuln, OutputFormat, Fail, ExtendedTable, Rescan,
 	},

cli/scancommands.go

@@ -2,6 +2,8 @@ package cli
 
 import (
 	"fmt"
+	enrichDocs "github.com/jfrog/jfrog-cli-security/cli/docs/enrich"
+	"github.com/jfrog/jfrog-cli-security/commands/enrich"
 	"os"
 	"strings"
 
@@ -51,6 +53,14 @@ func getAuditAndScansCommands() []components.Command {
 			Category:    auditScanCategory,
 			Action:      ScanCmd,
 		},
+		{
+			Name:        "sbom-enrich",
+			Aliases:     []string{"se"},
+			Flags:       flags.GetCommandFlags(flags.Enrich),
+			Description: enrichDocs.GetDescription(),
+			Arguments:   enrichDocs.GetArguments(),
+			Action:      EnrichCmd,
+		},
 		{
 			Name:        "build-scan",
 			Aliases:     []string{"bs"},
@@ -154,6 +164,32 @@ func getAuditAndScansCommands() []components.Command {
 	}
 }
 
+func EnrichCmd(c *components.Context) error {
+	if len(c.Arguments) == 0 {
+		return pluginsCommon.PrintHelpAndReturnError("providing a file path argument is mandatory", c)
+	}
+	serverDetails, err := createServerDetailsWithConfigOffer(c)
+	if err != nil {
+		return err
+	}
+	if err = validateXrayContext(c, serverDetails); err != nil {
+		return err
+	}
+	specFile := createDefaultScanSpec(c, addTrailingSlashToRepoPathIfNeeded(c))
+	if err = spec.ValidateSpec(specFile.Files, false, false); err != nil {
+		return err
+	}
+	threads, err := pluginsCommon.GetThreadsCount(c)
+	if err != nil {
+		return err
+	}
+	EnrichCmd := enrich.NewEnrichCommand().
+		SetServerDetails(serverDetails).
+		SetThreads(threads).
+		SetSpec(specFile)
+	return commandsCommon.Exec(EnrichCmd)
+}
+
 func ScanCmd(c *components.Context) error {
 	if len(c.Arguments) == 0 && !c.IsFlagSet(flags.SpecFlag) {
 		return pluginsCommon.PrintHelpAndReturnError("providing either a <source pattern> argument or the 'spec' option is mandatory", c)

commands/enrich/enrich.go

@@ -0,0 +1,275 @@
+package enrich
+
+import (
+	"encoding/xml"
+	"errors"
+	"github.com/jfrog/gofrog/parallel"
+	"github.com/jfrog/jfrog-cli-core/v2/common/spec"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+	"github.com/jfrog/jfrog-cli-security/commands/enrich/enrichgraph"
+	"github.com/jfrog/jfrog-cli-security/formats"
+	"github.com/jfrog/jfrog-cli-security/utils"
+	xrutils "github.com/jfrog/jfrog-cli-security/utils"
+	"github.com/jfrog/jfrog-cli-security/utils/xray"
+	"github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns"
+	clientutils "github.com/jfrog/jfrog-client-go/utils"
+	"github.com/jfrog/jfrog-client-go/utils/errorutils"
+	ioUtils "github.com/jfrog/jfrog-client-go/utils/io"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
+	"github.com/jfrog/jfrog-client-go/utils/log"
+	"github.com/jfrog/jfrog-client-go/xray/services"
+	"os"
+	"os/exec"
+)
+
+type FileContext func(string) parallel.TaskFunc
+type indexFileHandlerFunc func(file string)
+
+type ScanInfo struct {
+	Target string
+	Result *services.ScanResponse
+}
+
+type EnrichCommand struct {
+	serverDetails *config.ServerDetails
+	spec          *spec.SpecFiles
+	threads       int
+	progress      ioUtils.ProgressMgr
+}
+
+func (enrichCmd *EnrichCommand) SetProgress(progress ioUtils.ProgressMgr) {
+	enrichCmd.progress = progress
+}
+
+func (enrichCmd *EnrichCommand) SetThreads(threads int) *EnrichCommand {
+	enrichCmd.threads = threads
+	return enrichCmd
+}
+
+func (enrichCmd *EnrichCommand) SetServerDetails(server *config.ServerDetails) *EnrichCommand {
+	enrichCmd.serverDetails = server
+	return enrichCmd
+}
+
+func (enrichCmd *EnrichCommand) SetSpec(spec *spec.SpecFiles) *EnrichCommand {
+	enrichCmd.spec = spec
+	return enrichCmd
+}
+
+func (enrichCmd *EnrichCommand) ServerDetails() (*config.ServerDetails, error) {
+	return enrichCmd.serverDetails, nil
+}
+
+func isXML(scaResults []*utils.ScaScanResult) (bool, error) {
+	if len(scaResults) == 0 {
+		return false, errors.New("unable to retrieve file")
+	}
+	fileName := scaResults[0].Target
+	var x interface{}
+	content, err := os.ReadFile(fileName)
+	if err != nil {
+		return false, err
+	}
+	return xml.Unmarshal(content, &x) == nil, nil
+}
+
+func (enrichCmd *EnrichCommand) Run() (err error) {
+	defer func() {
+		if err != nil {
+			var e *exec.ExitError
+			if errors.As(err, &e) {
+				if e.ExitCode() != coreutils.ExitCodeVulnerableBuild.Code {
+					err = errors.New("Enrich command failed. " + err.Error())
+				}
+			}
+		}
+	}()
+	_, xrayVersion, err := xray.CreateXrayServiceManagerAndGetVersion(enrichCmd.serverDetails)
+	if err != nil {
+		return err
+	}
+
+	// Validate Xray minimum version for graph scan command
+	err = clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, enrichgraph.EnrichMinimumVersionXray)
+	if err != nil {
+		return err
+	}
+
+	log.Info("JFrog Xray version is:", xrayVersion)
+
+	threads := 1
+	if enrichCmd.threads > 1 {
+		threads = enrichCmd.threads
+	}
+
+	// resultsArr is a two-dimensional array. Each array in it contains a list of ScanResponses that were requested and collected by a specific thread.
+	resultsArr := make([][]*ScanInfo, threads)
+	fileProducerConsumer := parallel.NewRunner(enrichCmd.threads, 20000, false)
+	fileProducerErrors := make([][]formats.SimpleJsonError, threads)
+	indexedFileProducerConsumer := parallel.NewRunner(enrichCmd.threads, 20000, false)
+	indexedFileProducerErrors := make([][]formats.SimpleJsonError, threads)
+	fileCollectingErrorsQueue := clientutils.NewErrorsQueue(1)
+	// Start walking on the filesystem to "produce" files that match the given pattern
+	// while the consumer uses the indexer to index those files.
+	enrichCmd.prepareScanTasks(fileProducerConsumer, indexedFileProducerConsumer, resultsArr, indexedFileProducerErrors, fileCollectingErrorsQueue, xrayVersion)
+	enrichCmd.performScanTasks(fileProducerConsumer, indexedFileProducerConsumer)
+
+	// Handle results
+	var flatResults []*xrutils.ScaScanResult
+	for _, arr := range resultsArr {
+		for _, res := range arr {
+			flatResults = append(flatResults, &xrutils.ScaScanResult{Target: res.Target, XrayResults: []services.ScanResponse{*res.Result}})
+		}
+	}
+	if enrichCmd.progress != nil {
+		if err = enrichCmd.progress.Quit(); err != nil {
+			return err
+		}
+
+	}
+
+	fileCollectingErr := fileCollectingErrorsQueue.GetError()
+	var scanErrors []formats.SimpleJsonError
+	if fileCollectingErr != nil {
+		scanErrors = append(scanErrors, formats.SimpleJsonError{ErrorMessage: fileCollectingErr.Error()})
+	}
+	scanErrors = appendErrorSlice(scanErrors, fileProducerErrors)
+	scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors)
+
+	scanResults := xrutils.NewAuditResults()
+	scanResults.XrayVersion = xrayVersion
+	scanResults.ScaResults = flatResults
+
+	isxml, err := isXML(scanResults.ScaResults)
+	if err != nil {
+		return
+	}
+	if isxml {
+		if err = utils.AppendVulnsToXML(scanResults); err != nil {
+			return
+		}
+	} else {
+		if err = utils.AppendVulnsToJson(scanResults); err != nil {
+			return
+		}
+	}
+
+	if err != nil {
+		return err
+	}
+
+	if len(scanErrors) > 0 {
+		return errorutils.CheckErrorf(scanErrors[0].ErrorMessage)
+	}
+	log.Info("Enrich process completed successfully.")
+	return nil
+}
+
+func NewEnrichCommand() *EnrichCommand {
+	return &EnrichCommand{}
+}
+
+func (enrichCmd *EnrichCommand) CommandName() string {
+	return "xr_enrich"
+}
+
+func (enrichCmd *EnrichCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, indexedFileErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) {
+	go func() {
+		defer fileProducer.Done()
+		// Iterate over file-spec groups and produce indexing tasks.
+		// When encountering an error, log and move to next group.
+		specFiles := enrichCmd.spec.Files
+		artifactHandlerFunc := enrichCmd.createIndexerHandlerFunc(indexedFileProducer, resultsArr, indexedFileErrors, xrayVersion)
+		taskHandler := getAddTaskToProducerFunc(fileProducer, artifactHandlerFunc)
+
+		err := FileForEnriching(specFiles[0], taskHandler)
+		if err != nil {
+			log.Error(err)
+			fileCollectingErrorsQueue.AddError(err)
+		}
+	}()
+}
+
+func (enrichCmd *EnrichCommand) createIndexerHandlerFunc(indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, indexedFileErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext {
+	return func(filePath string) parallel.TaskFunc {
+		return func(threadId int) (err error) {
+			// Add a new task to the second producer/consumer
+			// which will send the indexed binary to Xray and then will store the received result.
+			taskFunc := func(threadId int) (err error) {
+				fileContent, err := os.ReadFile(filePath)
+				if err != nil {
+					return err
+				}
+				params := &services.XrayGraphImportParams{
+					SBOMInput: fileContent,
+					ScanType:  services.Binary,
+				}
+				importGraphParams := enrichgraph.NewEnrichGraphParams().
+					SetServerDetails(enrichCmd.serverDetails).
+					SetXrayGraphScanParams(params).
+					SetXrayVersion(xrayVersion)
+				xrayManager, err := xray.CreateXrayServiceManager(importGraphParams.ServerDetails())
+				if err != nil {
+					return err
+				}
+				scanResults, err := enrichgraph.RunImportGraphAndGetResults(importGraphParams, xrayManager)
+				if err != nil {
+					indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
+					return
+				}
+				resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults})
+				return
+			}
+
+			_, _ = indexedFileProducer.AddTask(taskFunc)
+			return
+		}
+	}
+}
+
+func getAddTaskToProducerFunc(producer parallel.Runner, fileHandlerFunc FileContext) indexFileHandlerFunc {
+	return func(filePath string) {
+		taskFunc := fileHandlerFunc(filePath)
+		_, _ = producer.AddTask(taskFunc)
+	}
+}
+
+func (enrichCmd *EnrichCommand) performScanTasks(fileConsumer parallel.Runner, indexedFileConsumer parallel.Runner) {
+	go func() {
+		// Blocking until consuming is finished.
+		fileConsumer.Run()
+		// After all files have been indexed, The second producer notifies that no more tasks will be produced.
+		indexedFileConsumer.Done()
+	}()
+	// Blocking until consuming is finished.
+	indexedFileConsumer.Run()
+}
+
+func FileForEnriching(fileData spec.File, dataHandlerFunc indexFileHandlerFunc) error {
+	fileData.Pattern = clientutils.ReplaceTildeWithUserHome(fileData.Pattern)
+	patternType := fileData.GetPatternType()
+	rootPath, err := fspatterns.GetRootPath(fileData.Pattern, fileData.Target, "", patternType, false)
+	if err != nil {
+		return err
+	}
+
+	isDir, err := fileutils.IsDirExists(rootPath, false)
+	if err != nil {
+		return err
+	}
+
+	// path should be a single file
+	if !isDir {
+		dataHandlerFunc(rootPath)
+		return nil
+	}
+	return errors.New("directory instead of a single file")
+}
+
+func appendErrorSlice(scanErrors []formats.SimpleJsonError, errorsToAdd [][]formats.SimpleJsonError) []formats.SimpleJsonError {
+	for _, errorSlice := range errorsToAdd {
+		scanErrors = append(scanErrors, errorSlice...)
+	}
+	return scanErrors
+}

commands/enrich/enrichgraph/enrichgraph.go

@@ -0,0 +1,23 @@
+package enrichgraph
+
+import (
+	"github.com/jfrog/jfrog-client-go/xray"
+	"github.com/jfrog/jfrog-client-go/xray/services"
+)
+
+const (
+	EnrichMinimumVersionXray = "3.90"
+)
+
+func RunImportGraphAndGetResults(params *EnrichGraphParams, xrayManager *xray.XrayServicesManager) (*services.ScanResponse, error) {
+	scanId, err := xrayManager.ImportGraph(*params.xrayGraphImportParams)
+	if err != nil {
+		return nil, err
+	}
+
+	scanResult, err := xrayManager.GetImportGraphResults(scanId)
+	if err != nil {
+		return nil, err
+	}
+	return scanResult, nil
+}