diff --git a/api/v2_vulnerabilities.go b/api/v2_vulnerabilities.go index 300755163..bf6eafe07 100644 --- a/api/v2_vulnerabilities.go +++ b/api/v2_vulnerabilities.go @@ -100,6 +100,7 @@ func (svc *v2ContainerVulnerabilityService) SearchAllPages(filters SearchFilter) } break } + fmt.Println("finished all length: ", len(all)) response.ResetPaging() response.Data = all diff --git a/cli/cmd/vuln_container.go b/cli/cmd/vuln_container.go index 7b6e45991..f7de6b98d 100644 --- a/cli/cmd/vuln_container.go +++ b/cli/cmd/vuln_container.go @@ -19,9 +19,13 @@ package cmd import ( + "github.com/lacework/go-sdk/v2/api" "github.com/lacework/go-sdk/v2/internal/array" "github.com/pkg/errors" flag "github.com/spf13/pflag" + "sort" + "strings" + "time" ) func init() { @@ -120,7 +124,9 @@ func setPollFlag(cmds ...*flag.FlagSet) { } } -func getContainerRegistries() ([]string, error) { +// Returns registry domains from the customer's container registry integrations +// This will not return registries scanned by inline and proxy scanner integrations +func getPlatformScannerIntegrationContainerRegistries() ([]string, error) { var ( registries = make([]string, 0) regsIntegrations, err = cli.LwApi.V2.ContainerRegistries.List() @@ -140,3 +146,64 @@ func getContainerRegistries() ([]string, error) { return registries, nil } + +func getContainerRegistries() ([]string, error) { + var ( + registries = make([]string, 0) + ) + // Get registry domains from container registry integrations + registries, err := getPlatformScannerIntegrationContainerRegistries() + if err != nil { + cli.Log.Debugw("error trying to retrieve configured registries", "error", err) + } + + // Build filter to fetch all container evaluations in the last 7 days. 7 days is an api limitation + // This is required to find registries that are only scanned by proxy and inline scanner since these + // integrations don't include the registry domain. + end := time.Now() + start := end.Add(-24 * 7 * time.Hour) + filter := api.SearchFilter{ + TimeFilter: &api.TimeFilter{ + StartTime: &start, + EndTime: &end, + }, + Returns: []string{"evalCtx"}, + Filters: []api.Filter{ + { + Expression: "not_in", + Field: "evalCtx.image_info.registry", + Values: registries, + }, + }, + } + registryMap := map[string]bool{} + for { + response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(filter) + if err != nil { + return registries, errors.Wrap(err, "unable to search for container registries") + } + // Use a map to get distinct registries from response + for _, ctr := range response.Data { + registryMap[strings.TrimSpace(ctr.EvalCtx.ImageInfo.Registry)] = true + } + // Convert map to slice + for reg := range registryMap { + registries = append(registries, reg) + } + // If we hit the 500,000 total row limit from the api, create a filter for registries we haven't seen yet + if len(response.Data) == 500000 { + filter.Filters = []api.Filter{ + { + Expression: "not_in", + Field: "evalCtx.image_info.registry", + Values: registries, + }, + } + continue + } + break + } + // Sort registries alphabetically + sort.Strings(registries) + return registries, nil +} diff --git a/cli/cmd/vuln_container_list_assessments.go b/cli/cmd/vuln_container_list_assessments.go index 91e683b1f..17acf9a67 100644 --- a/cli/cmd/vuln_container_list_assessments.go +++ b/cli/cmd/vuln_container_list_assessments.go @@ -210,7 +210,7 @@ func applyVulnCtrFilters(assessments []vulnerabilityAssessmentSummary) (filtered // all registries, repositories, local scanners, etc. (This is a memory // utilization improvement) // 2. If no filter by registries and/or repos, then fetch all data from all -// registries and all local scanners, we purposely split them in two search +// registries and scanner types, we purposely split them into four search // requests since there could be so much data that we get to the 500,000 rows // if data and we could potentially miss some information // 3. Either 1) or 2) will generate a tree of unique container vulnerability @@ -253,52 +253,67 @@ func listVulnCtrAssessments( } if len(filter.Filters) == 0 { - // if not, then we need to fetch information from 1) all - // container registries and 2) local scanners in two separate - // searches since platform scanners might have way too much - // data which may cause losing the local scanners data - // - // find all container registries + // If not, fetch all container assessments. We need to break up the api requests by scanner type + // This is to avoid the api 500,000 total row limit. + + // TODO: we currently only have the eval_details API, which returns 1 row per package/vulnerability + // Vuln already has a table that comes pre-aggregated, which would vastly improve performance and + // reduce the number of rows returned by teh API + // cli.StartProgress("Fetching container registries...") - registries, err := getContainerRegistries() // cli.StopProgress() + filter.Filters = []api.Filter{ + { + Expression: "eq", + Field: "evalCtx.request_source", + Value: "PLATFORM_SCANNER", + }, + } + response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter) + fmt.Println("platform size: ", len(response.Data)) if err != nil { - return nil, err + return assessments, errors.Wrap(err, "unable to search for container assessments") } - cli.Log.Infow("container registries found", "count", len(registries)) - - if len(registries) != 0 { - // 1) search for all assessments from configured container registries - filter.Filters = []api.Filter{ - { - Expression: "in", - Field: "evalCtx.image_info.registry", - Values: registries, - }, - } - response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter) - if err != nil { - return assessments, errors.Wrap(err, "unable to search for container assessments") - } - - treeOfContainerVuln.ParseData(response.Data) - - // 2) search for assessments from local scanners, that is, non container registries - filter.Filters = []api.Filter{ - { - Expression: "not_in", - Field: "evalCtx.image_info.registry", - Values: registries, - }, - } - } else { - response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter) - if err != nil { - return assessments, errors.Wrap(err, "unable to search for container assessments") - } - - treeOfContainerVuln.ParseData(response.Data) + treeOfContainerVuln.ParseData(response.Data) + filter.Filters = []api.Filter{ + { + Expression: "eq", + Field: "evalCtx.request_source", + Value: "INLINE_SCANNER", + }, } + response, err = cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter) + fmt.Println("inline size: ", len(response.Data)) + if err != nil { + return assessments, errors.Wrap(err, "unable to search for container assessments") + } + treeOfContainerVuln.ParseData(response.Data) + filter.Filters = []api.Filter{ + { + Expression: "eq", + Field: "evalCtx.request_source", + Value: "PROXY_SCANNER", + }, + } + response, err = cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter) + fmt.Println("proxy size: ", len(response.Data)) + if err != nil { + return assessments, errors.Wrap(err, "unable to search for container assessments") + } + treeOfContainerVuln.ParseData(response.Data) + filter.Filters = []api.Filter{ + { + Expression: "eq", + Field: "evalCtx.request_source", + Value: "AGENTLESS_SCANNER", + }, + } + response, err = cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter) + fmt.Println("agentless size: ", len(response.Data)) + if err != nil { + return assessments, errors.Wrap(err, "unable to search for container assessments") + } + treeOfContainerVuln.ParseData(response.Data) } else { response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter) if err != nil { diff --git a/cli/cmd/vuln_container_scan.go b/cli/cmd/vuln_container_scan.go index e6ac30f97..f668ad678 100644 --- a/cli/cmd/vuln_container_scan.go +++ b/cli/cmd/vuln_container_scan.go @@ -109,7 +109,7 @@ func userFriendlyErrorForOnDemandCtrVulnScan(err error, registry, repo, tag stri "Could not find vulnerability integrations", ) { - registries, errReg := getContainerRegistries() + registries, errReg := getPlatformScannerIntegrationContainerRegistries() if errReg != nil { cli.Log.Debugw("error trying to retrieve configured registries", "error", errReg) return errors.Errorf("container registry '%s' not found", registry) diff --git a/go.mod b/go.mod index 6a11bf096..e3e843f48 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/lacework/go-sdk/v2 -go 1.24 +go 1.24.0 require ( aead.dev/minisign v0.2.0