fix(LINK-2046: stop using integrations for get registries

api/v2_vulnerabilities.go

@@ -100,6 +100,7 @@ func (svc *v2ContainerVulnerabilityService) SearchAllPages(filters SearchFilter)
 		}
 		break
 	}
+	fmt.Println("finished all length: ", len(all))
 
 	response.ResetPaging()
 	response.Data = all

cli/cmd/vuln_container.go

@@ -19,9 +19,13 @@
 package cmd
 
 import (
+	"github.com/lacework/go-sdk/v2/api"
 	"github.com/lacework/go-sdk/v2/internal/array"
 	"github.com/pkg/errors"
 	flag "github.com/spf13/pflag"
+	"sort"
+	"strings"
+	"time"
 )
 
 func init() {
@@ -120,7 +124,9 @@
 	}
 }
 
-func getContainerRegistries() ([]string, error) {
+// Returns registry domains from the customer's container registry integrations
+// This will not return registries scanned by inline and proxy scanner integrations
+func getPlatformScannerIntegrationContainerRegistries() ([]string, error) {
 	var (
 		registries            = make([]string, 0)
 		regsIntegrations, err = cli.LwApi.V2.ContainerRegistries.List()
@@ -140,3 +146,64 @@
 
 	return registries, nil
 }
+
+func getContainerRegistries() ([]string, error) {
+	var (
+		registries = make([]string, 0)
+	)
+	// Get registry domains from container registry integrations
+	registries, err := getPlatformScannerIntegrationContainerRegistries()
+	if err != nil {
+		cli.Log.Debugw("error trying to retrieve configured registries", "error", err)
+	}
+
+	// Build filter to fetch all container evaluations in the last 7 days. 7 days is an api limitation
+	// This is required to find registries that are only scanned by proxy and inline scanner since these
+	// integrations don't include the registry domain.
+	end := time.Now()
+	start := end.Add(-24 * 7 * time.Hour)
+	filter := api.SearchFilter{
+		TimeFilter: &api.TimeFilter{
+			StartTime: &start,
+			EndTime:   &end,
+		},
+		Returns: []string{"evalCtx"},
+		Filters: []api.Filter{
+			{
+				Expression: "not_in",
+				Field:      "evalCtx.image_info.registry",
+				Values:     registries,
+			},
+		},
+	}
+	registryMap := map[string]bool{}
+	for {
+		response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(filter)
+		if err != nil {
+			return registries, errors.Wrap(err, "unable to search for container registries")
+		}
+		// Use a map to get distinct registries from response
+		for _, ctr := range response.Data {
+			registryMap[strings.TrimSpace(ctr.EvalCtx.ImageInfo.Registry)] = true
+		}
+		// Convert map to slice
+		for reg := range registryMap {
+			registries = append(registries, reg)
+		}
+		// If we hit the 500,000 total row limit from the api, create a filter for registries we haven't seen yet
+		if len(response.Data) == 500000 {
+			filter.Filters = []api.Filter{
+				{
+					Expression: "not_in",
+					Field:      "evalCtx.image_info.registry",
+					Values:     registries,
+				},
+			}
+			continue
+		}
+		break
+	}
+	// Sort registries alphabetically
+	sort.Strings(registries)
+	return registries, nil
+}

cli/cmd/vuln_container_list_assessments.go

@@ -210,7 +210,7 @@ func applyVulnCtrFilters(assessments []vulnerabilityAssessmentSummary) (filtered
 //     all registries, repositories, local scanners, etc. (This is a memory
 //     utilization improvement)
 //  2. If no filter by registries and/or repos, then fetch all data from all
-//     registries and all local scanners, we purposely split them in two search
+//     registries and scanner types, we purposely split them into four search
 //     requests since there could be so much data that we get to the 500,000 rows
 //     if data and we could potentially miss some information
 //  3. Either 1) or 2) will generate a tree of unique container vulnerability
@@ -253,52 +253,67 @@ func listVulnCtrAssessments(
 	}
 
 	if len(filter.Filters) == 0 {
-		// if not, then we need to fetch information from 1) all
-		// container registries and 2) local scanners in two separate
-		// searches since platform scanners might have way too much
-		// data which may cause losing the local scanners data
-		//
-		// find all container registries
+		// If not, fetch all container assessments. We need to break up the api requests by scanner type
+		// This is to avoid the api 500,000 total row limit.
+
+		// TODO: we currently only have the eval_details API, which returns 1 row per package/vulnerability
+		// Vuln already has a table that comes pre-aggregated, which would vastly improve performance and
+		// reduce the number of rows returned by teh API
+
 		// cli.StartProgress("Fetching container registries...")
-		registries, err := getContainerRegistries()
 		// cli.StopProgress()
+		filter.Filters = []api.Filter{
+			{
+				Expression: "eq",
+				Field:      "evalCtx.request_source",
+				Value:      "PLATFORM_SCANNER",
+			},
+		}
+		response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter)
+		fmt.Println("platform size: ", len(response.Data))
 		if err != nil {
-			return nil, err
+			return assessments, errors.Wrap(err, "unable to search for container assessments")
 		}
-		cli.Log.Infow("container registries found", "count", len(registries))
-
-		if len(registries) != 0 {
-			// 1) search for all assessments from configured container registries
-			filter.Filters = []api.Filter{
-				{
-					Expression: "in",
-					Field:      "evalCtx.image_info.registry",
-					Values:     registries,
-				},
-			}
-			response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter)
-			if err != nil {
-				return assessments, errors.Wrap(err, "unable to search for container assessments")
-			}
-
-			treeOfContainerVuln.ParseData(response.Data)
-
-			// 2) search for assessments from local scanners, that is, non container registries
-			filter.Filters = []api.Filter{
-				{
-					Expression: "not_in",
-					Field:      "evalCtx.image_info.registry",
-					Values:     registries,
-				},
-			}
-		} else {
-			response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter)
-			if err != nil {
-				return assessments, errors.Wrap(err, "unable to search for container assessments")
-			}
-
-			treeOfContainerVuln.ParseData(response.Data)
+		treeOfContainerVuln.ParseData(response.Data)
+		filter.Filters = []api.Filter{
+			{
+				Expression: "eq",
+				Field:      "evalCtx.request_source",
+				Value:      "INLINE_SCANNER",
+			},
 		}
+		response, err = cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter)
+		fmt.Println("inline size: ", len(response.Data))
+		if err != nil {
+			return assessments, errors.Wrap(err, "unable to search for container assessments")
+		}
+		treeOfContainerVuln.ParseData(response.Data)
+		filter.Filters = []api.Filter{
+			{
+				Expression: "eq",
+				Field:      "evalCtx.request_source",
+				Value:      "PROXY_SCANNER",
+			},
+		}
+		response, err = cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter)
+		fmt.Println("proxy size: ", len(response.Data))
+		if err != nil {
+			return assessments, errors.Wrap(err, "unable to search for container assessments")
+		}
+		treeOfContainerVuln.ParseData(response.Data)
+		filter.Filters = []api.Filter{
+			{
+				Expression: "eq",
+				Field:      "evalCtx.request_source",
+				Value:      "AGENTLESS_SCANNER",
+			},
+		}
+		response, err = cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter)
+		fmt.Println("agentless size: ", len(response.Data))
+		if err != nil {
+			return assessments, errors.Wrap(err, "unable to search for container assessments")
+		}
+		treeOfContainerVuln.ParseData(response.Data)
 	} else {
 		response, err := cli.LwApi.V2.Vulnerabilities.Containers.SearchAllPages(*filter)
 		if err != nil {

cli/cmd/vuln_container_scan.go

@@ -109,7 +109,7 @@ func userFriendlyErrorForOnDemandCtrVulnScan(err error, registry, repo, tag stri
 		"Could not find vulnerability integrations",
 	) {
 
-		registries, errReg := getContainerRegistries()
+		registries, errReg := getPlatformScannerIntegrationContainerRegistries()
 		if errReg != nil {
 			cli.Log.Debugw("error trying to retrieve configured registries", "error", errReg)
 			return errors.Errorf("container registry '%s' not found", registry)

go.mod

@@ -1,6 +1,6 @@
 module github.com/lacework/go-sdk/v2
 
-go 1.24
+go 1.24.0
 
 require (
 	aead.dev/minisign v0.2.0