add selectable updater sets

this commit adds the ability to specify individual updater sets
via libvuln opts.

some config values are removed as well as they may have been exposed
too soon.

cmd/libvulnhttp/main.go

@@ -99,7 +99,7 @@ func confToLibvulnOpts(conf Config) *libvuln.Opts {
 		ConnString:  conf.ConnString,
 		MaxConnPool: int32(conf.MaxConnPool),
 		Migrations:  true,
-		Run:         conf.Run,
+		UpdaterSets: nil,
 	}
 
 	return opts

debian/matcher_integration_test.go

@@ -33,7 +33,7 @@ func Test_Matcher_Integration(t *testing.T) {
 	m := &Matcher{}
 	// seed the test vulnstore with CVE data
 	deb := NewUpdater(Buster)
-	up := updater.New(&updater.Opts{
+	up := updater.NewController(&updater.Opts{
 		Name:    "test-debian-buster",
 		Updater: deb,
 		Store:   store,

internal/updater/controller.go

@@ -36,7 +36,7 @@ type Opts struct {
 }
 
 // New is a constructor for an Controller
-func New(opts *Opts) *Controller {
+func NewController(opts *Opts) *Controller {
 	return &Controller{
 		Opts: opts,
 	}

libvuln/driver/updaterset.go

@@ -63,10 +63,13 @@ func (s *UpdaterSet) Merge(set UpdaterSet) error {
 	return nil
 }
 
-// Updaters returns a map of updaters keyed by
-// their name.
-func (s *UpdaterSet) Updaters() map[string]Updater {
-	return s.Set
+// Updaters() returns the updaters within the set as slice.
+func (s *UpdaterSet) Updaters() []Updater {
+	u := make([]Updater, 0, len(s.Set))
+	for _, v := range s.Set {
+		u = append(u, v)
+	}
+	return u
 }
 
 // RegexFilter will remove any updaters from the set

libvuln/init.go

@@ -21,21 +21,14 @@ import (
 // initUpdaters provides initial burst control to not launch too many updaters at once.
 // returns any errors on eC and returns a CaneclFunc on dC to stop all updaters
 func initUpdaters(ctx context.Context, opts *Opts, db *sqlx.DB, store vulnstore.Updater, dC chan context.CancelFunc, eC chan error) {
-	// just to be defensive
-	err := opts.Parse()
-	if err != nil {
-		eC <- err
-		return
-	}
-
 	controllers := map[string]*updater.Controller{}
 
-	for _, u := range opts.Updaters.Set {
+	for _, u := range opts.Updaters {
 		if _, ok := controllers[u.Name()]; ok {
 			eC <- fmt.Errorf("duplicate updater found in UpdaterFactory. all names must be unique: %s", u.Name())
 			return
 		}
-		controllers[u.Name()] = updater.New(&updater.Opts{
+		controllers[u.Name()] = updater.NewController(&updater.Opts{
 			Updater:       u,
 			Store:         store,
 			Name:          u.Name(),
@@ -46,7 +39,7 @@ func initUpdaters(ctx context.Context, opts *Opts, db *sqlx.DB, store vulnstore.
 	}
 
 	// limit initial concurrent updates
-	cc := make(chan struct{}, opts.UpdaterInitConcurrency)
+	cc := make(chan struct{}, DefaultUpdaterInitConcurrency)
 
 	var wg sync.WaitGroup
 	wg.Add(len(controllers))

libvuln/libvuln.go

@@ -31,20 +31,24 @@ func New(ctx context.Context, opts *Opts) (*Libvuln, error) {
 		Str("component", "libvuln/New").
 		Logger()
 	ctx = log.WithContext(ctx)
-	err := opts.Parse()
+
+	err := opts.parse(ctx)
 	if err != nil {
 		return nil, err
 	}
+
 	log.Info().
 		Int32("count", opts.MaxConnPool).
 		Msg("initializing store")
+
 	db, vulnstore, err := initStore(ctx, opts)
 	if err != nil {
 		return nil, err
 	}
+
+	// block on updater initialization.
 	eC := make(chan error, 1024)
 	dC := make(chan context.CancelFunc, 1)
-	// block on updater initialization.
 	log.Info().Msg("updater initialization start")
 	go initUpdaters(ctx, opts, db, vulnstore, dC, eC)
 	killUpdaters := <-dC

libvuln/opts.go

@@ -1,6 +1,7 @@
 package libvuln
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -16,29 +17,70 @@ import (
 const (
 	DefaultUpdateInterval         = 30 * time.Minute
 	DefaultUpdaterInitConcurrency = 10
-	DefaultMaxConnPool            = 100
+	DefaultMaxConnPool            = 50
 )
 
 type Opts struct {
-	// the maximum size of the connection pool used by the database
+	// The maximum number of database connections in the
+	// connection pool.
 	MaxConnPool int32
-	// the connectiong string to the above data store implementation
+	// A connection string to the database Lbvuln will use.
 	ConnString string
-	// the interval in minutes which updaters will update the vulnstore
+	// An interval on which Libvuln will check for new security
+	// database updates.
 	UpdateInterval time.Duration
-	// number of updaters ran in parallel while libvuln initializes. use this to tune io/cpu on library start when using many updaters
-	UpdaterInitConcurrency int
-	// set to true to have libindex check and potentially run migrations
+	// Determines if Livuln will manage database migrations
 	Migrations bool
-	// returns the matchers to be used during libvuln runtime
+	// A pointer to a slice of strings representing which
+	// updaters libvuln will create.
+	//
+	// If the pointer is nil all default UpdaterSets
+	// will be used
+	//
+	// If the pointe points to an empty array no UpdaterSets
+	// will run.
+	//
+	// The following sets are supported:
+	// "alpine"
+	// "aws"
+	// "debian"
+	// "oracle"
+	// "photon"
+	// "pyupio"
+	// "rhel"
+	// "suse"
+	// "ubuntu"
+	UpdaterSets *[]string
+	// A list of out-of-tree updaters to run.
+	//
+	// This list will be merged with any defined UpdaterSets.
+	//
+	// If you desire no updaters to run do not add an updater
+	// into this slice.
+	Updaters []driver.Updater
+	// A list of out-of-tree matchers you'd like libvuln to
+	// use.
+	//
+	// This list will me merged with the default matchers.
 	Matchers []driver.Matcher
-	// returns the updaters to run on an interval
-	Updaters driver.UpdaterSet
-	// a regex string to filter running updaters by
-	Run string
 }
 
-func (o *Opts) Parse() error {
+// defaultMacheter is a variable containing
+// all the matchers libvuln will use to match
+// index records to vulnerabilities.
+var defaultMatchers = []driver.Matcher{
+	&alpine.Matcher{},
+	&aws.Matcher{},
+	&debian.Matcher{},
+	&python.Matcher{},
+	&ubuntu.Matcher{},
+	&rhel.Matcher{},
+}
+
+// parse is an internal method for constructing
+// the necessary Updaters and Matchers for Libvuln
+// usage
+func (o *Opts) parse(ctx context.Context) error {
 	// required
 	if o.ConnString == "" {
 		return fmt.Errorf("no connection string provided")
@@ -48,38 +90,20 @@ func (o *Opts) Parse() error {
 	if o.UpdateInterval == 0 || o.UpdateInterval < time.Minute {
 		o.UpdateInterval = DefaultUpdateInterval
 	}
-	if o.UpdaterInitConcurrency == 0 {
-		o.UpdaterInitConcurrency = DefaultUpdaterInitConcurrency
-	}
 	if o.MaxConnPool == 0 {
 		o.MaxConnPool = DefaultMaxConnPool
 	}
-	if len(o.Matchers) == 0 {
-		o.Matchers = []driver.Matcher{
-			&debian.Matcher{},
-			&ubuntu.Matcher{},
-			&alpine.Matcher{},
-			&aws.Matcher{},
-			&rhel.Matcher{},
-			&python.Matcher{},
-		}
-	}
-	if len(o.Updaters.Set) == 0 {
-		var err error
-		o.Updaters, err = updaterSets()
-		if err != nil {
-			return fmt.Errorf("failed to create default set of updaters: %w", err)
-		}
-	}
 
-	// filter out updaters if regex was passed
-	if o.Run != "" {
-		var err error
-		o.Updaters.RegexFilter(o.Run)
-		if err != nil {
-			return fmt.Errorf("regex filtering of updaters failed: %w", err)
-		}
+	// merge default matchers with any out-of-tree specified
+	o.Matchers = append(o.Matchers, defaultMatchers...)
+
+	// determine which updaters should be populated
+	set, err := updaterSets(ctx, o.UpdaterSets)
+	if err != nil {
+		return fmt.Errorf("failed to create updater sets: %v", err)
 	}
+	// merge determined updaters with any out-of-tree updaters
+	o.Updaters = append(o.Updaters, set.Updaters()...)
 
 	return nil
 }