[Thog-371] Utilize config struct for engine scans

main.go

@@ -17,9 +17,11 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/jpillora/overseer"
 	"github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/version"
-	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
@@ -189,36 +191,78 @@ func run(state overseer.State) {
 		if remote {
 			defer os.RemoveAll(repoPath)
 		}
-		err = e.ScanGit(ctx, repoPath, *gitScanBranch, *gitScanSinceCommit, *gitScanMaxDepth, filter)
-		if err != nil {
-			logrus.WithError(err).Fatal("Failed to scan git.")
+
+		g := func(c *sources.Config) {
+			c.RepoPath = repoPath
+			c.HeadRef = *gitScanBranch
+			c.BaseRef = *gitScanSinceCommit
+			c.MaxDepth = *gitScanMaxDepth
+			c.Filter = filter
+		}
+
+		if err = e.ScanGit(ctx, sources.NewConfig(g)); err != nil {
+			logrus.WithError(err).Fatal("Failed to scan Git.")
 		}
 	case githubScan.FullCommand():
 		if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
 			log.Fatal("You must specify at least one organization or repository.")
 		}
-		err = e.ScanGitHub(ctx, *githubScanEndpoint, *githubScanRepos, *githubScanOrgs, *githubScanToken, *githubIncludeForks, filter, *concurrency, *githubIncludeMembers)
-		if err != nil {
-			logrus.WithError(err).Fatal("Failed to scan git.")
+
+		github := func(c *sources.Config) {
+			c.Endpoint = *githubScanEndpoint
+			c.Repos = *githubScanRepos
+			c.Orgs = *githubScanOrgs
+			c.Token = *githubScanToken
+			c.IncludeForks = *githubIncludeForks
+			c.IncludeMembers = *githubIncludeMembers
+			c.Concurrency =
+				*concurrency
+		}
+
+		if err = e.ScanGitHub(ctx, sources.NewConfig(github)); err != nil {
+			logrus.WithError(err).Fatal("Failed to scan Github.")
 		}
 	case gitlabScan.FullCommand():
-		err := e.ScanGitLab(ctx, *gitlabScanEndpoint, *gitlabScanToken, *gitlabScanRepos)
-		if err != nil {
+		gitlab := func(c *sources.Config) {
+			c.Endpoint = *gitlabScanEndpoint
+			c.Token = *gitlabScanToken
+			c.Repos = *gitlabScanRepos
+		}
+
+		if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan GitLab.")
 		}
 	case filesystemScan.FullCommand():
-		err := e.ScanFileSystem(ctx, *filesystemDirectories)
-		if err != nil {
+		fs := func(c *sources.Config) {
+			c.Directories = *filesystemDirectories
+		}
+
+		if err = e.ScanFileSystem(ctx, sources.NewConfig(fs)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan filesystem")
 		}
 	case s3Scan.FullCommand():
-		err := e.ScanS3(ctx, *s3ScanKey, *s3ScanSecret, *s3ScanCloudEnv, *s3ScanBuckets)
-		if err != nil {
+		s3 := func(c *sources.Config) {
+			c.Key = *s3ScanKey
+			c.Secret = *s3ScanSecret
+			c.Buckets = *s3ScanBuckets
+
+		}
+
+		if err = e.ScanS3(ctx, sources.NewConfig(s3)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan S3.")
 		}
 	case syslogScan.FullCommand():
-		err := e.ScanSyslog(ctx, *syslogAddress, *syslogProtocol, *syslogTLSCert, *syslogTLSKey, *syslogFormat, *concurrency)
-		if err != nil {
+		syslog := func(c *sources.Config) {
+			c.Address = *syslogAddress
+			c.Protocol = *syslogProtocol
+			c.CertPath = *syslogTLSCert
+			c.KeyPath = *syslogTLSKey
+			c.Format = *syslogFormat
+			c.Concurrency = *concurrency
+
+		}
+
+		if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan syslog.")
 		}
 	}

pkg/engine/filesystem.go

@@ -6,15 +6,22 @@ import (
 
 	"github.com/go-errors/errors"
 	"github.com/sirupsen/logrus"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
 )
 
-func (e *Engine) ScanFileSystem(ctx context.Context, directories []string) error {
+// ScanFileSystem scans a given file system.
+func (e *Engine) ScanFileSystem(ctx context.Context, c *sources.Config) error {
+	if c == nil {
+		return errors.New("nil config provided to ScanFileSystem")
+	}
+
 	connection := &sourcespb.Filesystem{
-		Directories: directories,
+		Directories: c.Directories,
 	}
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})

pkg/engine/git.go

@@ -11,34 +11,39 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/sirupsen/logrus"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
-func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string, maxDepth int, filter *common.Filter) error {
+// ScanGit scans any git source.
+func (e *Engine) ScanGit(ctx context.Context, c *sources.Config) error {
+	if c == nil {
+		return errors.New("nil config for ScanGit")
+	}
+
 	logOptions := &gogit.LogOptions{}
 	opts := []git.ScanOption{
-		git.ScanOptionFilter(filter),
+		git.ScanOptionFilter(c.Filter),
 		git.ScanOptionLogOptions(logOptions),
 	}
 
-	repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
+	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
 	if err != nil {
-		return fmt.Errorf("could open repo: %s: %w", repoPath, err)
+		return fmt.Errorf("could open repo: %s: %w", c.RepoPath, err)
 	}
 
 	var baseCommit *object.Commit
-	if len(baseRef) > 0 {
-		baseHash := plumbing.NewHash(baseRef)
-		if !plumbing.IsHash(baseRef) {
-			base, err := git.TryAdditionalBaseRefs(repo, baseRef)
+	if len(c.BaseRef) > 0 {
+		baseHash := plumbing.NewHash(c.BaseRef)
+		if !plumbing.IsHash(c.BaseRef) {
+			base, err := git.TryAdditionalBaseRefs(repo, c.BaseRef)
 			if err != nil {
 				return errors.WrapPrefix(err, "unable to resolve base ref", 0)
 			} else {
-				baseRef = base.String()
-				baseCommit, _ = repo.CommitObject(plumbing.NewHash(baseRef))
+				c.BaseRef = base.String()
+				baseCommit, _ = repo.CommitObject(plumbing.NewHash(c.BaseRef))
 			}
 		} else {
 			baseCommit, err = repo.CommitObject(baseHash)
@@ -49,15 +54,15 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
 	}
 
 	var headCommit *object.Commit
-	if len(headRef) > 0 {
-		headHash := plumbing.NewHash(headRef)
-		if !plumbing.IsHash(headRef) {
-			head, err := git.TryAdditionalBaseRefs(repo, headRef)
+	if len(c.HeadRef) > 0 {
+		headHash := plumbing.NewHash(c.HeadRef)
+		if !plumbing.IsHash(c.HeadRef) {
+			head, err := git.TryAdditionalBaseRefs(repo, c.HeadRef)
 			if err != nil {
 				return errors.WrapPrefix(err, "unable to resolve head ref", 0)
 			} else {
-				headRef = head.String()
-				headCommit, _ = repo.CommitObject(plumbing.NewHash(baseRef))
+				c.HeadRef = head.String()
+				headCommit, _ = repo.CommitObject(plumbing.NewHash(c.BaseRef))
 			}
 		} else {
 			headCommit, err = repo.CommitObject(headHash)
@@ -67,23 +72,23 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
 		}
 	}
 
-	// If baseCommit is an ancestor of headCommit, update baseRef to be the common ancestor.
+	// If baseCommit is an ancestor of headCommit, update c.BaseRef to be the common ancestor.
 	if headCommit != nil && baseCommit != nil {
 		mergeBase, err := headCommit.MergeBase(baseCommit)
 		if err != nil || len(mergeBase) < 1 {
 			return errors.WrapPrefix(err, "could not find common base between the given references", 0)
 		}
-		baseRef = mergeBase[0].Hash.String()
+		c.BaseRef = mergeBase[0].Hash.String()
 	}
 
-	if maxDepth != 0 {
-		opts = append(opts, git.ScanOptionMaxDepth(int64(maxDepth)))
+	if c.MaxDepth != 0 {
+		opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
 	}
-	if baseRef != "" {
-		opts = append(opts, git.ScanOptionBaseHash(baseRef))
+	if c.BaseRef != "" {
+		opts = append(opts, git.ScanOptionBaseHash(c.BaseRef))
 	}
-	if headRef != "" {
-		opts = append(opts, git.ScanOptionHeadCommit(headRef))
+	if c.HeadRef != "" {
+		opts = append(opts, git.ScanOptionHeadCommit(c.HeadRef))
 	}
 	scanOptions := git.NewScanOptions(opts...)
 
@@ -106,7 +111,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
 	e.sourcesWg.Add(1)
 	go func() {
 		defer e.sourcesWg.Done()
-		err := gitSource.ScanRepo(ctx, repo, repoPath, scanOptions, e.ChunksChan())
+		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
 		if err != nil {
 			logrus.WithError(err).Fatal("could not scan repo")
 		}

pkg/engine/git_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
@@ -56,7 +57,14 @@ func TestGitEngine(t *testing.T) {
 			WithDecoders(decoders.DefaultDecoders()...),
 			WithDetectors(false, DefaultDetectors()...),
 		)
-		if err := e.ScanGit(ctx, path, tTest.branch, tTest.base, tTest.maxDepth, tTest.filter); err != nil {
+		cfg := sources.Config{
+			RepoPath: path,
+			HeadRef:  tTest.branch,
+			BaseRef:  tTest.base,
+			MaxDepth: tTest.maxDepth,
+			Filter:   tTest.filter,
+		}
+		if err := e.ScanGit(ctx, &cfg); err != nil {
 			return
 		}
 		go e.Finish()
@@ -106,7 +114,11 @@ func BenchmarkGitEngine(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		// TODO: this is measuring the time it takes to initialize the source
 		// and not to do the full scan
-		if err := e.ScanGit(ctx, path, "", "", 0, common.FilterEmpty()); err != nil {
+		cfg := sources.Config{
+			RepoPath: path,
+			Filter:   common.FilterEmpty(),
+		}
+		if err := e.ScanGit(ctx, &cfg); err != nil {
 			return
 		}
 	}

pkg/engine/github.go

@@ -3,38 +3,44 @@ package engine
 import (
 	"context"
 
+	"github.com/go-errors/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
 )
 
-func (e *Engine) ScanGitHub(ctx context.Context, endpoint string, repos, orgs []string, token string, includeForks bool, filter *common.Filter, concurrency int, includeMembers bool) error {
+// ScanGitHub scans Github with the provided options.
+func (e *Engine) ScanGitHub(ctx context.Context, c *sources.Config) error {
+	if c == nil {
+		return errors.New("nil config provided for ScanGitHub")
+	}
+
 	source := github.Source{}
 	connection := sourcespb.GitHub{
-		Endpoint:      endpoint,
-		Organizations: orgs,
-		Repositories:  repos,
-		ScanUsers:     includeMembers,
+		Endpoint:      c.Endpoint,
+		Organizations: c.Orgs,
+		Repositories:  c.Repos,
+		ScanUsers:     c.IncludeMembers,
 	}
-	if len(token) > 0 {
+	if len(c.Token) > 0 {
 		connection.Credential = &sourcespb.GitHub_Token{
-			Token: token,
+			Token: c.Token,
 		}
 	} else {
 		connection.Credential = &sourcespb.GitHub_Unauthenticated{}
 	}
-	connection.IncludeForks = includeForks
+	connection.IncludeForks = c.IncludeForks
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
 	if err != nil {
 		logrus.WithError(err).Error("failed to marshal github connection")
 		return err
 	}
-	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, concurrency)
+	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
 	if err != nil {
 		logrus.WithError(err).Error("failed to initialize github source")
 		return err

pkg/engine/gitlab.go

@@ -6,31 +6,38 @@ import (
 
 	"github.com/go-errors/errors"
 	"github.com/sirupsen/logrus"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
 	"golang.org/x/net/context"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
 )
 
-func (e *Engine) ScanGitLab(ctx context.Context, endpoint, token string, repositories []string) error {
+// ScanGitLab scans GitLab with the provided configuration.
+func (e *Engine) ScanGitLab(ctx context.Context, c *sources.Config) error {
+	if c == nil {
+		return errors.New("config is nil for ScanGitlab")
+	}
+
 	connection := &sourcespb.GitLab{}
 
 	switch {
-	case len(token) > 0:
+	case len(c.Token) > 0:
 		connection.Credential = &sourcespb.GitLab_Token{
-			Token: token,
+			Token: c.Token,
 		}
 	default:
 		return fmt.Errorf("must provide token")
 	}
 
-	if len(endpoint) > 0 {
-		connection.Endpoint = endpoint
+	if len(c.Endpoint) > 0 {
+		connection.Endpoint = c.Endpoint
 	}
 
-	if len(repositories) > 0 {
-		connection.Repositories = repositories
+	if len(c.Repos) > 0 {
+		connection.Repositories = c.Repos
 	}
 
 	var conn anypb.Any