Add Couchbase Detector

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
 	github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
+	github.com/couchbase/gocb/v2 v2.6.3
 	github.com/crewjam/rfc5424 v0.1.0
 	github.com/denisenkom/go-mssqldb v0.12.3
 	github.com/envoyproxy/protoc-gen-validate v1.0.1
@@ -93,6 +94,7 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+  github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect

go.sum

@@ -113,6 +113,13 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
 github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
+github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
+github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
+github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
+github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -419,6 +426,7 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
 github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -430,6 +438,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=

pkg/common/patterns.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -15,7 +16,11 @@ const RegexPattern = "0-9a-z"
 const AlphaNumPattern = "0-9a-zA-Z"
 const HexPattern = "0-9a-f"
 
-//Custom Regex functions
+type RegexState struct {
+	compiledRegex *regexp.Regexp
+}
+
+// Custom Regex functions
 func BuildRegex(pattern string, specialChar string, length int) string {
 	return fmt.Sprintf(`\b([%s%s]{%s})\b`, pattern, specialChar, strconv.Itoa(length))
 }
@@ -37,3 +42,30 @@ func RangeValidation(rangeInput string) bool {
 func ToUpperCase(input string) string {
 	return strings.ToUpper(input)
 }
+
+func (r RegexState) Matches(data []byte) []string {
+	matches := r.compiledRegex.FindAllStringSubmatch(string(data), -1)
+
+	res := make([]string, 0, len(matches))
+
+	// trim off spaces and different quote types ('").
+	for i := range matches {
+		res = append(res, strings.Trim(matches[i][1], `"' )`))
+	}
+
+	return res
+}
+
+// UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
+func UsernameRegexCheck(pattern string) RegexState {
+	raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern)
+
+	return RegexState{regexp.MustCompile(raw)}
+}
+
+// PasswordRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
+func PasswordRegexCheck(pattern string) RegexState {
+	raw := fmt.Sprintf(`(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})`, pattern)
+
+	return RegexState{regexp.MustCompile(raw)}
+}

pkg/common/patterns_test.go

@@ -0,0 +1,59 @@
+package common
+
+import (
+	"github.com/stretchr/testify/assert"
+	"regexp"
+	"testing"
+)
+
+const (
+	usernamePattern = `?()/\+=\s\n`
+	passwordPattern = `^<>;.*&|£\n\s`
+	usernameRegex   = `(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`
+	passwordRegex   = `(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:^<>;.*&|£\n\s]{4,40})`
+)
+
+func TestUsernameRegexCheck(t *testing.T) {
+	usernameRegexPat := UsernameRegexCheck(usernamePattern)
+
+	expectedRegexPattern := regexp.MustCompile(usernameRegex)
+
+	if usernameRegexPat.compiledRegex.String() != expectedRegexPattern.String() {
+		t.Errorf("\n got %v \n want %v", usernameRegexPat.compiledRegex, expectedRegexPattern)
+	}
+
+	testString := `username = "johnsmith123"
+                   username='johnsmith123'
+				   username:="johnsmith123"
+                   username = johnsmith123
+                   username=johnsmith123`
+
+	expectedStr := []string{"johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123"}
+
+	usernameRegexMatches := usernameRegexPat.Matches([]byte(testString))
+
+	assert.Exactly(t, usernameRegexMatches, expectedStr)
+
+}
+
+func TestPasswordRegexCheck(t *testing.T) {
+	passwordRegexPat := PasswordRegexCheck(passwordPattern)
+
+	expectedRegexPattern := regexp.MustCompile(passwordRegex)
+	assert.Equal(t, passwordRegexPat.compiledRegex, expectedRegexPattern)
+
+	testString := `password = "johnsmith123$!"
+                   password='johnsmith123$!'
+				   password:="johnsmith123$!"
+                   password = johnsmith123$!
+                   password=johnsmith123$!
+				   PasswordAuthenticator(username, "johnsmith123$!")`
+
+	expectedStr := []string{"johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!",
+		"johnsmith123$!"}
+
+	passwordRegexMatches := passwordRegexPat.Matches([]byte(testString))
+
+	assert.Exactly(t, passwordRegexMatches, expectedStr)
+
+}

pkg/detectors/couchbase/couchbase.go

@@ -0,0 +1,152 @@
+package couchbase
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/couchbase/gocb/v2"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	connectionStringPat = regexp.MustCompile(`\bcb\.[a-z0-9]+\.cloud\.couchbase\.com\b`)
+	usernamePat         = `?()/\+=\s\n`
+	passwordPat         = `^<>;.*&|£\n\s`
+	//passwordPat         = regexp.MustCompile(`(?i)(?:pass|pwd)(?:.|[\n\r]){0,15}(\b[^<>;.*&|£\n\s]{8,100}$)`)
+	// passwordPat = regexp.MustCompile(`(?im)(?:pass|pwd)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`)
+)
+
+func meetsCouchbasePasswordRequirements(password string) (string, bool) {
+	var hasLower, hasUpper, hasNumber, hasSpecialChar bool
+	for _, char := range password {
+		switch {
+		case unicode.IsLower(char):
+			hasLower = true
+		case unicode.IsUpper(char):
+			hasUpper = true
+		case unicode.IsNumber(char):
+			hasNumber = true
+		case unicode.IsPunct(char) || unicode.IsSymbol(char):
+			hasSpecialChar = true
+		}
+
+		if hasLower && hasUpper && hasNumber && hasSpecialChar {
+			return password, true
+		}
+	}
+
+	return "", false
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"couchbase://", "couchbases://"}
+}
+
+// FromData will find and optionally verify Couchbase secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	connectionStringMatches := connectionStringPat.FindAllStringSubmatch(dataStr, -1)
+
+	// prepend 'couchbases://' to the connection string as the connection
+	// string format is couchbases://cb.stuff.cloud.couchbase.com but the
+	// cb.stuff.cloud.couchbase.com may be separated from the couchbases:// in codebases.
+	for i, connectionStringMatch := range connectionStringMatches {
+		connectionStringMatches[i][0] = "couchbases://" + connectionStringMatch[0]
+	}
+
+	usernameRegexState := common.UsernameRegexCheck(usernamePat)
+	usernameMatches := usernameRegexState.Matches(data)
+
+	passwordRegexState := common.PasswordRegexCheck(passwordPat)
+	passwordMatches := passwordRegexState.Matches(data)
+
+	for _, connectionStringMatch := range connectionStringMatches {
+		resConnectionStringMatch := strings.TrimSpace(connectionStringMatch[0])
+
+		for _, resUsernameMatch := range usernameMatches {
+
+			for _, resPasswordMatch := range passwordMatches {
+				_, metPasswordRequirements := meetsCouchbasePasswordRequirements(resPasswordMatch)
+
+				if !metPasswordRequirements {
+					continue
+				}
+
+				s1 := detectors.Result{
+					DetectorType: detectorspb.DetectorType_Couchbase,
+					Raw:          []byte(fmt.Sprintf("%s:%s@%s", resUsernameMatch, resPasswordMatch, resConnectionStringMatch)),
+				}
+
+				if verify {
+
+					options := gocb.ClusterOptions{
+						Authenticator: gocb.PasswordAuthenticator{
+							Username: resUsernameMatch,
+							Password: resPasswordMatch,
+						},
+					}
+
+					// Sets a pre-configured profile called "wan-development" to help avoid latency issues
+					// when accessing Capella from a different Wide Area Network
+					// or Availability Zone (e.g. your laptop).
+					if err := options.ApplyProfile(gocb.ClusterConfigProfileWanDevelopment); err != nil {
+						log.Fatal("apply profile err", err)
+					}
+
+					// Initialize the Connection
+					cluster, err := gocb.Connect(resConnectionStringMatch, options)
+					if err != nil {
+						continue
+					}
+
+					// We'll ping the KV nodes in our cluster.
+					pings, err := cluster.Ping(&gocb.PingOptions{
+						Timeout: time.Second * 5,
+					})
+
+					if err != nil {
+						continue
+					}
+
+					for _, ping := range pings.Services {
+						for _, pingEndpoint := range ping {
+							if pingEndpoint.State == gocb.PingStateOk {
+								s1.Verified = true
+								break
+							} else {
+								// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+								if detectors.IsKnownFalsePositive(resPasswordMatch, detectors.DefaultFalsePositives, true) {
+									continue
+								}
+							}
+						}
+					}
+				}
+
+				results = append(results, s1)
+			}
+		}
+	}
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Couchbase
+}