Add missing pagination on github calls (trufflesecurity#30)

* Add missing pagination on github calls

Includes some refactoring to improve readability and code reuse.

* Close response body and handle rate limit

* Re-include support for including users as repos to github scans

* Fix gist test to match new func signature

* Add current test name to logging

* Support username as org use case

* Also include no-auth user as org

Co-authored-by: Bill Rich <bill.rich@trufflesec.com>

Makefile

@@ -21,16 +21,16 @@ check:
 	go vet $(shell go list ./... | grep -v /vendor/)
 
 test-failing:
-	CGO_ENABLED=0 go test -timeout=30s $(shell go list ./... | grep -v /vendor/) | grep FAIL
+	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/) | grep FAIL
 
 test:
-	CGO_ENABLED=0 go test -timeout=30s $(shell go list ./... | grep -v /vendor/ | grep -v /pkg/detectors)
+	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v /pkg/detectors)
 
 test-race:
-	CGO_ENABLED=1 go test -timeout=30s -race $(shell go list ./... | grep -v /vendor/ | grep -v /pkg/detectors)
+	CGO_ENABLED=1 go test -timeout=5m -race $(shell go list ./... | grep -v /vendor/ | grep -v /pkg/detectors)
 
 test-detectors:
-	CGO_ENABLED=0 go test -timeout=30s $(shell go list ./... | grep /pkg/detectors)
+	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep /pkg/detectors)
 
 bench:
 	CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .

pkg/common/utils.go

@@ -0,0 +1,19 @@
+package common
+
+func AddStringSliceItem(item string, slice *[]string) {
+	for _, i := range *slice {
+		if i == item {
+			return
+		}
+	}
+	*slice = append(*slice, item)
+}
+
+func RemoveStringSliceItem(item string, slice *[]string) {
+	for i, listItem := range *slice {
+		if item == listItem {
+			(*slice)[i] = (*slice)[len(*slice)-1]
+			*slice = (*slice)[:len(*slice)-1]
+		}
+	}
+}

pkg/common/utils_test.go

@@ -0,0 +1,76 @@
+package common
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestAddItem(t *testing.T) {
+	type Case struct {
+		Slice    []string
+		Modifier []string
+		Expected []string
+	}
+	tests := map[string]Case{
+		"newItem": {
+			Slice:    []string{"a", "b", "c"},
+			Modifier: []string{"d"},
+			Expected: []string{"a", "b", "c", "d"},
+		},
+		"newDuplicate": {
+			Slice:    []string{"a", "b", "c"},
+			Modifier: []string{"c"},
+			Expected: []string{"a", "b", "c"},
+		},
+	}
+
+	for name, test := range tests {
+		for _, item := range test.Modifier {
+			AddStringSliceItem(item, &test.Slice)
+		}
+
+		if !reflect.DeepEqual(test.Slice, test.Expected) {
+			t.Errorf("%s: expected:%v, got:%v", name, test.Expected, test.Slice)
+		}
+	}
+}
+
+func TestRemoveItem(t *testing.T) {
+	type Case struct {
+		Slice    []string
+		Modifier []string
+		Expected []string
+	}
+	tests := map[string]Case{
+		"existingItemEnd": {
+			Slice:    []string{"a", "b", "c"},
+			Modifier: []string{"c"},
+			Expected: []string{"a", "b"},
+		},
+		"existingItemMiddle": {
+			Slice:    []string{"a", "b", "c"},
+			Modifier: []string{"b"},
+			Expected: []string{"a", "c"},
+		},
+		"existingItemBeginning": {
+			Slice:    []string{"a", "b", "c"},
+			Modifier: []string{"a"},
+			Expected: []string{"c", "b"},
+		},
+		"nonExistingItem": {
+			Slice:    []string{"a", "b", "c"},
+			Modifier: []string{"d"},
+			Expected: []string{"a", "b", "c"},
+		},
+	}
+
+	for name, test := range tests {
+		for _, item := range test.Modifier {
+			RemoveStringSliceItem(item, &test.Slice)
+		}
+
+		if !reflect.DeepEqual(test.Slice, test.Expected) {
+			t.Errorf("%s: expected:%v, got:%v", name, test.Expected, test.Slice)
+		}
+	}
+}