Adding flag to specify which http statuses to ignore as results

pkg/cmd/config.go

@@ -23,6 +23,11 @@ func scanConfigFromCmd(cmd *cobra.Command) (*scan.Config, error) {
 		return nil, errors.Wrap(err, "failed to read http methods flag")
 	}
 
+	c.HTTPStatusesToIgnore, err = cmd.Flags().GetIntSlice(flagHTTPStatusesToIgnore)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read http methods flag")
+	}
+
 	c.Threads, err = cmd.Flags().GetInt(flagThreads)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read threads flag")

pkg/cmd/flags.go

@@ -6,18 +6,19 @@ const (
 	flagVerboseShort = "v"
 
 	// Scan flags
-	flagDictionary      = "dictionary"
-	flagDictionaryShort = "d"
-	flagHTTPMethods     = "http-methods"
-	flagHTTPTimeout     = "http-timeout"
-	flagScanDepth       = "scan-depth"
-	flagThreads         = "threads"
-	flagThreadsShort    = "t"
-	flagSocks5Host      = "socks5"
-	flagUserAgent       = "user-agent"
-	flagCookieJar       = "use-cookie-jar"
-	flagCookie          = "cookie"
-	flagHeader          = "header"
+	flagDictionary           = "dictionary"
+	flagDictionaryShort      = "d"
+	flagHTTPMethods          = "http-methods"
+	flagHTTPStatusesToIgnore = "http-statuses-to-ignore"
+	flagHTTPTimeout          = "http-timeout"
+	flagScanDepth            = "scan-depth"
+	flagThreads              = "threads"
+	flagThreadsShort         = "t"
+	flagSocks5Host           = "socks5"
+	flagUserAgent            = "user-agent"
+	flagCookieJar            = "use-cookie-jar"
+	flagCookie               = "cookie"
+	flagHeader               = "header"
 
 	// Generate dictionary flags
 	flagOutput           = "out"

pkg/cmd/scan.go

@@ -46,6 +46,12 @@ func NewScanCommand(logger *logrus.Logger) (*cobra.Command, error) {
 		"comma separated list of http methods to use; eg: GET,POST,PUT",
 	)
 
+	cmd.Flags().IntSlice(
+		flagHTTPStatusesToIgnore,
+		[]int{http.StatusNotFound},
+		"comma separated list of http statuses to ignore when showing results; eg: 404,301",
+	)
+
 	cmd.Flags().IntP(
 		flagThreads,
 		flagThreadsShort,
@@ -158,7 +164,7 @@ func startScan(logger *logrus.Logger, cnf *scan.Config, u *url.URL) error {
 	}
 
 	targetProducer := producer.NewDictionaryProducer(cnf.HTTPMethods, dict, cnf.ScanDepth)
-	reproducer := producer.NewReProducer(targetProducer)
+	reproducer := producer.NewReProducer(targetProducer, cnf.HTTPStatusesToIgnore)
 
 	s := scan.NewScanner(
 		c,
@@ -180,7 +186,7 @@ func startScan(logger *logrus.Logger, cnf *scan.Config, u *url.URL) error {
 		"user-agent":        cnf.UserAgent,
 	}).Info("Starting scan")
 
-	resultSummarizer := summarizer.NewResultSummarizer(logger)
+	resultSummarizer := summarizer.NewResultSummarizer(cnf.HTTPStatusesToIgnore, logger)
 
 	osSigint := make(chan os.Signal, 1)
 	signal.Notify(osSigint, os.Interrupt)

pkg/cmd/scan_integration_test.go

@@ -51,6 +51,8 @@ func TestScanCommand(t *testing.T) {
 		"--dictionary",
 		"testdata/dict2.txt",
 		"-v",
+		"--http-statuses-to-ignore",
+		"404",
 		"--http-timeout",
 		"300",
 	)
@@ -100,6 +102,37 @@ func TestScanCommand(t *testing.T) {
 	assert.Contains(t, loggerBuffer.String(), expectedResultTree)
 }
 
+func TestScanWithInvalidStatusesToIgnoreShouldErr(t *testing.T) {
+	logger, _ := test.NewLogger()
+
+	c, err := createCommand(logger)
+	assert.NoError(t, err)
+	assert.NotNil(t, c)
+
+	testServer, serverAssertion := test.NewServerWithAssertion(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+	)
+	defer testServer.Close()
+
+	_, _, err = executeCommand(
+		c,
+		"scan",
+		testServer.URL,
+		"--dictionary",
+		"testdata/dict2.txt",
+		"-v",
+		"--http-statuses-to-ignore",
+		"300,gibberish,404",
+		"--http-timeout",
+		"300",
+	)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "strconv.Atoi: parsing")
+	assert.Contains(t, err.Error(), "gibberish")
+
+	assert.Equal(t, 0, serverAssertion.Len())
+}
+
 func TestScanWithNoTargetShouldErr(t *testing.T) {
 	logger, _ := test.NewLogger()
 

pkg/scan/config.go

@@ -9,6 +9,7 @@ import (
 type Config struct {
 	DictionaryPath        string
 	HTTPMethods           []string
+	HTTPStatusesToIgnore  []int
 	Threads               int
 	TimeoutInMilliseconds int
 	ScanDepth             int

pkg/scan/producer/reproducer.go

@@ -10,20 +10,25 @@ import (
 
 const defaultChannelBuffer = 25
 
-var statusCodesToSkip = map[int]bool{
-	404: false,
-}
-
 func NewReProducer(
 	producer scan.Producer,
+	httpStatusesToIgnore []int,
 ) *ReProducer {
+	httpStatusesToIgnoreMap := make(map[int]struct{}, len(httpStatusesToIgnore))
+
+	for _, statusToIgnore := range httpStatusesToIgnore {
+		httpStatusesToIgnoreMap[statusToIgnore] = struct{}{}
+	}
+
 	return &ReProducer{
-		producer: producer,
+		producer:                producer,
+		httpStatusesToIgnoreMap: httpStatusesToIgnoreMap,
 	}
 }
 
 type ReProducer struct {
-	producer scan.Producer
+	producer                scan.Producer
+	httpStatusesToIgnoreMap map[int]struct{}
 }
 
 // Reproduce will check if it is possible to go deeper on the result provided, if so will
@@ -40,7 +45,7 @@ func (r *ReProducer) buildReproducer() func(result scan.Result) <-chan scan.Targ
 		go func() {
 			defer close(resultChannel)
 
-			if _, ok := statusCodesToSkip[result.StatusCode]; ok {
+			if _, ok := r.httpStatusesToIgnoreMap[result.StatusCode]; ok {
 				return
 			}
 

pkg/scan/producer/reproducer_test.go

@@ -20,7 +20,7 @@ func TestNewReProducer(t *testing.T) {
 
 	dictionaryProducer := producer.NewDictionaryProducer(methods, dictionary, 1)
 
-	sut := producer.NewReProducer(dictionaryProducer)
+	sut := producer.NewReProducer(dictionaryProducer, []int{http.StatusNotFound})
 
 	result := scan.NewResult(
 		scan.Target{
@@ -92,7 +92,7 @@ func TestReProducerShouldProduceNothingForDepthZero(t *testing.T) {
 
 	dictionaryProducer := producer.NewDictionaryProducer(methods, dictionary, 1)
 
-	sut := producer.NewReProducer(dictionaryProducer)
+	sut := producer.NewReProducer(dictionaryProducer, []int{http.StatusNotFound})
 
 	result := scan.NewResult(
 		scan.Target{
@@ -127,7 +127,7 @@ func TestReProducerShouldProduceNothingFor404Response(t *testing.T) {
 
 	dictionaryProducer := producer.NewDictionaryProducer(methods, dictionary, 1)
 
-	sut := producer.NewReProducer(dictionaryProducer)
+	sut := producer.NewReProducer(dictionaryProducer, []int{http.StatusNotFound})
 
 	result := scan.NewResult(
 		scan.Target{

pkg/scan/scanner_integration_test.go

@@ -20,7 +20,7 @@ func TestScanningWithEmptyProducerWillProduceNoResults(t *testing.T) {
 	sut := scan.NewScanner(
 		c,
 		prod,
-		producer.NewReProducer(prod),
+		producer.NewReProducer(prod, []int{http.StatusNotFound}),
 		logger,
 	)
 
@@ -58,7 +58,7 @@ func TestScannerWillLogAnErrorWithInvalidDictionary(t *testing.T) {
 	sut := scan.NewScanner(
 		c,
 		prod,
-		producer.NewReProducer(prod),
+		producer.NewReProducer(prod, []int{http.StatusNotFound}),
 		logger,
 	)
 
@@ -109,7 +109,7 @@ func TestScannerWillNotRedirectIfStatusCodeIsInvalid(t *testing.T) {
 	sut := scan.NewScanner(
 		c,
 		prod,
-		producer.NewReProducer(prod),
+		producer.NewReProducer(prod, []int{http.StatusNotFound}),
 		logger,
 	)
 
@@ -175,7 +175,7 @@ func TestScannerWillChangeMethodForRedirect(t *testing.T) {
 	sut := scan.NewScanner(
 		c,
 		prod,
-		producer.NewReProducer(prod),
+		producer.NewReProducer(prod, []int{http.StatusNotFound}),
 		logger,
 	)
 
@@ -249,7 +249,7 @@ func TestScannerWhenOutOfDepthWillNotFollowRedirect(t *testing.T) {
 	sut := scan.NewScanner(
 		c,
 		prod,
-		producer.NewReProducer(prod),
+		producer.NewReProducer(prod, []int{http.StatusNotFound}),
 		logger,
 	)
 
@@ -312,7 +312,7 @@ func TestScannerWillSkipRedirectWhenLocationHostIsDifferent(t *testing.T) {
 	sut := scan.NewScanner(
 		c,
 		prod,
-		producer.NewReProducer(prod),
+		producer.NewReProducer(prod, []int{http.StatusNotFound}),
 		logger,
 	)
 

pkg/scan/summarizer/result_summarizer.go

@@ -16,19 +16,30 @@ import (
 const (
 	breakingText = "Found something breaking"
 	foundText    = "Found"
-	notFoundText = "Not found"
+	ignoredText  = "Ignored"
 )
 
-func NewResultSummarizer(logger *logrus.Logger) *ResultSummarizer {
-	return &ResultSummarizer{logger: logger, resultMap: make(map[string]struct{})}
+func NewResultSummarizer(httpStatusesToIgnore []int, logger *logrus.Logger) *ResultSummarizer {
+	httpStatusesToIgnoreMap := make(map[int]struct{}, len(httpStatusesToIgnore))
+
+	for _, statusToIgnore := range httpStatusesToIgnore {
+		httpStatusesToIgnoreMap[statusToIgnore] = struct{}{}
+	}
+
+	return &ResultSummarizer{
+		httpStatusesToIgnoreMap: httpStatusesToIgnoreMap,
+		logger:                  logger,
+		resultMap:               make(map[string]struct{}),
+	}
 }
 
 type ResultSummarizer struct {
-	logger          *logrus.Logger
-	results         []scan.Result
-	resultMap       map[string]struct{}
-	resultsReceived int
-	mux             sync.RWMutex
+	httpStatusesToIgnoreMap map[int]struct{}
+	logger                  *logrus.Logger
+	results                 []scan.Result
+	resultMap               map[string]struct{}
+	resultsReceived         int
+	mux                     sync.RWMutex
 }
 
 func (s *ResultSummarizer) Add(result scan.Result) {
@@ -44,7 +55,7 @@ func (s *ResultSummarizer) Add(result scan.Result) {
 	}
 
 	s.log(result)
-	if result.StatusCode == http.StatusNotFound {
+	if _, found := s.httpStatusesToIgnoreMap[result.StatusCode]; found {
 		return
 	}
 
@@ -131,8 +142,8 @@ func (s *ResultSummarizer) log(result scan.Result) {
 		"url":         result.URL.String(),
 	})
 
-	if statusCode == http.StatusNotFound {
-		l.Debug(notFoundText)
+	if _, found := s.httpStatusesToIgnoreMap[result.StatusCode]; found {
+		l.Debug(ignoredText)
 	} else if statusCode >= http.StatusInternalServerError {
 		l.Warn(breakingText)
 	} else {

pkg/scan/summarizer/result_summarizer_test.go

@@ -17,7 +17,7 @@ func TestResultSummarizerShouldSummarizeResults(t *testing.T) {
 	logger, loggerBuffer := test.NewLogger()
 	logger.SetLevel(logrus.FatalLevel)
 
-	sut := summarizer.NewResultSummarizer(logger)
+	sut := summarizer.NewResultSummarizer([]int{http.StatusNotFound}, logger)
 
 	sut.Add(
 		scan.NewResult(
@@ -250,7 +250,7 @@ func TestResultSummarizerShouldLogResults(t *testing.T) {
 				},
 			),
 			expectedToContain: []string{
-				"Not found",
+				"Ignored",
 				"method=GET",
 				"status-code=404",
 				`url="http://mysite/gibberish"`,
@@ -263,7 +263,7 @@ func TestResultSummarizerShouldLogResults(t *testing.T) {
 		t.Run(tc.result.Target.Path, func(t *testing.T) {
 			t.Parallel()
 			logger, loggerBuffer := test.NewLogger()
-			sut := summarizer.NewResultSummarizer(logger)
+			sut := summarizer.NewResultSummarizer([]int{http.StatusNotFound}, logger)
 
 			sut.Add(tc.result)