Merge branch 'develop' into patch-1

owasp-amass · Mar 26, 2023 · 89c975a · 89c975a
2 parents 637f755 + 96e26b1
commit 89c975a
Show file tree

Hide file tree

Showing 203 changed files with 3,561 additions and 4,906 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,6 +1,5 @@
 Dockerfile
 *.md
-snapcraft.yaml
 images/*
 examples/*
 .idea
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@
 [![GitHub Release](https://img.shields.io/github/release/OWASP/Amass)](https://github.com/OWASP/Amass/releases/latest)
 [![Docker Images](https://img.shields.io/docker/pulls/caffix/amass.svg)](https://hub.docker.com/r/caffix/amass)
 [![Follow on Twitter](https://img.shields.io/twitter/follow/owaspamass.svg?logo=twitter)](https://twitter.com/owaspamass)
-[![Chat on Discord](https://img.shields.io/discord/433729817918308352.svg?logo=discord)](https://discord.gg/rtN8GMd)
+[![Chat on Discord](https://img.shields.io/discord/433729817918308352.svg?logo=discord)](https://discord.gg/TMMyYtBMTR)
 
 ![GitHub Test Status](https://github.com/OWASP/Amass/workflows/tests/badge.svg)
 [![GoDoc](https://pkg.go.dev/badge/github.com/OWASP/Amass/v3?utm_source=godoc)](https://pkg.go.dev/github.com/OWASP/Amass/v3)
@@ -24,17 +24,17 @@ The OWASP Amass Project performs network mapping of attack surfaces and external
 
 | Technique    | Data Sources |
 |:-------------|:-------------|
-| APIs         | 360PassiveDNS, Ahrefs, AnubisDB, BeVigil, BinaryEdge, BufferOver, BuiltWith, C99, Chaos, CIRCL, Cloudflare, DNSDB, DNSRepo, Deepinfo, Detectify, FOFA, FullHunt, GitHub, GitLab, Greynoise, HackerTarget, Hunter, IntelX, LeakIX, Maltiverse, Mnemonic, Netlas, Pastebin, PassiveTotal, PentestTools, Pulsedive, Quake, SOCRadar, Searchcode, Shodan, SonarSearch, Spamhaus, Spyse, Sublist3rAPI, ThreatBook, ThreatCrowd, ThreatMiner, Twitter, URLScan, VirusTotal, Yandex, ZETAlytics, ZoomEye |
-| Certificates | Active pulls (optional), Censys, CertCentral, CertSpotter, Crtsh, Digitorus, FacebookCT, GoogleCT |
+| APIs         | 360PassiveDNS, Ahrefs, AnubisDB, BeVigil, BinaryEdge, BufferOver, BuiltWith, C99, Chaos, CIRCL, DNSDB, DNSRepo, Deepinfo, Detectify, FOFA, FullHunt, GitHub, GitLab, GrepApp, Greynoise, HackerTarget, Hunter, IntelX, LeakIX, Maltiverse, Mnemonic, Netlas, Pastebin, PassiveTotal, PentestTools, Pulsedive, Quake, SOCRadar, Searchcode, Shodan, Spamhaus, Sublist3rAPI, ThreatBook, ThreatMiner, URLScan, VirusTotal, Yandex, ZETAlytics, ZoomEye |
+| Certificates | Active pulls (optional), Censys, CertCentral, CertSpotter, Crtsh, Digitorus, FacebookCT |
 | DNS          | Brute forcing, Reverse DNS sweeping, NSEC zone walking, Zone transfers, FQDN alterations/permutations, FQDN Similarity-based Guessing |
-| Routing      | ASNLookup, BGPTools, BGPView, BigDataCloud, IPdata, IPinfo, NetworksDB, RADb, Robtex, ShadowServer, TeamCymru |
-| Scraping     | AbuseIPDB, Ask, Baidu, Bing, DNSDumpster, DNSHistory, DNSSpy, DuckDuckGo, Gists, Google, HackerOne, HyperStat, PKey, RapidDNS, Riddler, Searx, SiteDossier, Yahoo |
-| Web Archives | ArchiveIt, Arquivo, CommonCrawl, HAW, PublicWWW, UKWebArchive, Wayback |
-| WHOIS        | AlienVault, AskDNS, DNSlytics, ONYPHE, SecurityTrails, SpyOnWeb, Umbrella, WhoisXMLAPI |
+| Routing      | ASNLookup, BGPTools, BGPView, BigDataCloud, IPdata, IPinfo, RADb, Robtex, ShadowServer, TeamCymru |
+| Scraping     | AbuseIPDB, Ask, Baidu, Bing, CSP Header, DNSDumpster, DNSHistory, DNSSpy, DuckDuckGo, Gists, Google, HackerOne, HyperStat, PKey, RapidDNS, Riddler, Searx, SiteDossier, Yahoo |
+| Web Archives | Arquivo, CommonCrawl, HAW, PublicWWW, UKWebArchive, Wayback |
+| WHOIS        | AlienVault, AskDNS, DNSlytics, ONYPHE, SecurityTrails, SpyOnWeb, WhoisXMLAPI |
 
 ----
 
-## Installation [![Go Version](https://img.shields.io/github/go-mod/go-version/OWASP/Amass)](https://golang.org/dl/) [![Docker Images](https://img.shields.io/docker/pulls/caffix/amass.svg)](https://hub.docker.com/r/caffix/amass) [![Snapcraft](https://snapcraft.io/amass/badge.svg)](https://snapcraft.io/amass) [![GitHub Downloads](https://img.shields.io/github/downloads/OWASP/Amass/latest/total.svg)](https://github.com/OWASP/Amass/releases/latest)
+## Installation [![Go Version](https://img.shields.io/github/go-mod/go-version/OWASP/Amass)](https://golang.org/dl/) [![Docker Images](https://img.shields.io/docker/pulls/caffix/amass.svg)](https://hub.docker.com/r/caffix/amass) [![GitHub Downloads](https://img.shields.io/github/downloads/OWASP/Amass/latest/total.svg)](https://github.com/OWASP/Amass/releases/latest)
 
 > You can find some additional installation variations in the [Installation Guide](./doc/install.md).
 
@@ -51,12 +51,6 @@ brew tap caffix/amass
 brew install amass
 ```
 
-#### Snapcraft
-
-```bash
-sudo snap install amass
-```
-
 ### Docker Container
 
 1. Install [Docker](https://www.docker.com)
@@ -134,6 +128,6 @@ Add it to our ever-growing list of [REFERENCES.md](REFERENCES.md) by forking and
 
 ## Licensing [![License](https://img.shields.io/badge/license-apache%202-blue)](https://www.apache.org/licenses/LICENSE-2.0)
 
-This program is free software: you can redistribute it and/or modify it under the terms of the [Apache license](LICENSE). OWASP Amass and any contributions are Copyright © by Jeff Foley 2017-2022. Some subcomponents have separate licenses.
+This program is free software: you can redistribute it and/or modify it under the terms of the [Apache license](LICENSE). OWASP Amass and any contributions are Copyright © by Jeff Foley 2017-2023. Some subcomponents have separate licenses.
 
 ![Network graph](./images/network_06092018.png "Amass Network Mapping")
diff --git a/cmd/amass/db.go b/cmd/amass/db.go
@@ -1,4 +1,4 @@
-// Copyright © by Jeff Foley 2017-2022. All rights reserved.
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -11,7 +11,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"os"
 	"strconv"
@@ -102,8 +101,8 @@ func runDBCommand(clArgs []string) {
 		color.NoColor = true
 	}
 	if args.Options.Silent {
-		color.Output = ioutil.Discard
-		color.Error = ioutil.Discard
+		color.Output = io.Discard
+		color.Error = io.Discard
 	}
 	if args.Filepaths.Domains != "" {
 		list, err := config.GetListFromFile(args.Filepaths.Domains)

diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go
@@ -1,4 +1,4 @@
-// Copyright © by Jeff Foley 2017-2022. All rights reserved.
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -12,9 +12,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
-	"math/rand"
 	"net"
 	"os"
 	"os/signal"
@@ -163,8 +161,6 @@ func defineEnumFilepathFlags(enumFlags *flag.FlagSet, args *enumArgs) {
 }
 
 func runEnumCommand(clArgs []string) {
-	// Seed the default pseudo-random number generator
-	rand.Seed(time.Now().UTC().UnixNano())
 	// Extract the correct config from the user provided arguments and/or configuration file
 	cfg, args := argsAndConfig(clArgs)
 	if cfg == nil {
@@ -173,9 +169,10 @@ func runEnumCommand(clArgs []string) {
 	createOutputDirectory(cfg)
 
 	rLog, wLog := io.Pipe()
+	dir := config.OutputDirectory(cfg.Dir)
 	// Setup logging so that messages can be written to the file and used by the program
 	cfg.Log = log.New(wLog, "", log.Lmicroseconds)
-	logfile := filepath.Join(config.OutputDirectory(cfg.Dir), "amass.log")
+	logfile := filepath.Join(dir, "amass.log")
 	if args.Filepaths.LogFile != "" {
 		logfile = args.Filepaths.LogFile
 	}
@@ -196,9 +193,8 @@ func runEnumCommand(clArgs []string) {
 	// Expand data source category names into the associated source names
 	initializeSourceTags(sys.DataSources())
 	cfg.SourceFilter.Sources = expandCategoryNames(cfg.SourceFilter.Sources, generateCategoryMap(sys))
-	// Create the in-memory graph database used to store enumeration findings
-	graph := netmap.NewGraph(netmap.NewCayleyGraphMemory())
-	defer graph.Close()
+
+	graph := sys.GraphDatabases()[0]
 	// Setup the new enumeration
 	e := enum.NewEnumeration(cfg, sys, graph)
 	if e == nil {
@@ -263,10 +259,9 @@ func runEnumCommand(clArgs []string) {
 	// Let all the output goroutines know that the enumeration has finished
 	close(done)
 	wg.Wait()
+	fmt.Fprintf(color.Error, "\n%s\n", green("The enumeration has finished"))
 	// If necessary, handle graph database migration
-	if len(e.Sys.GraphDatabases()) > 0 {
-		fmt.Fprintf(color.Error, "\n%s\n", green("The enumeration has finished"))
-
+	if len(e.Sys.GraphDatabases()) > 1 {
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 		defer cancel()
 		// Monitor for cancellation by the user
@@ -279,7 +274,7 @@ func runEnumCommand(clArgs []string) {
 			c()
 		}(cancel)
 		// Copy the graph of findings into the system graph databases
-		for _, g := range e.Sys.GraphDatabases() {
+		for _, g := range e.Sys.GraphDatabases()[1:] {
 			fmt.Fprintf(color.Error, "%s%s%s\n",
 				yellow("Discoveries are being migrated into the "), yellow(g.String()), yellow(" database"))
 
@@ -345,8 +340,8 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) {
 		color.NoColor = true
 	}
 	if args.Options.Silent {
-		color.Output = ioutil.Discard
-		color.Error = ioutil.Discard
+		color.Output = io.Discard
+		color.Error = io.Discard
 	}
 	if args.AltWordListMask.Len() > 0 {
 		args.AltWordList.Union(args.AltWordListMask)
@@ -551,7 +546,7 @@ func processOutput(ctx context.Context, g *netmap.Graph, e *enum.Enumeration, ou
 		}
 	}
 
-	t := time.NewTicker(3 * time.Second)
+	t := time.NewTimer(10 * time.Second)
 	defer t.Stop()
 	for {
 		select {
@@ -562,7 +557,8 @@ func processOutput(ctx context.Context, g *netmap.Graph, e *enum.Enumeration, ou
 			extract(0)
 			return
 		case <-t.C:
-			extract(100)
+			extract(500)
+			t.Reset(10 * time.Second)
 		}
 	}
 }

diff --git a/cmd/amass/help.go b/cmd/amass/help.go
@@ -1,3 +1,7 @@
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+// SPDX-License-Identifier: Apache-2.0
+
 package main
 
 import (

diff --git a/cmd/amass/intel.go b/cmd/amass/intel.go
@@ -1,4 +1,4 @@
-// Copyright © by Jeff Foley 2017-2022. All rights reserved.
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"math/rand"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -140,9 +139,6 @@ func runIntelCommand(clArgs []string) {
 		commandUsage(intelUsageMsg, intelCommand, intelBuf)
 		os.Exit(1)
 	}
-
-	// Seed the default pseudo-random number generator
-	rand.Seed(time.Now().UTC().UnixNano())
 	if err := processIntelInputFiles(&args); err != nil {
 		fmt.Fprintf(color.Error, "%v\n", err)
 		os.Exit(1)

diff --git a/cmd/amass/io.go b/cmd/amass/io.go
@@ -1,4 +1,4 @@
-// Copyright © by Jeff Foley 2017-2022. All rights reserved.
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -43,14 +43,7 @@ func EventOutput(ctx context.Context, g *netmap.Graph, uuid string, f *stringset
 		defer f.Close()
 	}
 
-	var fqdns []string
-	for _, name := range g.EventFQDNs(ctx, uuid) {
-		if !f.Has(name) {
-			fqdns = append(fqdns, name)
-		}
-	}
-
-	names := randomSelection(fqdns, limit)
+	names := randomSelection(g.EventFQDNs(ctx, uuid), f, limit)
 	lookup := make(outLookup, len(names))
 	for _, o := range buildNameInfo(ctx, g, uuid, names) {
 		lookup[o.Name] = o
@@ -73,16 +66,19 @@ func EventOutput(ctx context.Context, g *netmap.Graph, uuid string, f *stringset
 	return addInfrastructureInfo(lookup, f, cache)
 }
 
-func randomSelection(names []string, limit int) []string {
+func randomSelection(names []string, filter *stringset.Set, limit int) []string {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
+	var count int
 	var sel []string
-	for i, n := range r.Perm(len(names)) {
-		if limit > 0 && i >= limit {
+	for _, n := range r.Perm(len(names)) {
+		if limit > 0 && count >= limit {
 			break
 		}
-
-		sel = append(sel, names[n])
+		if name := names[n]; !filter.Has(name) {
+			count++
+			sel = append(sel, name)
+		}
 	}
 	return sel
 }

diff --git a/cmd/amass/main.go b/cmd/amass/main.go
@@ -1,24 +1,25 @@
-// Copyright © by Jeff Foley 2017-2022. All rights reserved.
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 // SPDX-License-Identifier: Apache-2.0
 
 // In-depth Attack Surface Mapping and Asset Discovery
-//  +----------------------------------------------------------------------------+
-//  | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░  OWASP Amass  ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ |
-//  +----------------------------------------------------------------------------+
-//  |      .+++:.            :                             .+++.                 |
-//  |    +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+   |
-//  |   &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&   |
-//  |  +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8         |
-//  |  8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:       |
-//  |  WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:     |
-//  |  #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8    |
-//  |  o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.  |
-//  |   WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o  |
-//  |   :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+  |
-//  |    :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&   |
-//  |      +o&&&&+.                                                    +oooo.    |
-//  +----------------------------------------------------------------------------+
+//
+//	+----------------------------------------------------------------------------+
+//	| ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░  OWASP Amass  ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ |
+//	+----------------------------------------------------------------------------+
+//	|      .+++:.            :                             .+++.                 |
+//	|    +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+   |
+//	|   &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&   |
+//	|  +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8         |
+//	|  8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:       |
+//	|  WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:     |
+//	|  #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8    |
+//	|  o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.  |
+//	|   WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o  |
+//	|   :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+  |
+//	|    :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&   |
+//	|      +o&&&&+.                                                    +oooo.    |
+//	+----------------------------------------------------------------------------+
 package main
 
 import (

diff --git a/cmd/amass/track.go b/cmd/amass/track.go
@@ -1,4 +1,4 @@
-// Copyright © by Jeff Foley 2017-2022. All rights reserved.
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -9,8 +9,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
-	"math/rand"
+	"io"
 	"os"
 	"time"
 
@@ -81,8 +80,8 @@ func runTrackCommand(clArgs []string) {
 		color.NoColor = true
 	}
 	if args.Options.Silent {
-		color.Output = ioutil.Discard
-		color.Error = ioutil.Discard
+		color.Output = io.Discard
+		color.Error = io.Discard
 	}
 	// Some input validation
 	if args.Since != "" && args.Last != 0 {
@@ -116,8 +115,6 @@ func runTrackCommand(clArgs []string) {
 		}
 	}
 
-	rand.Seed(time.Now().UTC().UnixNano())
-
 	cfg := config.NewConfig()
 	// Check if a configuration file was provided, and if so, load the settings
 	if err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {

diff --git a/cmd/amass/viz.go b/cmd/amass/viz.go
@@ -1,4 +1,4 @@
-// Copyright © by Jeff Foley 2017-2022. All rights reserved.
+// Copyright © by Jeff Foley 2017-2023. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -8,11 +8,9 @@ import (
 	"bytes"
 	"context"
 	"flag"
-	"io/ioutil"
-	"math/rand"
+	"io"
 	"os"
 	"path/filepath"
-	"time"
 
 	"github.com/OWASP/Amass/v3/config"
 	"github.com/OWASP/Amass/v3/viz"
@@ -91,8 +89,8 @@ func runVizCommand(clArgs []string) {
 		color.NoColor = true
 	}
 	if args.Options.Silent {
-		color.Output = ioutil.Discard
-		color.Error = ioutil.Discard
+		color.Output = io.Discard
+		color.Error = io.Discard
 	}
 	// Make sure at least one graph file format has been identified on the command-line
 	if !args.Options.D3 && !args.Options.DOT &&
@@ -109,8 +107,6 @@ func runVizCommand(clArgs []string) {
 		args.Domains.InsertMany(list...)
 	}
 
-	rand.Seed(time.Now().UTC().UnixNano())
-
 	cfg := new(config.Config)
 	// Check if a configuration file was provided, and if so, load the settings
 	if err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {