Skip to content

Commit

Permalink
Merge branch 'develop' into patch-68
Browse files Browse the repository at this point in the history
  • Loading branch information
caffix committed Jun 27, 2022
2 parents b03af20 + 631798f commit 6acea62
Show file tree
Hide file tree
Showing 21 changed files with 494 additions and 72 deletions.
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ The OWASP Amass Project performs network mapping of attack surfaces and external

| Technique | Data Sources |
|:-------------|:-------------|
| APIs | 360PassiveDNS, Ahrefs, AnubisDB, BinaryEdge, BufferOver, BuiltWith, C99, Chaos, CIRCL, Cloudflare, DNSDB, DNSRepo, Detectify, FOFA, FullHunt, GitHub, GitLab, Greynoise, HackerTarget, Hunter, IntelX, LeakIX, Maltiverse, Mnemonic, Netlas, Pastebin, PassiveTotal, PentestTools, Quake, Searchcode, Shodan, SonarSearch, Spamhaus, Spyse, Sublist3rAPI, ThreatBook, ThreatCrowd, ThreatMiner, Twitter, URLScan, VirusTotal, Yandex, ZETAlytics, ZoomEye |
| Certificates | Active pulls (optional), Censys, CertSpotter, Crtsh, Digitorus, FacebookCT, GoogleCT |
| APIs | 360PassiveDNS, Ahrefs, AnubisDB, BinaryEdge, BufferOver, BuiltWith, C99, Chaos, CIRCL, Cloudflare, DNSDB, DNSRepo, Deepinfo, Detectify, FOFA, FullHunt, GitHub, GitLab, Greynoise, HackerTarget, Hunter, IntelX, LeakIX, Maltiverse, Mnemonic, Netlas, Pastebin, PassiveTotal, PentestTools, Pulsedive, Quake, SOCRadar, Searchcode, Shodan, SonarSearch, Spamhaus, Spyse, Sublist3rAPI, ThreatBook, ThreatCrowd, ThreatMiner, Twitter, URLScan, VirusTotal, Yandex, ZETAlytics, ZoomEye |
| Certificates | Active pulls (optional), Censys, CertCentral, CertSpotter, Crtsh, Digitorus, FacebookCT, GoogleCT |
| DNS | Brute forcing, Reverse DNS sweeping, NSEC zone walking, Zone transfers, FQDN alterations/permutations, FQDN Similarity-based Guessing |
| Routing | ASNLookup, BGPTools, BGPView, BigDataCloud, IPdata, IPinfo, NetworksDB, RADb, Robtex, ShadowServer, TeamCymru |
| Scraping | AbuseIPDB, Ask, Baidu, Bing, DNSDumpster, DNSHistory, DuckDuckGo, Gists, Google, HackerOne, HyperStat, PKey, RapidDNS, Riddler, Searx, SiteDossier, Yahoo |
| Scraping | AbuseIPDB, Ask, Baidu, Bing, DNSDumpster, DNSHistory, DNSSpy, DuckDuckGo, Gists, Google, HackerOne, HyperStat, PKey, RapidDNS, Riddler, Searx, SiteDossier, Yahoo |
| Web Archives | ArchiveIt, Arquivo, CommonCrawl, HAW, PublicWWW, UKWebArchive, Wayback |
| WHOIS | AlienVault, AskDNS, DNSlytics, ONYPHE, SecurityTrails, SpyOnWeb, Umbrella, WhoisXMLAPI |

Expand Down
32 changes: 26 additions & 6 deletions examples/config.ini
Original file line number Diff line number Diff line change
Expand Up @@ -175,17 +175,24 @@ minimum_ttl = 1440 ; One day
#[data_sources.Chaos.Credentials]
#apikey =

# https://circl.lu (Contact)
#[data_sources.CIRCL]
#[data_sources.CIRCL.Credentials]
#username =
#password =

# https://cloudflare.com (Free)
# Cloudflare apikey is the API token with dns_records and zone read permission
#[data_sources.Cloudflare]
#[data_sources.Cloudflare.Credentials]
#apikey =

# https://circl.lu (Contact)
#[data_sources.CIRCL]
#[data_sources.CIRCL.Credentials]
# https://www.digicert.com/tls-ssl/certcentral-tls-ssl-manager (Free)
# CertCentral username is the account ID (account number)
#[data_sources.CertCentral]
#[data_sources.CertCentral.Credentials]
#username =
#password =
#apikey =

# https://dnsdb.info (Paid)
#[data_sources.DNSDB]
Expand All @@ -203,6 +210,11 @@ minimum_ttl = 1440 ; One day
#[data_sources.DNSRepo.Credentials]
#apikey =

# https://deepinfo.com (Paid/Free-Trial)
#[data_sources.Deepinfo]
#[data_sources.Deepinfo.Credentials]
#apikey =

# https://detectify.com (Paid)
#[data_sources.Detectify]
#[data_sources.Detectify.Credentials]
Expand Down Expand Up @@ -256,6 +268,7 @@ minimum_ttl = 1440 ; One day
#[data_sources.Hunter.Credentials]
#apikey =

# https://intelx.io (Freemium)
#[data_sources.IntelX]
#[data_sources.IntelX.Credentials]
#apikey =
Expand Down Expand Up @@ -297,7 +310,7 @@ minimum_ttl = 1440 ; One day
#[data_sources.Pastebin.Credentials]
#apikey =

# https://passivetotal.com (Paid/Free-trial)
# https://www.riskiq.com/products/passivetotal (Paid/Free-trial)
#[data_sources.PassiveTotal]
#ttl = 10080
#[data_sources.PassiveTotal.Credentials]
Expand All @@ -322,6 +335,13 @@ minimum_ttl = 1440 ; One day
#[data_sources.Quake.Credentials]
#apikey =

# https://socradar.io (Paid)
# This requires a SOCRadar ThreatFusion API key, which is different from a general SOCRadar API key.
# To obtain it, contact the SOCRadar operation team via operation@socradar.io
#[data_sources.SOCRadar]
#[data_sources.SOCRadar.Credentials]
#apikey =

# https://securitytrails.com (Paid/Free-trial)
#[data_sources.SecurityTrails]
#ttl = 1440
Expand All @@ -334,7 +354,7 @@ minimum_ttl = 1440 ; One day
#[data_sources.Shodan.Credentials]
#apikey =

# https://spamhaus.com (Free)
# https://spamhaus.com (Freemium)
#[data_sources.Spamhaus]
#ttl = 1440
#[data_sources.Spamhaus.Credentials]
Expand Down
2 changes: 1 addition & 1 deletion resources/scripts/api/360passivedns.ads
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,5 @@ function vertical(ctx, domain)
end

function build_url(domain)
return "https://api.passivedns.cn/flint/rrset/*." .. domain .. "/?source=ALL&batch=1000"
return "https://api.passivedns.cn/flint/rrset/*." .. domain .. "/?source=ALL&batch=5000"
end
15 changes: 9 additions & 6 deletions resources/scripts/api/ahrefs.ads
Original file line number Diff line number Diff line change
Expand Up @@ -41,24 +41,27 @@ function vertical(ctx, domain)
return
end

local d = json.decode(resp)
if (d == nil or d.refpages == nil or #(d.refpages) == 0) then
local j = json.decode(resp)
if j == nil then
return
elseif j.error ~= nil then
log(ctx, "vertical request to service failed: " .. j.error)
return
end

for _, r in pairs(d.refpages) do
send_names(ctx, r.url_to)
for _, item in pairs(d.pages) do
send_names(ctx, item.url)
end
end

function build_url(domain, key)
local params = {
['target']=domain,
['token']=key,
['from']="backlinks",
['from']="ahrefs_rank",
['mode']="subdomains",
['limit']="1000",
['order_by']="first_seen%3Adesc",
['order_by']="ahrefs_rank%3Adesc",
['output']="json",
}

Expand Down
6 changes: 3 additions & 3 deletions resources/scripts/api/bgptools.ads
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ end

function origin(ctx, addr)
local conn, err = socket.connect(ctx, bgptoolsWhoisAddress, 43, "tcp")
if (err ~= nil and err ~= "") then
if (err ~= nil and err ~= "") then
log(ctx, "failed to connect to the whois server: " .. err)
return nil
end
Expand All @@ -66,9 +66,9 @@ function origin(ctx, addr)
end

local data
data, err = conn:recv_all()
data, err = conn:recv_all()
conn:close()
if (err ~= nil and err ~= "") then
if (err ~= nil and err ~= "") then
log(ctx, "failed to read the whois server response: " .. err)
return nil
end
Expand Down
160 changes: 160 additions & 0 deletions resources/scripts/api/deepinfo.ads
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
-- Copyright 2022 Jeff Foley. All rights reserved.
-- Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

local json = require("json")

name = "Deepinfo"
type = "api"

function start()
set_rate_limit(1)
end

function check()
local c
local cfg = datasrc_config()
if cfg ~= nil then
c = cfg.credentials
end

if (c ~= nil and c.key ~= nil and c.key ~= "") then
return true
end
return false
end

function vertical(ctx, domain)
local c
local cfg = datasrc_config()
if cfg ~= nil then
c = cfg.credentials
end

if (c == nil or c.key == nil or c.key == "") then
return
end

local p = 1
while(true) do
local resp, err = request(ctx, {
['url']=vert_url(domain, p),
['headers']={
['Accept']="application/json",
['apikey']=c.key,
},
})
if (err ~= nil and err ~= "") then
log(ctx, "vertical request to service failed: " .. err)
return
end

local j = json.decode(resp)
if (j == nil or j.results == nil) then
return
end

for _, r in pairs(j.results) do
new_name(ctx, r.punycode)
end

if j.result_count <= 100 * p then
break
end
p = p + 1
end
end

function vert_url(domain, pagenum)
return "https://api.deepinfo.com/v1/discovery/subdomain-finder?domain=" .. domain .. "&page=" .. pagenum
end

function horizontal(ctx, domain)
local c
local cfg = datasrc_config()
if cfg ~= nil then
c = cfg.credentials
end

if (c == nil or c.key == nil or c.key == "") then
return
end

local p = 1
while(true) do
local resp, err = request(ctx, {
['url']=horizon_url(domain, p),
['headers']={
['Accept']="application/json",
['apikey']=c.key,
},
})
if (err ~= nil and err ~= "") then
log(ctx, "vertical request to service failed: " .. err)
return
end

local j = json.decode(resp)
if (j == nil or j.results == nil) then
return
end

for _, r in pairs(j.results) do
associated(ctx, domain, r.punycode)
end

if j.result_count <= 100 * i then
break
end
p = p + 1
end
end

function horizon_url(domain, pagenum)
return "https://api.deepinfo.com/v1/discovery/associated-domain-finder?domain=" .. domain .. "&page=" .. pagenum
end

function asn(ctx, addr, asn)
local c
local cfg = datasrc_config()
if cfg ~= nil then
c = cfg.credentials
end

if (c == nil or c.key == nil or c.key == "") then
return
end

if (addr == nil or addr == "") then
return
end

local resp, err = request(ctx, {
['url']=asn_url(addr),
['headers']={
['Accept']="application/json",
['apikey']=c.key,
},
})
if (err ~= nil and err ~= "") then
log(ctx, "vertical request to service failed: " .. err)
return
end

local j = json.decode(resp)
if (j == nil or j.ipwhois == nil) then
return
end

new_asn(ctx, {
['addr']=addr,
['asn']=tonumber(string.sub(j.ipwhois.asn, 3)),
['desc']=j.ipwhois.asn_description,
['prefix']=j.ipwhois.asn_cidr,
['cc']=j.ipwhois.asn_country_code,
['registry']=j.ipwhois.asn_registry,
})
end

function asn_url(addr)
return "https://api.deepinfo.com/v1/lookup/ip-whois?ip=" .. addr
end
Loading

0 comments on commit 6acea62

Please sign in to comment.