diff --git a/datasrcs/sources.go b/datasrcs/sources.go index b4d879083..a2f23617a 100644 --- a/datasrcs/sources.go +++ b/datasrcs/sources.go @@ -24,7 +24,6 @@ func GetAllSources(sys systems.System) []service.Service { NewDNSDB(sys), NewNetworksDB(sys), NewRADb(sys), - NewTwitter(sys), NewUmbrella(sys), } diff --git a/datasrcs/twitter.go b/datasrcs/twitter.go deleted file mode 100644 index 1aa971a7f..000000000 --- a/datasrcs/twitter.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright © by Jeff Foley 2017-2022. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. -// SPDX-License-Identifier: Apache-2.0 - -package datasrcs - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - - "github.com/OWASP/Amass/v3/config" - "github.com/OWASP/Amass/v3/net/http" - "github.com/OWASP/Amass/v3/requests" - "github.com/OWASP/Amass/v3/systems" - "github.com/caffix/service" - "github.com/dghubble/go-twitter/twitter" - "golang.org/x/oauth2" -) - -// Twitter is the Service that handles access to the Twitter data source. -type Twitter struct { - service.BaseService - - SourceType string - sys systems.System - creds *config.Credentials - client *twitter.Client -} - -// NewTwitter returns he object initialized, but not yet started. -func NewTwitter(sys systems.System) *Twitter { - t := &Twitter{ - SourceType: requests.API, - sys: sys, - } - - go t.requests() - t.BaseService = *service.NewBaseService(t, "Twitter") - return t -} - -// Description implements the Service interface. -func (t *Twitter) Description() string { - return t.SourceType -} - -// OnStart implements the Service interface. -func (t *Twitter) OnStart() error { - t.creds = t.sys.Config().GetDataSourceConfig(t.String()).GetCredentials() - - if t.creds == nil || t.creds.Key == "" || t.creds.Secret == "" { - t.sys.Config().Log.Printf("%s: API key data was not provided", t.String()) - } else { - if bearer, err := t.getBearerToken(); err == nil { - config := &oauth2.Config{} - token := &oauth2.Token{AccessToken: bearer} - // OAuth2 http.Client will automatically authorize Requests - httpClient := config.Client(context.Background(), token) - // Twitter client - t.client = twitter.NewClient(httpClient) - } - } - - t.SetRateLimit(1) - return t.checkConfig() -} - -// CheckConfig implements the Service interface. -func (t *Twitter) checkConfig() error { - creds := t.sys.Config().GetDataSourceConfig(t.String()).GetCredentials() - - if creds == nil || creds.Key == "" || creds.Secret == "" { - estr := fmt.Sprintf("%s: check callback failed for the configuration", t.String()) - t.sys.Config().Log.Print(estr) - return errors.New(estr) - } - - return nil -} - -func (t *Twitter) requests() { - for { - select { - case <-t.Done(): - return - case in := <-t.Input(): - switch req := in.(type) { - case *requests.DNSRequest: - t.CheckRateLimit() - t.dnsRequest(context.TODO(), req) - } - } - } -} - -func (t *Twitter) dnsRequest(ctx context.Context, req *requests.DNSRequest) { - re := t.sys.Config().DomainRegex(req.Domain) - if t.client == nil || re == nil { - return - } - - numRateLimitChecks(t, 2) - t.sys.Config().Log.Printf("Querying %s for %s subdomains", t.String(), req.Domain) - - searchParams := &twitter.SearchTweetParams{ - Query: req.Domain, - Count: 100, - } - search, _, err := t.client.Search.Tweets(searchParams) - if err != nil { - t.sys.Config().Log.Printf("%s: %v", t.String(), err) - return - } - - for _, tweet := range search.Statuses { - // URLs in the tweet body - for _, urlEntity := range tweet.Entities.Urls { - for _, name := range re.FindAllString(urlEntity.ExpandedURL, -1) { - genNewNameEvent(ctx, t.sys, t, name) - } - } - // Source of the tweet - for _, name := range re.FindAllString(tweet.Source, -1) { - genNewNameEvent(ctx, t.sys, t, name) - } - } -} - -func (t *Twitter) getBearerToken() (string, error) { - headers := map[string]string{"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"} - page, err := http.RequestWebPage(context.Background(), "https://api.twitter.com/oauth2/token", - strings.NewReader("grant_type=client_credentials"), headers, - &http.BasicAuth{ - Username: t.creds.Key, - Password: t.creds.Secret, - }) - if err != nil { - return "", fmt.Errorf("token request failed: %+v", err) - } - - var v struct { - AccessToken string `json:"access_token"` - } - if err := json.Unmarshal([]byte(page), &v); err != nil { - return "", fmt.Errorf("error parsing json in token response: %+v", err) - } - if v.AccessToken == "" { - return "", fmt.Errorf("token response does not have access_token") - } - return v.AccessToken, nil -} diff --git a/examples/config.ini b/examples/config.ini index 0a7cbc4e9..d877b2e69 100644 --- a/examples/config.ini +++ b/examples/config.ini @@ -1,4 +1,4 @@ -# Copyright © by Jeff Foley 2017-2022. All rights reserved. +# Copyright © by Jeff Foley 2017-2023. All rights reserved. # Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. # SPDX-License-Identifier: Apache-2.0 @@ -377,16 +377,6 @@ minimum_ttl = 1440 ; One day #[data_sources.ThreatBook.account1] #apikey= -# https://developer.twitter.com (Free) -# Provide your Twitter App Consumer API key and Consumer API secret key -#[data_sources.Twitter] -#[data_sources.Twitter.account1] -#apikey = -#secret = -#[data_sources.Twitter.account2] -#apikey = -#secret = - # https://umbrella.cisco.com (Paid-Enterprise) # The apikey must be an API access token created through the Investigate management UI #[data_sources.Umbrella] diff --git a/go.mod b/go.mod index 6fa97ec2e..541772aa3 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/cayleygraph/quad v1.2.4 github.com/cjoudrey/gluaurl v0.0.0-20161028222611-31cbb9bef199 github.com/cloudflare/cloudflare-go v0.49.0 - github.com/dghubble/go-twitter v0.0.0-20221104224141-912508c3888b github.com/fatih/color v1.13.0 github.com/geziyor/geziyor v0.0.0-20221223163247-7349b81754b8 github.com/go-ini/ini v1.67.0 @@ -24,7 +23,6 @@ require ( github.com/yl2chen/cidranger v1.0.2 github.com/yuin/gopher-lua v1.0.0 golang.org/x/net v0.5.0 - golang.org/x/oauth2 v0.4.0 layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf ) @@ -36,14 +34,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/cayleygraph/cayley v0.7.7-0.20220304214302-275a7428fb10 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chromedp/cdproto v0.0.0-20230104010638-078e50cebfff // indirect github.com/chromedp/chromedp v0.8.6 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dennwc/base v1.0.0 // indirect - github.com/dghubble/sling v1.4.0 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect @@ -94,7 +90,6 @@ require ( golang.org/x/text v0.6.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.5.0 // indirect - google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e1b1df404..210874152 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,6 @@ github.com/cayleygraph/quad v1.2.4 h1:+6u09WxA7zg9ILonK8DChwzWKLKsDkjyvX+CXXhI/m github.com/cayleygraph/quad v1.2.4/go.mod h1:XOianlRdDK5Upno/6svE6APe/wD8XgYrL9smqK875nU= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -171,10 +169,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dennwc/base v1.0.0 h1:xlBzvBNRvkQ1LFI/jom7rr0vZsvYDKtvMM6lIpjFb3M= github.com/dennwc/base v1.0.0/go.mod h1:zaTDIiAcg2oKW9XhjIaRc1kJVteCFXSSW6jwmCedUaI= github.com/dennwc/graphql v0.0.0-20180603144102-12cfed44bc5d/go.mod h1:lg9KQn0BgRCSCGNpcGvJp/0Ljf1Yxk8TZq9HSYc43fk= -github.com/dghubble/go-twitter v0.0.0-20221104224141-912508c3888b h1:XQu6o3AwJx/jsg9LZ41uIeUdXK5be099XFfFn6H9ikk= -github.com/dghubble/go-twitter v0.0.0-20221104224141-912508c3888b/go.mod h1:B0/qdW5XUupJvcsx40hnVbfjzz9He5YpYXx6eVVdiSY= -github.com/dghubble/sling v1.4.0 h1:/n8MRosVTthvMbwlNZgLx579OGVjUOy3GNEv5BIqAWY= -github.com/dghubble/sling v1.4.0/go.mod h1:0r40aNsU9EdDUVBNhfCstAtFgutjgJGYbO1oNzkMoM8= github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.5.5/go.mod h1:QgCntgIUPsjnp7cMLhUybJHb7iIoQWAHT6tF8ngCjWk= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= @@ -897,8 +891,6 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1123,7 +1115,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=