diff --git a/README.md b/README.md index b0a87aa..1dc3f2e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,77 @@ # wiredog -a terminal-based monitor of local HTTP traffic + +Wiredog monitors the local HTTP traffic by scanning the +network packets. All network interfaces are monitored. +At least one active interface is needed. + +When the requests rate for a given period exceeds a +threshold, an alert message is displayed. The top 10 +website sections ordered by number of requests are displayed. + +Working and tested on Linux. OSX and Windows support is planned. + +

+ +

+ +``` +$ wiredog -help + +Wiredog is a tool for monitoring network traffic. + +Only local HTTP requests are studied. +Administrator permission is required to access network devices. +Consider using sudo on Linux or get admin rights on Windows. + +Usage: sudo \path\to\wiredog [flags] + + -assembly_debug_log + If true, the github.com/google/gopacket/tcpassembly library will log verbose debugging information (at least one line per packet) + -assembly_memuse_log + If true, the github.com/google/gopacket/tcpassembly library will log information regarding its memory use every once in a while. + -d string + network device name (default "wlp2s0") + -h int + threshold of HTTP request hits (default 2) + -log string + log directory (default ".") + -p duration + time period between traffic checks (default 2m0s) + -t generates test HTTP requests; sets h=2 and p=2s + + +``` + +## Built With (from go.mod file) +``` +Go 1.12 + +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c +github.com/gizak/termui v2.3.0+incompatible // indirect +github.com/gizak/termui/v3 v3.0.0 +github.com/google/gopacket v1.1.16 +github.com/maruel/panicparse v1.1.1 // indirect +github.com/mattn/go-runewidth v0.0.4 // indirect +github.com/mdlayher/raw v0.0.0-20190419142535-64193704e472 // indirect +github.com/mitchellh/go-wordwrap v1.0.0 // indirect +github.com/nsf/termbox-go v0.0.0-20190325093121-288510b9734e // indirect +github.com/onsi/ginkgo v1.8.0 // indirect +github.com/onsi/gomega v1.5.0 // indirect +github.com/pkg/errors v0.8.1 +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc // indirect +``` + +## Thanks + +gopacket and termui are cool Go packages. And of course, the Go programming language is beautiful. + +## Useful links + +https://www.devdungeon.com/content/packet-capture-injection-and-analysis-gopacket + +https://medium.com/@cjoudrey/capturing-http-packets-the-hard-way-b9c799bfb6 + +https://stackoverflow.com/questions/21145781/how-do-you-use-the-tcp-assembly-package-in-gopacket + + + diff --git a/cmd/wiredog/main.go b/cmd/wiredog/main.go new file mode 100644 index 0000000..028f154 --- /dev/null +++ b/cmd/wiredog/main.go @@ -0,0 +1,159 @@ +// Wiredog is a command for network monitoring. +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "code.cloudfoundry.org/clock" + ui "github.com/gizak/termui/v3" + "github.com/raypereda/wiredog/pkg/alerts" + asm "github.com/raypereda/wiredog/pkg/httpassembler" + "github.com/raypereda/wiredog/pkg/screens" + "github.com/raypereda/wiredog/pkg/stats" +) + +var ( + device string + hitThreshold int + logDirectory string + alertInterval time.Duration + testRequests bool +) + +func settings() string { + var sb strings.Builder + fmt.Fprintf(&sb, "Alert Interval : %s\n", alertInterval) + fmt.Fprintf(&sb, "Hit Threshold : %d\n", hitThreshold) + fmt.Fprintf(&sb, "Network Interface : %s\n", device) + absDirectory, err := filepath.Abs(logDirectory) + if err != nil { + log.Fatalf("error getting absolute path for %s: %v", logDirectory, err) + } + logDirectory = absDirectory + fmt.Fprintf(&sb, "Log Directory : %8s\n", logDirectory) + return sb.String() +} + +// Usage prints usage, overwrites default +var Usage = func() { + fmt.Fprintln(flag.CommandLine.Output(), "Wiredog is a tool for monitoring network traffic.") + fmt.Fprintln(flag.CommandLine.Output()) + fmt.Fprintln(flag.CommandLine.Output(), "Only local HTTP requests are studied.") + fmt.Fprintln(flag.CommandLine.Output(), "Administrator permission is required to access network devices.") + fmt.Fprintln(flag.CommandLine.Output(), "Consider using sudo on Linux or get admin rights on Windows.") + fmt.Fprintln(flag.CommandLine.Output()) + fmt.Fprintln(flag.CommandLine.Output(), "Usage: sudo \\path\\to\\wiredog [flags]") + fmt.Fprintln(flag.CommandLine.Output()) + flag.PrintDefaults() +} + +func main() { + // other network devices on my laptop: lo, enp0s31f6, but they don't work in promiscuous mode + flag.StringVar(&device, "d", "wlp2s0", "network device name") + flag.IntVar(&hitThreshold, "h", 2, "threshold of HTTP request hits") + flag.StringVar(&logDirectory, "log", ".", "log directory") + flag.DurationVar(&alertInterval, "p", 2*time.Minute, "time period between traffic checks") + flag.BoolVar(&testRequests, "t", false, "generates test HTTP requests; sets h=2 and p=2s") + flag.Usage = Usage + flag.Parse() + + logFile := logDirectory + "/wiredog.log" + f, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.Fatalf("error opening file: %v", err) + } + defer f.Close() + log.SetOutput(f) + + if err := ui.Init(); err != nil { + log.Fatalf("failed to initialize termui: %v", err) + } + defer ui.Close() + + if testRequests { + hitThreshold = 5 + alertInterval = 2 * time.Second + go makeTestRequests() + } + + requests := make(chan *http.Request) // receiver of requests + asm, err := asm.NewHTTPAssembler(device, requests) + if err != nil { + log.Fatalln("error connecting to network devices", err) + } + go asm.Run() + + s := screens.New(settings()) + + clock := clock.NewClock() + hist := alerts.NewHistory(clock, alertInterval) + // BUGFIX: avoid sharing screen state with alert, need to rethink + alert := alerts.New(clock, hist, alertInterval, hitThreshold, s) + go alert.Run() + + metrics := stats.New() + go func() { + for { + r := <-requests + metrics.Tally(r) // collects statistics + hist.Record(r) // keep history for alerter + } + }() + + go func() { + ticker := time.NewTicker(time.Second) + for { + <-ticker.C + s.UpdateTopN(metrics.GetTopSections()) + s.Render() + } + }() + + go func() { + ticker := time.NewTicker(alertInterval) + for { + <-ticker.C + count := float64(hist.RecentRequestCount()) + s.AddRequestCount(count) + s.Render() + } + }() + + s.Start() + log.Println("Stopped monitoring.") +} + +func get(url string) { + _, _ = http.Get(url) +} + +func makeTestRequests() { + time.Sleep(1 * time.Second) // allow alerts to setup + get("http://example1.com") + get("http://example2.com/sectionA/") + get("http://example2.com/sectionA/") + get("http://example3.com") + get("http://example3.com") + get("http://example3.com") + + for { + time.Sleep(time.Second) + get("http://example4.com") + get("http://example4.com") + get("http://example4.com") + get("http://example4.com") + time.Sleep(time.Second) + get("http://example5.com") + get("http://example5.com") + get("http://example5.com") + get("http://example5.com") + get("http://example5.com") + } +} diff --git a/cmd/wiredog/wiredog b/cmd/wiredog/wiredog new file mode 100755 index 0000000..6f7368e Binary files /dev/null and b/cmd/wiredog/wiredog differ diff --git a/cmd/wiredog/wiredog.log b/cmd/wiredog/wiredog.log new file mode 100644 index 0000000..cf88ae8 --- /dev/null +++ b/cmd/wiredog/wiredog.log @@ -0,0 +1,185 @@ +2019/05/03 15:57:52 Started monitoring. +2019/05/03 15:57:54 trimmed 0 requests from recent history +2019/05/03 15:57:54 number of requests: 1 +2019/05/03 15:57:55 trimmed 0 requests from recent history +2019/05/03 15:57:56 trimmed 0 requests from recent history +2019/05/03 15:57:56 number of requests: 8 +2019/05/03 15:57:56 High traffic generated an alert - hits = 8 +2019/05/03 15:57:57 trimmed 0 requests from recent history +2019/05/03 15:57:58 trimmed 1 requests from recent history +2019/05/03 15:57:58 number of requests: 4 +2019/05/03 15:57:58 Alert recovered; back to normal traffic +2019/05/03 15:57:59 trimmed 6 requests from recent history +2019/05/03 15:58:00 number of requests: 5 +2019/05/03 15:58:00 trimmed 2 requests from recent history +2019/05/03 15:58:01 trimmed 0 requests from recent history +2019/05/03 15:58:02 trimmed 4 requests from recent history +2019/05/03 15:58:02 number of requests: 4 +2019/05/03 15:58:03 trimmed 1 requests from recent history +2019/05/03 15:58:04 number of requests: 6 +2019/05/03 15:58:04 High traffic generated an alert - hits = 6 +2019/05/03 15:58:04 trimmed 4 requests from recent history +2019/05/03 15:58:05 trimmed 4 requests from recent history +2019/05/03 15:58:06 number of requests: 5 +2019/05/03 15:58:06 Alert recovered; back to normal traffic +2019/05/03 15:58:06 trimmed 0 requests from recent history +2019/05/03 15:58:07 trimmed 5 requests from recent history +2019/05/03 15:58:08 number of requests: 6 +2019/05/03 15:58:08 High traffic generated an alert - hits = 6 +2019/05/03 15:58:08 trimmed 1 requests from recent history +2019/05/03 15:58:09 trimmed 3 requests from recent history +2019/05/03 15:58:10 number of requests: 6 +2019/05/03 15:58:10 trimmed 2 requests from recent history +2019/05/03 15:58:11 trimmed 3 requests from recent history +2019/05/03 15:58:12 number of requests: 4 +2019/05/03 15:58:12 Alert recovered; back to normal traffic +2019/05/03 15:58:12 trimmed 3 requests from recent history +2019/05/03 15:58:13 trimmed 1 requests from recent history +2019/05/03 15:58:14 trimmed 5 requests from recent history +2019/05/03 15:58:14 number of requests: 5 +2019/05/03 15:58:15 trimmed 2 requests from recent history +2019/05/03 15:58:16 number of requests: 6 +2019/05/03 15:58:16 High traffic generated an alert - hits = 6 +2019/05/03 15:58:16 trimmed 2 requests from recent history +2019/05/03 15:58:17 trimmed 4 requests from recent history +2019/05/03 15:58:18 number of requests: 7 +2019/05/03 15:58:18 trimmed 1 requests from recent history +2019/05/03 15:58:19 trimmed 4 requests from recent history +2019/05/03 15:58:20 trimmed 2 requests from recent history +2019/05/03 15:58:20 number of requests: 5 +2019/05/03 15:58:20 Alert recovered; back to normal traffic +2019/05/03 15:58:21 trimmed 3 requests from recent history +2019/05/03 15:58:22 trimmed 4 requests from recent history +2019/05/03 15:58:22 number of requests: 4 +2019/05/03 15:58:23 trimmed 0 requests from recent history +2019/05/03 15:58:24 number of requests: 6 +2019/05/03 15:58:24 High traffic generated an alert - hits = 6 +2019/05/03 15:58:24 trimmed 5 requests from recent history +2019/05/03 15:58:25 trimmed 3 requests from recent history +2019/05/03 15:58:26 number of requests: 6 +2019/05/03 15:58:26 trimmed 1 requests from recent history +2019/05/03 15:58:27 trimmed 5 requests from recent history +2019/05/03 15:58:28 number of requests: 6 +2019/05/03 15:58:28 trimmed 1 requests from recent history +2019/05/03 15:58:29 trimmed 3 requests from recent history +2019/05/03 15:58:30 number of requests: 5 +2019/05/03 15:58:30 Alert recovered; back to normal traffic +2019/05/03 15:58:30 trimmed 3 requests from recent history +2019/05/03 15:58:31 trimmed 2 requests from recent history +2019/05/03 15:58:32 number of requests: 4 +2019/05/03 15:58:32 trimmed 4 requests from recent history +2019/05/03 15:58:33 trimmed 1 requests from recent history +2019/05/03 15:58:34 number of requests: 6 +2019/05/03 15:58:34 High traffic generated an alert - hits = 6 +2019/05/03 15:58:34 trimmed 4 requests from recent history +2019/05/03 15:58:35 trimmed 4 requests from recent history +2019/05/03 15:58:36 number of requests: 7 +2019/05/03 15:58:36 trimmed 0 requests from recent history +2019/05/03 15:58:37 trimmed 5 requests from recent history +2019/05/03 15:58:38 number of requests: 5 +2019/05/03 15:58:38 Alert recovered; back to normal traffic +2019/05/03 15:58:38 trimmed 1 requests from recent history +2019/05/03 15:58:39 trimmed 3 requests from recent history +2019/05/03 15:58:39 Stopped monitoring. +2019/05/03 15:59:22 Started monitoring. +2019/05/03 15:59:24 number of requests: 7 +2019/05/03 15:59:24 High traffic generated an alert - hits = 7 +2019/05/03 15:59:24 trimmed 0 requests from recent history +2019/05/03 15:59:25 trimmed 0 requests from recent history +2019/05/03 15:59:26 number of requests: 5 +2019/05/03 15:59:26 Alert recovered; back to normal traffic +2019/05/03 15:59:26 trimmed 0 requests from recent history +2019/05/03 15:59:27 trimmed 0 requests from recent history +2019/05/03 15:59:28 number of requests: 5 +2019/05/03 15:59:28 trimmed 7 requests from recent history +2019/05/03 15:59:29 trimmed 2 requests from recent history +2019/05/03 15:59:30 number of requests: 5 +2019/05/03 15:59:30 trimmed 3 requests from recent history +2019/05/03 15:59:31 trimmed 1 requests from recent history +2019/05/03 15:59:32 number of requests: 5 +2019/05/03 15:59:32 trimmed 4 requests from recent history +2019/05/03 15:59:33 trimmed 1 requests from recent history +2019/05/03 15:59:34 number of requests: 4 +2019/05/03 15:59:34 trimmed 4 requests from recent history +2019/05/03 15:59:35 trimmed 1 requests from recent history +2019/05/03 15:59:36 number of requests: 5 +2019/05/03 15:59:36 trimmed 4 requests from recent history +2019/05/03 15:59:37 trimmed 0 requests from recent history +2019/05/03 15:59:38 number of requests: 4 +2019/05/03 15:59:38 trimmed 4 requests from recent history +2019/05/03 15:59:39 trimmed 2 requests from recent history +2019/05/03 15:59:40 number of requests: 5 +2019/05/03 15:59:40 trimmed 3 requests from recent history +2019/05/03 15:59:41 trimmed 4 requests from recent history +2019/05/03 15:59:42 number of requests: 6 +2019/05/03 15:59:42 High traffic generated an alert - hits = 6 +2019/05/03 15:59:42 trimmed 0 requests from recent history +2019/05/03 15:59:43 trimmed 4 requests from recent history +2019/05/03 15:59:44 number of requests: 6 +2019/05/03 15:59:44 trimmed 1 requests from recent history +2019/05/03 15:59:45 trimmed 4 requests from recent history +2019/05/03 15:59:46 number of requests: 5 +2019/05/03 15:59:46 Alert recovered; back to normal traffic +2019/05/03 15:59:46 trimmed 2 requests from recent history +2019/05/03 15:59:47 trimmed 3 requests from recent history +2019/05/03 15:59:48 number of requests: 5 +2019/05/03 15:59:48 trimmed 3 requests from recent history +2019/05/03 15:59:49 trimmed 1 requests from recent history +2019/05/03 15:59:50 number of requests: 5 +2019/05/03 15:59:50 trimmed 4 requests from recent history +2019/05/03 15:59:51 trimmed 1 requests from recent history +2019/05/03 15:59:52 number of requests: 4 +2019/05/03 15:59:52 trimmed 4 requests from recent history +2019/05/03 15:59:53 trimmed 1 requests from recent history +2019/05/03 15:59:54 number of requests: 5 +2019/05/03 15:59:54 trimmed 4 requests from recent history +2019/05/03 15:59:55 trimmed 2 requests from recent history +2019/05/03 15:59:56 number of requests: 4 +2019/05/03 15:59:56 trimmed 2 requests from recent history +2019/05/03 15:59:57 trimmed 3 requests from recent history +2019/05/03 15:59:58 number of requests: 3 +2019/05/03 15:59:58 trimmed 2 requests from recent history +2019/05/03 15:59:59 trimmed 1 requests from recent history +2019/05/03 16:00:00 number of requests: 6 +2019/05/03 16:00:00 High traffic generated an alert - hits = 6 +2019/05/03 16:00:00 trimmed 3 requests from recent history +2019/05/03 16:00:01 trimmed 0 requests from recent history +2019/05/03 16:00:02 number of requests: 5 +2019/05/03 16:00:02 Alert recovered; back to normal traffic +2019/05/03 16:00:02 trimmed 3 requests from recent history +2019/05/03 16:00:03 trimmed 2 requests from recent history +2019/05/03 16:00:04 number of requests: 4 +2019/05/03 16:00:04 trimmed 4 requests from recent history +2019/05/03 16:00:05 trimmed 1 requests from recent history +2019/05/03 16:00:06 number of requests: 5 +2019/05/03 16:00:06 trimmed 4 requests from recent history +2019/05/03 16:00:07 trimmed 2 requests from recent history +2019/05/03 16:00:08 number of requests: 4 +2019/05/03 16:00:08 trimmed 2 requests from recent history +2019/05/03 16:00:09 trimmed 3 requests from recent history +2019/05/03 16:00:10 number of requests: 6 +2019/05/03 16:00:10 High traffic generated an alert - hits = 6 +2019/05/03 16:00:10 trimmed 2 requests from recent history +2019/05/03 16:00:11 trimmed 4 requests from recent history +2019/05/03 16:00:12 number of requests: 6 +2019/05/03 16:00:12 trimmed 0 requests from recent history +2019/05/03 16:00:13 trimmed 5 requests from recent history +2019/05/03 16:00:14 number of requests: 6 +2019/05/03 16:00:14 trimmed 1 requests from recent history +2019/05/03 16:00:15 trimmed 3 requests from recent history +2019/05/03 16:00:16 number of requests: 5 +2019/05/03 16:00:16 Alert recovered; back to normal traffic +2019/05/03 16:00:16 trimmed 3 requests from recent history +2019/05/03 16:00:17 trimmed 2 requests from recent history +2019/05/03 16:00:18 number of requests: 4 +2019/05/03 16:00:18 trimmed 4 requests from recent history +2019/05/03 16:00:19 trimmed 0 requests from recent history +2019/05/03 16:00:20 number of requests: 5 +2019/05/03 16:00:20 trimmed 5 requests from recent history +2019/05/03 16:00:21 trimmed 1 requests from recent history +2019/05/03 16:00:22 number of requests: 4 +2019/05/03 16:00:22 trimmed 3 requests from recent history +2019/05/03 16:00:23 trimmed 1 requests from recent history +2019/05/03 16:00:24 number of requests: 5 +2019/05/03 16:00:24 trimmed 4 requests from recent history +2019/05/03 16:00:25 Stopped monitoring. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..37c66df --- /dev/null +++ b/go.mod @@ -0,0 +1,19 @@ +module github.com/raypereda/wiredog + +go 1.12 + +require ( + code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c + github.com/gizak/termui v2.3.0+incompatible // indirect + github.com/gizak/termui/v3 v3.0.0 + github.com/google/gopacket v1.1.16 + github.com/maruel/panicparse v1.1.1 // indirect + github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/mdlayher/raw v0.0.0-20190419142535-64193704e472 // indirect + github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/nsf/termbox-go v0.0.0-20190325093121-288510b9734e // indirect + github.com/onsi/ginkgo v1.8.0 // indirect + github.com/onsi/gomega v1.5.0 // indirect + github.com/pkg/errors v0.8.1 + github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b4a0c2a --- /dev/null +++ b/go.sum @@ -0,0 +1,61 @@ +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ= +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +github.com/cjbassi/drawille-go v0.0.0-20190126131713-27dc511fe6fd h1:XtfPmj9tQRilnrEmI1HjQhxXWRhEM+m8CACtaMJE/kM= +github.com/cjbassi/drawille-go v0.0.0-20190126131713-27dc511fe6fd/go.mod h1:vjcQJUZJYD3MeVGhtZXSMnCHfUNZxsyYzJt90eCYxK4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gizak/termui v2.3.0+incompatible h1:S8wJoNumYfc/rR5UezUM4HsPEo3RJh0LKdiuDWQpjqw= +github.com/gizak/termui v2.3.0+incompatible/go.mod h1:PkJoWUt/zacQKysNfQtcw1RW+eK2SxkieVBtl+4ovLA= +github.com/gizak/termui/v3 v3.0.0 h1:NYTUG6ig/sJK05O5FyhWemwlVPO8ilNpvS/PgRtrKAE= +github.com/gizak/termui/v3 v3.0.0/go.mod h1:uinu2dMdtMI+FTIdEFUJQT5y+KShnhQRshvPblXq3lY= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/gopacket v1.1.16 h1:u6Afvia5C5srlLcbTwpHaFW918asLYPxieziOaWwz8M= +github.com/google/gopacket v1.1.16/go.mod h1:UCLx9mCmAwsVbn6qQl1WIEt2SO7Nd2fD0th1TBAsqBw= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/maruel/panicparse v1.1.1 h1:k62YPcEoLncEEpjMt92GtG5ugb8WL/510Ys3/h5IkRc= +github.com/maruel/panicparse v1.1.1/go.mod h1:nty42YY5QByNC5MM7q/nj938VbgPU7avs45z6NClpxI= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mdlayher/raw v0.0.0-20190419142535-64193704e472 h1:3QZ+iCYiorY0MXIia13+KK+uJHIsOJDHl7Y8Ul1Endc= +github.com/mdlayher/raw v0.0.0-20190419142535-64193704e472/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= +github.com/nsf/termbox-go v0.0.0-20190325093121-288510b9734e h1:Vbib8wJAaMEF9jusI/kMSYMr/LtRzM7+F9MJgt/nH8k= +github.com/nsf/termbox-go v0.0.0-20190325093121-288510b9734e/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc h1:LUUe4cdABGrIJAhl1P1ZpWY76AwukVszFdwkVFVLwIk= +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190419010253-1f3472d942ba h1:h0zCzEL5UW1mERvwTN6AXcc75PpLkY6OcReia6Dq1BM= +golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be h1:mI+jhqkn68ybP0ORJqunXn+fq+Eeb4hHKqLQcFICjAc= +golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/alerts/alerts.go b/pkg/alerts/alerts.go new file mode 100644 index 0000000..fc78193 --- /dev/null +++ b/pkg/alerts/alerts.go @@ -0,0 +1,80 @@ +// Package alerts provides alerts based on history of events. +package alerts + +import ( + "fmt" + "log" + "sync" + "time" + + // "github.com/jonboulle/clockwork" + "code.cloudfoundry.org/clock" + "github.com/raypereda/wiredog/pkg/screens" +) + +// Alerter is based on number of recent requests. +type Alerter struct { + sync.Mutex + isActivated bool + History *History + Events []*Event + checkInterval time.Duration + threshold int + clock clock.Clock + screen *screens.Screen +} + +// New creates a new alerter. +func New(c clock.Clock, h *History, i time.Duration, t int, s *screens.Screen) *Alerter { + return &Alerter{ + History: h, + checkInterval: i, + threshold: t, + clock: c, + screen: s, + } +} + +// Run starts the process of periodically processing the alert. +func (a *Alerter) Run() { + ticker := time.NewTicker(a.checkInterval) + // TODO: check if need to sleep for first interval + for { + <-ticker.C + a.Check() + } +} + +// IsActivated ... +func (a *Alerter) IsActivated() bool { + a.Lock() + defer a.Unlock() + return a.isActivated +} + +// Check activates and clears the alert as needed +func (a *Alerter) Check() { + a.Lock() + defer a.Unlock() + numRequests := a.History.RecentRequestCount() + log.Println("number of requests:", numRequests) + if !a.isActivated { + if numRequests > a.threshold { + msg := fmt.Sprintf("High traffic generated an alert - hits = %d", numRequests) + if a.screen != nil { // for easier testing + a.screen.LogPrintln(msg) + a.screen.Render() + } + a.isActivated = true + } + } else { + if numRequests <= a.threshold { + msg := "Alert recovered; back to normal traffic" + if a.screen != nil { // for easier testing + a.screen.LogPrintln(msg) + a.screen.Render() + } + a.isActivated = false + } + } +} diff --git a/pkg/alerts/alerts_test.go b/pkg/alerts/alerts_test.go new file mode 100644 index 0000000..1daa24b --- /dev/null +++ b/pkg/alerts/alerts_test.go @@ -0,0 +1,47 @@ +package alerts_test + +import ( + "net/http" + "testing" + "time" + + // "github.com/jonboulle/clockwork" + "code.cloudfoundry.org/clock/fakeclock" + + "github.com/raypereda/wiredog/pkg/alerts" + "github.com/raypereda/wiredog/pkg/screens" +) + +func TestAlerter(t *testing.T) { + clock := fakeclock.NewFakeClock(time.Now()) + interval := time.Second + threshold := 2 + hist := alerts.NewHistory(clock, interval) + screen := screens.New("fake") + alerter := alerts.New(clock, hist, interval, threshold, screen) + + alerter.Check() + if alerter.IsActivated() { + t.Fatal("Alerter should NOT be activated") + } + + r := &http.Request{} + alerter.History.Record(r) + alerter.History.Record(r) + alerter.Check() + if alerter.IsActivated() { + t.Fatal("Alerter should NOT be activated") + } + + alerter.History.Record(r) + alerter.Check() + if !alerter.IsActivated() { + t.Fatal("Alerter should be activated") + } + + clock.Increment(2 * time.Second) + alerter.Check() + if alerter.IsActivated() { + t.Fatal("Alerter should have cleared") + } +} diff --git a/pkg/alerts/events.go b/pkg/alerts/events.go new file mode 100644 index 0000000..1129d00 --- /dev/null +++ b/pkg/alerts/events.go @@ -0,0 +1,20 @@ +package alerts + +import ( + "time" +) + +// Event is message with a time stamp +type Event struct { + message string + time time.Time // when added +} + +// AddEvent adds an event to store. +func AddEvent(events []*Event, message string) { + e := &Event{ + message: message, + time: time.Now(), + } + events = append(events, e) +} diff --git a/pkg/alerts/history.go b/pkg/alerts/history.go new file mode 100644 index 0000000..047a13c --- /dev/null +++ b/pkg/alerts/history.go @@ -0,0 +1,99 @@ +package alerts + +import ( + "log" + "net/http" + "sync" + "time" + + // "github.com/jonboulle/clockwork" + // "code.cloudfoundry.org/clock/fakeclock" + "code.cloudfoundry.org/clock" +) + +// History stores a list of recent requests. +type History struct { + sync.Mutex + requests []*request + duration time.Duration + clock clock.Clock +} + +// NewHistory creates a new instance of History. +// A clock is passed in to facilate testing. +func NewHistory(c clock.Clock, d time.Duration) *History { + h := &History{ + duration: d, + clock: c, + } + go h.periodicallyTidy() + return h +} + +// Record processes the HTTP request. +func (h *History) Record(req *http.Request) { + h.Lock() + defer h.Unlock() + + r := &request{ + request: req, + recordTime: h.clock.Now(), + } + h.requests = append(h.requests, r) +} + +func (h *History) periodicallyTidy() { + ticker := h.clock.NewTicker(time.Second) + for { + <-ticker.C() + h.Tidy() + } +} + +// Tidy deletes requests that are old. +func (h *History) Tidy() { + h.Lock() + defer h.Unlock() + + if len(h.requests) == 0 { + return + } + // keep two time periods to keep steady RecentRequestCount + startTime := h.clock.Now().Add(-2 * h.duration) + var skip int + for _, request := range h.requests { + if request.recordTime.After(startTime) { + break + } + skip++ + } + log.Printf("trimmed %d requests from recent history\n", skip) + h.requests = h.requests[skip:] +} + +// RecentRequestCount returns number of requests since last watch interval. +func (h *History) RecentRequestCount() int { + h.Lock() + defer h.Unlock() + + startTime := h.clock.Now().Add(-h.duration) + var count int + + // the most recent requests are on the end + for i := len(h.requests) - 1; i >= 0; i-- { + request := h.requests[i] + // for _, request := range h.requests { + if request.recordTime.After(startTime) { + count++ + } else { // requests are ordered by record time and + break // serialized with a channel + } + } + return count +} + +// request represents a request with a timestamp. +type request struct { + request *http.Request + recordTime time.Time +} diff --git a/pkg/alerts/history_test.go b/pkg/alerts/history_test.go new file mode 100644 index 0000000..c2b4c03 --- /dev/null +++ b/pkg/alerts/history_test.go @@ -0,0 +1,44 @@ +package alerts_test + +import ( + "net/http" + "testing" + "time" + + // This clockwork did not work out well. + // "github.com/jonboulle/clockwork" + "code.cloudfoundry.org/clock/fakeclock" + "github.com/raypereda/wiredog/pkg/alerts" +) + +func TestCacheRequests(t *testing.T) { + clk := fakeclock.NewFakeClock(time.Now()) + history := alerts.NewHistory(clk, time.Second) + + r := &http.Request{} + history.Record(r) + history.Record(r) + + c := history.RecentRequestCount() + if c != 2 { + t.Error("Expected 2 recent requests, got", c) + } + + clk.IncrementBySeconds(1) + // history.Tidy() automatically called. + + history.Record(r) + c = history.RecentRequestCount() + if c != 1 { + t.Error("Expected 1 recent requests, got", c) + } + + clk.IncrementBySeconds(2) + // history.Tidy() automatically called. + + c = history.RecentRequestCount() + if c != 0 { + t.Error("Expected 0 recent requests, got", c) + } + +} diff --git a/pkg/httpassembler/assembler.go b/pkg/httpassembler/assembler.go new file mode 100644 index 0000000..104cdb9 --- /dev/null +++ b/pkg/httpassembler/assembler.go @@ -0,0 +1,131 @@ +package httpassembler + +// The code in this file is based on the example from the gopacket package +// https://github.com/google/gopacket/blob/master/examples/httpassembly/main.go + +import ( + "bufio" + "io" + "log" + "net/http" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" + "github.com/google/gopacket/tcpassembly" + "github.com/google/gopacket/tcpassembly/tcpreader" + "github.com/pkg/errors" +) + +type httpStreamFactory struct { + requests chan<- *http.Request +} + +func (h *httpStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream { + hstream := &httpStream{ + net: net, + transport: transport, + r: tcpreader.NewReaderStream(), + requests: h.requests, + } + go hstream.run() + return &hstream.r +} + +type httpStream struct { + requests chan<- *http.Request + net, transport gopacket.Flow + r tcpreader.ReaderStream +} + +func (h *httpStream) run() { + buf := bufio.NewReader(&h.r) + for { + req, err := http.ReadRequest(buf) + if err == io.EOF { + return + } else if err != nil { + log.Println("error reading stream", h.net, h.transport, ":", err) + } else { + req.Body.Close() + h.requests <- req + } + } +} + +// HTTPAssembler assembles HTTP requests from a local network packets. +type HTTPAssembler struct { + device string + handle *pcap.Handle + assembler *tcpassembly.Assembler +} + +// NewHTTPAssembler creates a new source of assembled HTTP requests from packets. +// The requests are passed back via the provided channel. +func NewHTTPAssembler(device string, requests chan<- *http.Request) (*HTTPAssembler, error) { + maxReadSize := int32(0) // unbounded + promiscuous := true // put device in mode to read all packets on all devices + timeout := pcap.BlockForever + handle, err := pcap.OpenLive(device, maxReadSize, promiscuous, timeout) + if err != nil { + return nil, errors.Wrapf(err, "error opening pcap handle") + } + + filter := "tcp and dst port 80" // BPF filter for pcap looking for HTTP requests + err = handle.SetBPFFilter(filter) + if err != nil { + return nil, errors.Wrapf(err, "error setting BPF filter") + } + + streamFactory := &httpStreamFactory{requests: requests} + streamPool := tcpassembly.NewStreamPool(streamFactory) + assembler := tcpassembly.NewAssembler(streamPool) + + return &HTTPAssembler{ + device: device, + handle: handle, + assembler: assembler, + }, nil +} + +// Run starts the HTTP request sniffing process. +func (r *HTTPAssembler) Run() { + defer r.handle.Close() + packetSource := gopacket.NewPacketSource(r.handle, r.handle.LinkType()) + packets := packetSource.Packets() + + ticker := time.Tick(time.Minute) + for { + select { + case packet := <-packets: + if packet == nil { + return + } + if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || + packet.TransportLayer().LayerType() != layers.LayerTypeTCP { + continue + } + + tcp := packet.TransportLayer().(*layers.TCP) + r.assembler.AssembleWithTimestamp( + packet.NetworkLayer().NetworkFlow(), + tcp, time.Now()) + // tcp, packet.Metadata().Timestamp) + // The metadata timestamp is often empty. + + // Experiment: Identify HTTP requests by looking in payload. + // Decided against it and rely on Berkeley Packet Filter instead. + // Search for a string inside the payload. + // applicationLayer := packet.ApplicationLayer() + // if applicationLayer != nil && strings.Contains(string(applicationLayer.Payload()), "HTTP") { + // fmt.Println("HTTP found!") + // } + + case <-ticker: + // Every minute, flush connections that haven't seen + // activity in the past 2 minutes. + r.assembler.FlushOlderThan(time.Now().Add(time.Minute * -2)) + } + } +} diff --git a/pkg/httpassembler/httpassembler_test.go b/pkg/httpassembler/httpassembler_test.go new file mode 100644 index 0000000..d53f407 --- /dev/null +++ b/pkg/httpassembler/httpassembler_test.go @@ -0,0 +1,36 @@ +package httpassembler_test + +import ( + "net/http" + "testing" + "time" + + asm "github.com/raypereda/wiredog/pkg/httpassembler" +) + +func TestAssembler(t *testing.T) { + requests := make(chan *http.Request) + // Here are the other devices on my laptop: lo, enp0s31f6 + asm, err := asm.NewHTTPAssembler("wlp2s0", requests) + if err != nil { + t.Fatal(err) + } + go asm.Run() + + _, err = http.Get("http://example.com") + if err != nil { + t.Fatal(err) + } + + timer := time.NewTimer(3 * time.Second) + for { + select { + case <-timer.C: + t.Fatal("http request timed out") + case request := <-requests: + if request.Host == "example.com" { + return + } + } + } +} diff --git a/pkg/screens/screens.go b/pkg/screens/screens.go new file mode 100644 index 0000000..0ab47a2 --- /dev/null +++ b/pkg/screens/screens.go @@ -0,0 +1,156 @@ +// Package screens manages a terminal screen. +package screens + +import ( + "fmt" + "log" + "time" + + ui "github.com/gizak/termui/v3" + "github.com/gizak/termui/v3/widgets" +) + +// topN sets the top *N* websites +const topN = 10 + +// Screen is a terminal interface +type Screen struct { + sparkline *widgets.Sparkline + topN *widgets.List + messages *widgets.List + grid *ui.Grid +} + +// New creates a new screen +func New(settings string) *Screen { + if settings == "fake" { + return nil + } + p1 := widgets.NewParagraph() + + p1.Title = "WireDog - monitors local HTTP network traffic" + p1.Border = false + + p2 := widgets.NewParagraph() + p2.Title = "Settings" + p2.Text = settings + + sl := widgets.NewSparkline() + sl.LineColor = ui.ColorGreen + slg := widgets.NewSparklineGroup(sl) + slg.Title = "Traffic Trend" + + l1 := widgets.NewList() + l1.Title = "Top " + string(topN) + " Website By Hits" + l1.WrapText = false + + l2 := widgets.NewList() + l2.Title = "messages" + l2.TextStyle = ui.NewStyle(ui.ColorClear, ui.ColorClear, ui.ModifierClear) // for scrolling + l2.WrapText = false + l2.ScrollDown() + + grid := ui.NewGrid() + termWidth, termHeight := ui.TerminalDimensions() + grid.SetRect(0, 0, termWidth, termHeight) + + grid.Set( + ui.NewRow(0.05, + ui.NewCol(1, p1), + ), + ui.NewRow(0.25, + ui.NewCol(1.0/2, p2), + ui.NewCol(1.0/2, slg), + ), + ui.NewRow(0.7/2, + ui.NewCol(1.0, l1), + ), + ui.NewRow(0.7/2, + ui.NewCol(1.0, l2), + ), + ) + return &Screen{ + sparkline: sl, + topN: l1, + messages: l2, + grid: grid, + } +} + +// UpdateTopN updates the top N sections +func (s *Screen) UpdateTopN(rows []string) { + s.topN.Rows = rows +} + +// AddRequestCount add to the sparkline data +func (s *Screen) AddRequestCount(d float64) { + s.sparkline.Data = append(s.sparkline.Data, d) + // experimented with value at full screen on laptop + lastN := 83 + if len(s.sparkline.Data) > lastN { // plot most recent + skip := len(s.sparkline.Data) - lastN + s.sparkline.Data = s.sparkline.Data[skip:] + } +} + +// LogPrintln adds to the standard logger and the message wideget +func (s *Screen) LogPrintln(msg string) { + log.Println(msg) + now := time.Now() // TODO: share exact time with log and messages widget + line := fmt.Sprintf("%s %s", now.Format("1968-05-29 15:04:05"), msg) + + s.messages.Rows = append(s.messages.Rows, line) + s.messages.ScrollBottom() +} + +// Render renders the whole screen +func (s *Screen) Render() { + ui.Render(s.grid) +} + +// Start kicks off rendering loop +func (s *Screen) Start() { + s.LogPrintln("Started monitoring.") + s.messages.ScrollBottom() + ui.Render(s.grid) + previousKey := "" + uiEvents := ui.PollEvents() + for { + e := <-uiEvents + switch e.ID { + case "q", "": + return + case "j", "": + s.messages.ScrollDown() + case "k", "": + s.messages.ScrollUp() + case "": + s.messages.ScrollHalfPageDown() + case "": + s.messages.ScrollHalfPageUp() + case "": + s.messages.ScrollPageDown() + case "": + s.messages.ScrollPageUp() + case "g": + if previousKey == "g" { + s.messages.ScrollTop() + } + case "": + s.messages.ScrollTop() + case "G", "": + s.messages.ScrollBottom() + case "": + payload := e.Payload.(ui.Resize) + s.grid.SetRect(0, 0, payload.Width, payload.Height) + ui.Clear() + ui.Render(s.grid) + } + if previousKey == "g" { + previousKey = "" + } else { + previousKey = e.ID + } + ui.Render(s.grid) + } +} diff --git a/pkg/stats/example_test.go b/pkg/stats/example_test.go new file mode 100644 index 0000000..362eb87 --- /dev/null +++ b/pkg/stats/example_test.go @@ -0,0 +1,61 @@ +package stats_test + +import ( + "fmt" + "net/url" + "strings" + + "github.com/raypereda/wiredog/pkg/stats" +) + +func ExampleGetTopSections() { + m := map[string]int{ + "google.com/sectionA": 60, + "google.com/sectionB": 70, + "google.com/sectionC": 80, + "google.com/sectionD": 90, + "google.com": 100, + "amazon.com/sectionA": 55, + "amazon.com/sectionB": 65, + "amazon.com/sectionC": 75, + "amazon.com/sectionD": 85, + "amazon.com": 100, + } + + s := stats.New() + s.SectionCounts = m + + topSections := s.GetTopSections() + + fmt.Print(strings.Join(topSections, "\n")) + // Output: + // 100 amazon.com + // 100 google.com + // 90 google.com/sectionD + // 85 amazon.com/sectionD + // 80 google.com/sectionC + // 75 amazon.com/sectionC + // 70 google.com/sectionB + // 65 amazon.com/sectionB + // 60 google.com/sectionA + // 55 amazon.com/sectionA +} + +func ExampleGetSection() { + urls := []string{ + "http://datadog.com", + "http://datadog.com/", + "http://datadog.com/section/", + "http://datadog.com/section/misc", + } + for _, u := range urls { + url, _ := url.Parse(u) + section := stats.GetSection(url) + fmt.Println(u, "==>", section) + } + // Output: + // http://datadog.com ==> datadog.com + // http://datadog.com/ ==> datadog.com + // http://datadog.com/section/ ==> datadog.com/section + // http://datadog.com/section/misc ==> datadog.com/section +} diff --git a/pkg/stats/metrics.go b/pkg/stats/metrics.go new file mode 100644 index 0000000..6c72571 --- /dev/null +++ b/pkg/stats/metrics.go @@ -0,0 +1,83 @@ +// Package stats collects statistics of HTTP requests. +package stats + +import ( + "fmt" + "net/http" + "net/url" + "sort" + "strings" + "sync" +) + +// Metrics stores HTTP metrics for a stream of requests. +type Metrics struct { + sync.Mutex + SectionCounts map[string]int + // TODO: metrics like body length, first line of body +} + +// New creates a new set of metrics. +func New() *Metrics { + return &Metrics{ + SectionCounts: make(map[string]int), + } +} + +// Tally adds a request to the stats +func (m *Metrics) Tally(req *http.Request) { + m.Lock() + defer m.Unlock() + req.URL.Host = req.Host + section := GetSection(req.URL) + m.SectionCounts[section]++ // TODO: add more metrics +} + +// GetSection returns the "section", which is defined as the host and +// up to the first part of the path before the second /. +func GetSection(url *url.URL) string { + path := url.EscapedPath() + if path == "" || path == "/" { + return url.Host + } + section := strings.SplitN(path, "/", 3)[1] + return url.Host + "/" + section +} + +// entry represents a key and value entry +type entry struct { + name string + count int +} + +// getTopNEntries sorts the map by value then returns top N +func getTopNEntries(m map[string]int, topN int) []*entry { + var entries []*entry + for k, v := range m { + entries = append(entries, &entry{k, v}) + } + isMore := func(i, j int) bool { + if entries[i].count == entries[j].count { + return entries[i].name < entries[j].name + } + return entries[i].count > entries[j].count + } + sort.Slice(entries, isMore) + if topN > len(entries) { + return entries + } + return entries[:topN] +} + +// GetTopSections returns the top 10 sections +func (m *Metrics) GetTopSections() []string { + m.Lock() + defer m.Unlock() + entries := getTopNEntries(m.SectionCounts, 10) + var top []string + for _, entry := range entries { + s := fmt.Sprintf("%3d %s", entry.count, entry.name) + top = append(top, s) + } + return top +} diff --git a/wiredog-demo.gif b/wiredog-demo.gif new file mode 100644 index 0000000..41f1815 Binary files /dev/null and b/wiredog-demo.gif differ diff --git a/wiredog-otis.png b/wiredog-otis.png new file mode 100644 index 0000000..2759aa2 Binary files /dev/null and b/wiredog-otis.png differ