Add --filter-track-bpf-helpers

When --filter-track-bpf-helpers is set, --filter-track-skb-by-stackid is enabled automatically. Using --filter-track-bpf-helpers and --output-caller together can roughly trace bpf tailcalls, such as: ``` 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) tcf_classify sch_handle_ingress.constprop.0 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_skb_event_output bpf_prog_306e2b9160b2fb74_cil_from_container[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_skb_pull_data bpf_prog_a76cce51ceb41c61_tail_handle_ipv4[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) skb_ensure_writable bpf_skb_pull_data 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_skb_load_bytes bpf_prog_a76cce51ceb41c61_tail_handle_ipv4[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_a76cce51ceb41c61_tail_handle_ipv4[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_skb_load_bytes bpf_prog_5fd02288bd4a682e_tail_ipv4_ct_egress[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_skb_load_bytes bpf_prog_5fd02288bd4a682e_tail_ipv4_ct_egress[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_map_lookup_elem bpf_prog_5fd02288bd4a682e_tail_ipv4_ct_egress[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_map_lookup_elem bpf_prog_5fd02288bd4a682e_tail_ipv4_ct_egress[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) array_map_update_elem bpf_prog_5fd02288bd4a682e_tail_ipv4_ct_egress[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) trie_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) percpu_array_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 ~bin/curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_ktime_get_ns bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_map_update_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 curl:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) htab_lru_map_update_elem bpf_map_update_elem 0xffff97cdbd7138e8 3 <empty>:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) htab_lru_map_update_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 <empty>:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) __htab_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 <empty>:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) htab_percpu_map_lookup_elem bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 <empty>:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) bpf_skb_event_output bpf_prog_4f62c9416e340c61_tail_handle_ipv4_cont[bpf] 0xffff97cdbd7138e8 3 <empty>:227116 10.244.1.141:45000->10.244.3.20:8080(tcp) ip_rcv __netif_receive_skb_one_core ``` Signed-off-by: gray <gray.liang@isovalent.com>
cilium · Sep 2, 2024 · 7deb4a0 · 7deb4a0
1 parent fc7e097
commit 7deb4a0
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 22 deletions.
diff --git a/internal/pwru/kprobe.go b/internal/pwru/kprobe.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"slices"
 	"sync"
 	"syscall"
 
@@ -245,6 +246,9 @@ func NewKprober(ctx context.Context, funcs Funcs, coll *ebpf.Collection, a2n Add
 }
 
 func NewNonSkbFuncsKprober(nonSkbFuncs []string, funcs Funcs, coll *ebpf.Collection) *kprober {
+	slices.Sort(nonSkbFuncs)
+	nonSkbFuncs = slices.Compact(nonSkbFuncs)
+
 	var k kprober
 	k.kprobeBatch = uint(len(nonSkbFuncs))
 
@@ -255,10 +259,13 @@ func NewNonSkbFuncsKprober(nonSkbFuncs []string, funcs Funcs, coll *ebpf.Collect
 
 		kp, err := link.Kprobe(fn, coll.Programs["kprobe_skb_by_stackid"], nil)
 		if err != nil {
-			log.Fatalf("Opening kprobe %s: %s\n", fn, err)
+			if errors.Is(err, os.ErrNotExist) {
+				continue
+			}
+			log.Printf("Opening non-skb-kprobe %s: %s\n", fn, err)
+		} else {
+			k.links = append(k.links, kp)
 		}
-
-		k.links = append(k.links, kp)
 	}
 
 	return &k

diff --git a/internal/pwru/ksym.go b/internal/pwru/ksym.go
@@ -90,18 +90,5 @@ func ParseKallsyms(funcs Funcs, all bool) (Addr2Name, BpfProgName2Addr, error) {
 }
 
 func extractBpfProgName(name string) (string, bool) {
-	if !strings.HasPrefix(name, "bpf_prog_") || !strings.HasSuffix(name, "[bpf]") {
-		return name, false
-	}
-
-	// The symbol of bpf prog is "bpf_prog_<tag>_<name>\t[bpf]". We want
-	// to get the tag and the name.
-
-	items := strings.Split(name, "_")
-	if len(items) > 3 {
-		name = strings.Join(items[3:], "_")
-		name = strings.TrimSpace(name[:len(name)-5])
-	}
-
-	return name, true
+	return strings.ReplaceAll(name, "\t", ""), strings.HasSuffix(name, "[bpf]")
 }
diff --git a/internal/pwru/types.go b/internal/pwru/types.go
@@ -7,7 +7,6 @@ package pwru
 import (
 	"fmt"
 	"os"
-	"slices"
 	"strings"
 
 	flag "github.com/spf13/pflag"
@@ -34,6 +33,7 @@ type Flags struct {
 	FilterTrackSkbByStackid bool
 	FilterTraceTc           bool
 	FilterTraceXdp          bool
+	FilterTrackBpfHelpers   bool
 	FilterIfname            string
 	FilterPcap              string
 	FilterKprobeBatch       uint
@@ -71,6 +71,7 @@ func (f *Flags) SetFlags() {
 	flag.BoolVar(&f.FilterTrackSkbByStackid, "filter-track-skb-by-stackid", false, "trace a packet even after it is kfreed (e.g., traffic going through bridge)")
 	flag.BoolVar(&f.FilterTraceTc, "filter-trace-tc", false, "trace TC bpf progs")
 	flag.BoolVar(&f.FilterTraceXdp, "filter-trace-xdp", false, "trace XDP bpf progs")
+	flag.BoolVar(&f.FilterTrackBpfHelpers, "filter-track-bpf-helpers", false, "trace BPF helper functions")
 	flag.StringVar(&f.FilterIfname, "filter-ifname", "", "filter skb ifname in --filter-netns (if not specified, use current netns)")
 	flag.UintVar(&f.FilterKprobeBatch, "filter-kprobe-batch", 10, "batch size for kprobe attaching/detaching")
 	flag.StringVar(&f.OutputTS, "timestamp", "none", "print timestamp per skb (\"current\", \"relative\", \"absolute\", \"none\")")
@@ -107,10 +108,8 @@ func (f *Flags) PrintHelp() {
 func (f *Flags) Parse() {
 	flag.Parse()
 	f.FilterPcap = strings.Join(flag.Args(), " ")
-	if len(f.FilterNonSkbFuncs) > 0 {
+	if len(f.FilterNonSkbFuncs) > 0 || f.FilterTrackBpfHelpers {
 		f.FilterTrackSkbByStackid = true
-		slices.Sort(f.FilterNonSkbFuncs)
-		f.FilterNonSkbFuncs = slices.Compact(f.FilterNonSkbFuncs)
 	}
 }
 

diff --git a/main.go b/main.go
@@ -99,7 +99,7 @@ func main() {
 	// prog's name.
 	addr2name, name2addr, err := pwru.ParseKallsyms(funcs, flags.OutputStack ||
 		len(flags.KMods) != 0 || flags.FilterTraceTc || flags.FilterTraceXdp ||
-		len(flags.FilterNonSkbFuncs) > 0 || flags.OutputCaller)
+		len(flags.FilterNonSkbFuncs) > 0 || flags.OutputCaller || flags.FilterTrackBpfHelpers)
 	if err != nil {
 		log.Fatalf("Failed to get function addrs: %s", err)
 	}
@@ -235,6 +235,14 @@ func main() {
 		defer t.Detach()
 	}
 
+	if flags.FilterTrackBpfHelpers {
+		bpfHelpers, err := pwru.GetBpfHelpers(addr2name)
+		if err != nil {
+			log.Fatalf("Failed to get bpf helpers: %s\n", err)
+		}
+		flags.FilterNonSkbFuncs = append(flags.FilterNonSkbFuncs, bpfHelpers...)
+	}
+
 	if nonSkbFuncs := flags.FilterNonSkbFuncs; len(nonSkbFuncs) != 0 {
 		k := pwru.NewNonSkbFuncsKprober(nonSkbFuncs, funcs, coll)
 		defer k.DetachKprobes()