Added tempo vulture example (#3829)

* added Tempo vulture example

* vulture example

* use proper port

* added changelog

* replace deprecated grcp dial method by NewClient one

* added pr information in the changelog

* set 14250 as default grpc port if missing from tempo push address

* format dialAddress

Co-authored-by: Mario <mariorvinas@gmail.com>

* Use a more explicit const name for the Jaeger endpoint

Co-authored-by: Mario <mariorvinas@gmail.com>

* fix typo

---------

Co-authored-by: Mario <mariorvinas@gmail.com>

CHANGELOG.md

@@ -21,6 +21,7 @@
 * [ENHANCEMENT] Added a boolean flag to enable or disable dualstack mode on Storage block config for S3 [#3721](https://github.com/grafana/tempo/pull/3721) (@sid-jar, @mapno)
 * [ENHANCEMENT] Add caching to query range queries [#3796](https://github.com/grafana/tempo/pull/3796) (@mapno)
 * [ENHANCEMENT] Add data quality metric to measure traces without a root [#3812](https://github.com/grafana/tempo/pull/3812) (@mapno)
+* [ENHANCEMENT] Added an example for running Tempo vulture [#3829](https://github.com/grafana/tempo/pull/3829) (@javiermolinar)
 * [ENHANCEMENT] Add a new helper method to allow debugging e2e tests [#3836](https://github.com/grafana/tempo/pull/3836) (@javiermolinar)
 * [ENHANCEMENT] Self document makefile [#3844](https://github.com/grafana/tempo/pull/3844)  (@javiermolinar)
 * [BUGFIX] Fix panic in certain metrics queries using `rate()` with `by` [#3847](https://github.com/grafana/tempo/pull/3847) (@stoewer)

cmd/tempo-vulture/main.go

@@ -58,6 +58,10 @@ type traceMetrics struct {
 	notFoundSearchAttribute int
 }
 
+const (
+	defaultJaegerGRPCEndpoint = 14250
+)
+
 func init() {
 	flag.StringVar(&prometheusPath, "prometheus-path", "/metrics", "The path to publish Prometheus metrics to.")
 	flag.StringVar(&prometheusListenAddress, "prometheus-listen-address", ":80", "The address to listen on for Prometheus scrapes.")
@@ -83,6 +87,11 @@ func main() {
 		zapcore.DebugLevel,
 	))
 
+	grpcEndpoint, err := getGRPCEndpoint(tempoPushURL)
+	if err != nil {
+		panic(err)
+	}
+
 	logger.Info("Tempo Vulture starting")
 
 	actualStartTime := time.Now()
@@ -120,7 +129,7 @@ func main() {
 
 	// Write
 	go func() {
-		client, err := newJaegerGRPCClient(tempoPushURL)
+		client, err := newJaegerGRPCClient(grpcEndpoint)
 		if err != nil {
 			panic(err)
 		}
@@ -285,15 +294,22 @@ func selectPastTimestamp(start, stop time.Time, interval, retention time.Duratio
 	return newStart.Round(interval), ts.Round(interval)
 }
 
-func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) {
-	// remove scheme and port
+func getGRPCEndpoint(endpoint string) (string, error) {
 	u, err := url.Parse(endpoint)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
+	dialAddress := u.Host
 
+	if u.Port() == "" {
+		dialAddress = fmt.Sprintf("%s:%d", dialAddress, defaultJaegerGRPCEndpoint)
+	}
+	return dialAddress, nil
+}
+
+func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) {
 	logger.Info("dialing grpc",
-		zap.String("endpoint", fmt.Sprintf("%s:14250", u.Host)),
+		zap.String("endpoint", endpoint),
 	)
 
 	var dialOpts []grpc.DialOption
@@ -311,7 +327,7 @@ func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) {
 	}
 
 	// new jaeger grpc exporter
-	conn, err := grpc.Dial(u.Host+":14250", dialOpts...)
+	conn, err := grpc.NewClient(endpoint, dialOpts...)
 	if err != nil {
 		return nil, err
 	}

cmd/tempo-vulture/main_test.go

@@ -111,3 +111,16 @@ func TestEqualTraces(t *testing.T) {
 
 	require.True(t, equalTraces(a, b))
 }
+
+func TestGetGrpcEndpoint(t *testing.T) {
+	_, err := getGRPCEndpoint("http://%gh&%ij")
+	require.Error(t, err)
+
+	got, err := getGRPCEndpoint("http://localhost:4000")
+	require.NoError(t, err)
+	assert.Equal(t, "localhost:4000", got, "Address endpoint should keep the given port")
+
+	got, err = getGRPCEndpoint("http://localhost")
+	require.NoError(t, err)
+	assert.Equal(t, "localhost:14250", got, "Address without a port should be defaulted to 14250")
+}

example/docker-compose/local/readme.md

@@ -10,7 +10,6 @@ permissions set in order to start correctly.
 
 ```console
 mkdir tempo-data/
-sudo chown 10001:10001 tempo-data/
 docker compose up -d
 ```
 

example/docker-compose/vulture/docker-compose.yaml

@@ -0,0 +1,31 @@
+services:
+
+   # Tempo runs as user 10001, and docker compose creates the volume as root.
+  # As such, we need to chown the volume in order for Tempo to start correctly.
+  init:
+    image: &tempoImage grafana/tempo:latest
+    user: root
+    entrypoint:
+      - "chown"
+      - "10001:10001"
+      - "/var/tempo"
+    volumes:
+      - ./tempo-data:/var/tempo
+
+  tempo:
+    image: *tempoImage
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ../shared/tempo.yaml:/etc/tempo.yaml
+      - ./tempo-data:/var/tempo
+    ports:
+      - "3200:3200"   # tempo
+      - "14250:14250"  # jaeger ingest grpc
+    depends_on:
+      - init
+
+  vulture:
+    image: grafana/tempo-vulture:latest
+    command: [ "-tempo-query-url=http://tempo:3200", "-tempo-push-url=http://tempo:14250" ]
+    depends_on:
+      - tempo

example/docker-compose/vulture/readme.md

@@ -0,0 +1,42 @@
+## Vulture
+
+This example set up a local Tempo instance and Tempo vulture.
+
+1. First create the storage directory with the correct permissions and start up the local stack.
+
+```console
+mkdir tempo-data/
+docker compose up -d
+```
+
+At this point, the following containers should be spun up -
+
+```console
+docker compose ps
+```
+```
+NAME                IMAGE                          COMMAND                  SERVICE   CREATED         STATUS         PORTS
+vulture-tempo-1     grafana/tempo:latest           "/tempo -config.file…"   tempo     2 minutes ago   Up 2 minutes   0.0.0.0:3200->3200/tcp, 0.0.0.0:14250->14250/tcp
+vulture-vulture-1   grafana/tempo-vulture:latest   "/tempo-vulture -tem…"   vulture   2 minutes ago   Up 2 minutes  
+```
+
+2. If you're interested you can see the wal/blocks as they are being created.
+
+```console
+ls tempo-data/
+```
+
+3. Tail logs of a container (eg: tempo)
+```bash
+docker logs vulture_tempo_1 -f
+```
+
+4. To stop the setup use -
+
+```console
+docker compose down -v
+```
+
+you can use Grafana or tempo-cli to make a query.
+
+tempo-cli: `$ tempo-cli query api search "0.0.0.0:3200" --use-grpc "{}" "2023-12-05T08:11:18Z" "2023-12-05T08:12:18Z" --org-id="test"`