Tempo distributor panics when built with go 1.21

[joe-elliott]

Describe the bug
When built with go 1.21 the distributors repeatedly panic with the following:

distributor_1        | panic: invalid Go type tempopb.PreallocBytes for field tempopb.PushBytesRequest.traces
distributor_1        | 
distributor_1        | goroutine 434 [running]:
distributor_1        | google.golang.org/protobuf/internal/impl.newSingularConverter({0x295e640?, 0x2030a00}, {0x295fc98, 0xc0006aa600})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/convert.go:142 +0xaa8
distributor_1        | google.golang.org/protobuf/internal/impl.newListConverter({0x295e640, 0xc000983c00}, {0x295fc98, 0xc0006aa600})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/convert_list.go:17 +0x156
distributor_1        | google.golang.org/protobuf/internal/impl.NewConverter({0x295e640, 0xc000983c00}, {0x295fc98?, 0xc0006aa600?})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/convert.go:56 +0xbc
distributor_1        | google.golang.org/protobuf/internal/impl.fieldInfoForList({0x295fc98?, 0xc0006aa600}, {{0x1fb9917, 0x6}, {0x0, 0x0}, {0x295e640, 0x1eaf080}, {0x1fb991e, 0x50}, ...}, ...)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/message_reflect_field.go:205 +0xcf
distributor_1        | google.golang.org/protobuf/internal/impl.(*MessageInfo).makeKnownFieldsFunc(0xc000978dc0, {0xffffffffffffffff, {0x0, 0x0}, 0xffffffffffffffff, {0x0, 0x0}, 0xffffffffffffffff, {0x0, 0x0}, ...})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/message_reflect.go:74 +0x52b
distributor_1        | google.golang.org/protobuf/internal/impl.(*MessageInfo).makeReflectFuncs(0xc000978dc0, {0x295e640, 0x20fa840}, {0xffffffffffffffff, {0x0, 0x0}, 0xffffffffffffffff, {0x0, 0x0}, 0xffffffffffffffff, ...})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/message_reflect.go:42 +0x58
distributor_1        | google.golang.org/protobuf/internal/impl.(*MessageInfo).initOnce(0xc000978dc0)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/message.go:90 +0x1b0
distributor_1        | google.golang.org/protobuf/internal/impl.(*MessageInfo).init(...)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/message.go:72
distributor_1        | google.golang.org/protobuf/internal/impl.(*messageReflectWrapper).ProtoMethods(0xc000a70ad0)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/internal/impl/message_reflect_gen.go:150 +0x28
distributor_1        | google.golang.org/protobuf/proto.protoMethods(...)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/proto/proto_methods.go:19
distributor_1        | google.golang.org/protobuf/proto.MarshalOptions.marshal({{}, 0xc0?, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x2950158, 0xc000a70ad0})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/proto/encode.go:143 +0x5d
distributor_1        | google.golang.org/protobuf/proto.MarshalOptions.MarshalAppend({{}, 0x80?, 0xf5?, 0x23?}, {0x0, 0x0, 0x0}, {0x291eae0?, 0xc000a70ad0?})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/protobuf/proto/encode.go:125 +0x73
distributor_1        | github.com/golang/protobuf/proto.marshalAppend({0x0, 0x0, 0x0}, {0x7f8eed31ebc8?, 0xc0007efb30?}, 0x30?)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/github.com/golang/protobuf/proto/wire.go:40 +0x9e
distributor_1        | github.com/golang/protobuf/proto.Marshal(...)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/github.com/golang/protobuf/proto/wire.go:23
distributor_1        | google.golang.org/grpc/encoding/proto.codec.Marshal({}, {0x223f580, 0xc0007efb30})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/encoding/proto/proto.go:45 +0x48
distributor_1        | google.golang.org/grpc.encode({0x7f8eed31d140?, 0x3c33420?}, {0x223f580?, 0xc0007efb30?})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/rpc_util.go:596 +0x3e
distributor_1        | google.golang.org/grpc.prepareMsg({0x223f580?, 0xc0007efb30?}, {0x7f8eed31d140?, 0x3c33420?}, {0x0, 0x0}, {0x2931fd0, 0xc0000a8cd0})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/stream.go:1730 +0xc7
distributor_1        | google.golang.org/grpc.(*clientStream).SendMsg(0xc00096f440, {0x223f580?, 0xc0007efb30})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/stream.go:865 +0xf2
distributor_1        | google.golang.org/grpc.invoke({0x2933cf8?, 0xc000a67080?}, {0x237b787?, 0x83e5f8?}, {0x223f580, 0xc0007efb30}, {0x223f440, 0x3c33420}, 0xc000a66f90?, {0xc000a66c90, ...})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/call.go:70 +0x9f
distributor_1        | github.com/weaveworks/common/middleware.ClientUserHeaderInterceptor({0x2933cf8?, 0xc000a66ff0?}, {0x237b787, 0x1b}, {0x223f580, 0xc0007efb30}, {0x223f440, 0x3c33420}, 0xc0006aac00?, 0x24359d8, ...)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:17 +0xd7
distributor_1        | google.golang.org/grpc.getChainUnaryInvoker.func1({0x2933cf8, 0xc000a66ff0}, {0x237b787, 0x1b}, {0x223f580, 0xc0007efb30}, {0x223f440, 0x3c33420}, 0x0?, {0xc000a66c90, ...})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/clientconn.go:361 +0x155
distributor_1        | github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1({0x2933cf8, 0xc000a66c00}, {0x237b787, 0x1b}, {0x223f580, 0xc0007efb30}, {0x223f440, 0x3c33420}, 0xc0009d1c70?, 0xc000a66f30, ...)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:60 +0x47a
distributor_1        | google.golang.org/grpc.DialContext.chainUnaryClientInterceptors.func3({0x2933cf8, 0xc000a66c00}, {0x237b787, 0x1b}, {0x223f580, 0xc0007efb30}, {0x223f440, 0x3c33420}, 0x24?, 0x24359d8, ...)
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/clientconn.go:349 +0x117
distributor_1        | google.golang.org/grpc.(*ClientConn).Invoke(0xc0007d2a80, {0x2933cf8?, 0xc000a66c00?}, {0x237b787?, 0xc0009d1d68?}, {0x223f580?, 0xc0007efb30?}, {0x223f440?, 0x3c33420?}, {0x0, ...})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/google.golang.org/grpc/call.go:35 +0x205
distributor_1        | github.com/grafana/tempo/pkg/tempopb.(*pusherClient).PushBytesV2(0x0?, {0x2933cf8?, 0xc000a66c00?}, 0x40e78c?, {0x0?, 0x2212b40?, 0xc0009d1d00?})
distributor_1        | 	/home/joe/dev/grafana/tempo/pkg/tempopb/tempo.pb.go:2238 +0x5d
distributor_1        | github.com/grafana/tempo/modules/distributor.(*Distributor).sendToIngestersViaBytes.func1({{0xc00021f330, 0x10}, 0x64e38bad, 0x0, {0xc000696600, 0x80, 0x80}, {0x0, 0x0}, 0x64e38ba8}, ...)
distributor_1        | 	/home/joe/dev/grafana/tempo/modules/distributor/distributor.go:393 +0x3f6
distributor_1        | github.com/grafana/dskit/ring.DoBatch.func1({{{0xc00021f330, 0x10}, 0x64e38bad, 0x0, {0xc000696600, 0x80, 0x80}, {0x0, 0x0}, 0x64e38ba8}, ...})
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/github.com/grafana/dskit/ring/batch.go:108 +0x79
distributor_1        | created by github.com/grafana/dskit/ring.DoBatch in goroutine 433
distributor_1        | 	/home/joe/dev/grafana/tempo/vendor/github.com/grafana/dskit/ring/batch.go:107 +0x538

[joe-elliott]

The issue appears to be with package init orders. This behavior changed in go 1.21:

Here is where we init:

tempo/pkg/gogocodec/gogocodec.go

Line 30 in c2a7c17

func init() {