[vParquet5] Fix buffer reuse bug with event-level dedicated columns

CHANGELOG.md

@@ -68,6 +68,7 @@
 * [BUGFIX] fix(traceql): err on division by zero [#6580](https://github.com/grafana/tempo/pull/6580) (@Proximyst)
 * [BUGFIX] fix(traceql): stop intPow from hanging [#6581](https://github.com/grafana/tempo/pull/6581) (@Proximyst)
 * [BUGFIX] fix(traceql): Fix incorrect search results for some queries on new blob columns [#6815](https://github.com/grafana/tempo/pull/6815) (@mdisibio)
+* [BUGFIX] fix(vparquet5) Fix buffer-reuse bug where event attributes in dedicated columns could be persisted on additional spans and events [6914](https://github.com/grafana/tempo/pull/6914) (@mdisibio)
 * [BUGFIX] fix: race condition where `remove_owner_on_shutdown` flag was set too late — after context cancellation already triggered the lifecycler's shutdown, causing the partition owner to remain in the ring [#6693](https://github.com/grafana/tempo/pull/6693) (@oleg-kozlyuk-grafana)
 * [BUGFIX] Return 400 instead of 500 when query_range or query_instant requests have unparseable start/end parameters [#6694](https://github.com/grafana/tempo/pull/6694) (@ruslan-mikhailov)
 * [BUGFIX] fix: correct block-builder fetch metrics to use counters instead of gauges.

pkg/util/test/req.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"math/rand"
+	"slices"
+	"strings"
 	"testing"
 	"time"
 
@@ -39,6 +41,46 @@ func MakeSpanWithTimeWindow(traceID []byte, startTime uint64, endTime uint64) *v
 	return makeSpanWithAttributeCount(traceID, rand.Int()%10+1, startTime, endTime) // nolint:gosec // G404: Use of weak random number generator
 }
 
+// DedicatedBlobTestSize is the length of blob payloads for dedicated columns (see DedicatedBlobTestString).
+const DedicatedBlobTestSize = 1000
+
+// DedicatedBlobTestString returns a fixed payload used by AddDedicatedAttributes for string columns
+// with the blob option so parquet/proto round-trip tests can assert exact values.
+func DedicatedBlobTestString() string {
+	return strings.Repeat("B", DedicatedBlobTestSize)
+}
+
+func randomDedicatedBlobString() string {
+	letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") // same set as util.TraceInfo.generateRandomBlob
+	s := make([]rune, DedicatedBlobTestSize)
+	for i := range s {
+		s[i] = letters[rand.Intn(len(letters))] // nolint:gosec // G404: test RNG
+	}
+	return string(s)
+}
+
+func anyValueForDedicatedColumn(col backend.DedicatedColumn, idx int) *v1_common.AnyValue {
+	if col.Type == backend.DedicatedColumnTypeInt {
+		return &v1_common.AnyValue{
+			Value: &v1_common.AnyValue_IntValue{
+				IntValue: int64(idx + 1 + rand.Intn(1000)), // nolint:gosec // G404: test RNG
+			},
+		}
+	}
+	if slices.Contains(col.Options, backend.DedicatedColumnOptionBlob) {
+		return &v1_common.AnyValue{
+			Value: &v1_common.AnyValue_StringValue{
+				StringValue: randomDedicatedBlobString(),
+			},
+		}
+	}
+	return &v1_common.AnyValue{
+		Value: &v1_common.AnyValue_StringValue{
+			StringValue: fmt.Sprintf("dedicated-%s-%s-%d", col.Scope, col.Name, rand.Int()), // nolint:gosec // G404
+		},
+	}
+}
+
 func makeSpanWithAttributeCount(traceID []byte, count int, startTime uint64, endTime uint64) *v1_trace.Span {
 	attributes := make([]*v1_common.KeyValue, 0, count)
 	for range count {
@@ -199,7 +241,7 @@ func MakeTrace(requests int, traceID []byte) *tempopb.Trace {
 		ResourceSpans: make([]*v1_trace.ResourceSpans, 0),
 	}
 
-	for i := 0; i < requests; i++ {
+	for range requests {
 		trace.ResourceSpans = append(trace.ResourceSpans, MakeBatch(rand.Int()%20+1, traceID)) // nolint:gosec // G404: Use of weak random number generator
 	}
 
@@ -242,6 +284,7 @@ var (
 		{Scope: "resource", Name: "dedicated.resource.5", Type: "string"},
 		{Scope: "resource", Name: "dedicated.resource.6", Type: "int"},
 		{Scope: "resource", Name: "dedicated.resource.7", Type: "int"},
+		{Scope: "resource", Name: "dedicated.resource.8", Type: "string", Options: []backend.DedicatedColumnOption{backend.DedicatedColumnOptionBlob}},
 	}
 	dedicatedColumnsSpan = backend.DedicatedColumns{
 		{Scope: "span", Name: "dedicated.span.1", Type: "string"},
@@ -255,23 +298,33 @@ var (
 	dedicatedColumnsEvent = backend.DedicatedColumns{
 		{Scope: "event", Name: "dedicated.event.1", Type: "string"},
 		{Scope: "event", Name: "dedicated.event.2", Type: "string"},
+		{Scope: "event", Name: "dedicated.event.3", Type: "int"},
+		{Scope: "event", Name: "dedicated.event.4", Type: "int"},
+		{Scope: "event", Name: "dedicated.event.5", Type: "string", Options: []backend.DedicatedColumnOption{backend.DedicatedColumnOptionBlob}},
 	}
 )
 
 // AddDedicatedAttributes adds resource and span attributes to a trace that are stored in dedicated
 // columns when a backend.BlockMeta is created with the column assignments from MakeDedicatedColumns.
 func AddDedicatedAttributes(trace *tempopb.Trace) *tempopb.Trace {
-	makeVal := func(typ backend.DedicatedColumnType, scope backend.DedicatedColumnScope, i int) *v1_common.AnyValue {
-		if typ == backend.DedicatedColumnTypeInt {
+	makeVal := func(c backend.DedicatedColumn, i int) *v1_common.AnyValue {
+		if c.Type == backend.DedicatedColumnTypeInt {
 			return &v1_common.AnyValue{
 				Value: &v1_common.AnyValue_IntValue{
 					IntValue: int64(i + 1),
 				},
 			}
 		}
+		if slices.Contains(c.Options, backend.DedicatedColumnOptionBlob) {
+			return &v1_common.AnyValue{
+				Value: &v1_common.AnyValue_StringValue{
+					StringValue: DedicatedBlobTestString(),
+				},
+			}
+		}
 		return &v1_common.AnyValue{
 			Value: &v1_common.AnyValue_StringValue{
-				StringValue: fmt.Sprintf("dedicated-%s-attr-value-%d", scope, i+1),
+				StringValue: fmt.Sprintf("dedicated-%s-attr-value-%d", c.Scope, i+1),
 			},
 		}
 	}
@@ -280,21 +333,21 @@ func AddDedicatedAttributes(trace *tempopb.Trace) *tempopb.Trace {
 	for i, c := range dedicatedColumnsSpan {
 		spanAttrs = append(spanAttrs, &v1_common.KeyValue{
 			Key:   c.Name,
-			Value: makeVal(c.Type, c.Scope, i),
+			Value: makeVal(c, i),
 		})
 	}
 	resourceAttrs := make([]*v1_common.KeyValue, 0, len(dedicatedColumnsResource))
 	for i, c := range dedicatedColumnsResource {
 		resourceAttrs = append(resourceAttrs, &v1_common.KeyValue{
 			Key:   c.Name,
-			Value: makeVal(c.Type, c.Scope, i),
+			Value: makeVal(c, i),
 		})
 	}
 	eventAttrs := make([]*v1_common.KeyValue, 0, len(dedicatedColumnsEvent))
 	for i, c := range dedicatedColumnsEvent {
 		eventAttrs = append(eventAttrs, &v1_common.KeyValue{
 			Key:   c.Name,
-			Value: makeVal(c.Type, c.Scope, i),
+			Value: makeVal(c, i),
 		})
 	}
 
@@ -321,6 +374,62 @@ func AddDedicatedAttributes(trace *tempopb.Trace) *tempopb.Trace {
 	return trace
 }
 
+// AddRandomDedicatedAttributes adds a random subset of the package test dedicated columns
+// (dedicatedColumnsResource, dedicatedColumnsSpan, dedicatedColumnsEvent) at resource, span,
+// and event levels (each existing span event). String columns with the blob option receive
+// DedicatedBlobTestSize random letters (same character set as util.TraceInfo.generateRandomBlob).
+// The trace is modified in place.
+// Parquet round-trips may reorder attributes; use model/trace.SortTraceAndAttributes before
+// proto.Equal when comparing decoded traces.
+func AddRandomDedicatedAttributes(trace *tempopb.Trace) *tempopb.Trace {
+	for _, batch := range trace.ResourceSpans {
+		if batch.Resource != nil {
+			for i, col := range dedicatedColumnsResource {
+				if rand.Intn(2) != 0 { // nolint:gosec // G404: 50% each column
+					continue
+				}
+				batch.Resource.Attributes = append(batch.Resource.Attributes, &v1_common.KeyValue{
+					Key:   col.Name,
+					Value: anyValueForDedicatedColumn(col, i),
+				})
+			}
+		}
+		for _, ss := range batch.ScopeSpans {
+			for _, span := range ss.Spans {
+				if span == nil {
+					continue
+				}
+				for i, col := range dedicatedColumnsSpan {
+					if rand.Intn(2) != 0 { // nolint:gosec // G404
+						continue
+					}
+					span.Attributes = append(span.Attributes, &v1_common.KeyValue{
+						Key:   col.Name,
+						Value: anyValueForDedicatedColumn(col, i),
+					})
+				}
+
+				for _, e := range span.Events {
+					if e == nil {
+						continue
+					}
+					for i, col := range dedicatedColumnsEvent {
+						if rand.Intn(2) != 0 { // nolint:gosec // G404
+							continue
+						}
+						e.Attributes = append(e.Attributes, &v1_common.KeyValue{
+							Key:   col.Name,
+							Value: anyValueForDedicatedColumn(col, i),
+						})
+					}
+				}
+			}
+		}
+	}
+
+	return trace
+}
+
 func MakeReqWithMultipleTraceWithSpanCount(spanCounts []int, traceIDs [][]byte) *tempopb.Trace {
 	if len(spanCounts) != len(traceIDs) {
 		panic("spanCounts and traceIDs lengths do not match")

pkg/util/trace_info.go

@@ -225,26 +225,32 @@ func (t *TraceInfo) generateRandomBlob(size int) string {
 
 // generateFixedAttributesWithPrefix returns the fixed attributes with a prefix. Keys are lowercase with a hyphen before the numeric suffix (e.g. string-01, int-01, blob-01).
 func (t *TraceInfo) generateFixedAttributesWithPrefix(prefix string) []*jaeger.Tag {
-	return []*jaeger.Tag{
-		{Key: fmt.Sprintf("%s-string-01", prefix), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
-		{Key: fmt.Sprintf("%s-string-02", prefix), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
-		{Key: fmt.Sprintf("%s-string-03", prefix), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
-		{Key: fmt.Sprintf("%s-string-04", prefix), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
-		{Key: fmt.Sprintf("%s-string-05", prefix), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
-		{Key: fmt.Sprintf("%s-blob-01", prefix), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomBlob(vultureBlobSize))},
-		{Key: fmt.Sprintf("%s-int-01", prefix), VType: jaeger.TagType_LONG, VLong: int64Ptr(t.generateRandomInt(1, 1000000))},
-		{Key: fmt.Sprintf("%s-int-02", prefix), VType: jaeger.TagType_LONG, VLong: int64Ptr(t.generateRandomInt(1, 1000000))},
-		{Key: fmt.Sprintf("%s-int-03", prefix), VType: jaeger.TagType_LONG, VLong: int64Ptr(t.generateRandomInt(1, 1000000))},
-		{Key: fmt.Sprintf("%s-int-04", prefix), VType: jaeger.TagType_LONG, VLong: int64Ptr(t.generateRandomInt(1, 1000000))},
-		{Key: fmt.Sprintf("%s-int-05", prefix), VType: jaeger.TagType_LONG, VLong: int64Ptr(t.generateRandomInt(1, 1000000))},
+	var (
+		numStrings = t.r.Intn(6)
+		numBlobs   = t.r.Intn(2)
+		numInts    = t.r.Intn(6)
+		tags       = make([]*jaeger.Tag, 0, numStrings+numBlobs+numInts)
+	)
+	for i := 0; i < numStrings; i++ {
+		tags = append(tags, &jaeger.Tag{Key: fmt.Sprintf("%s-string-%02d", prefix, i+1), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())})
+	}
+	for i := 0; i < numBlobs; i++ {
+		tags = append(tags, &jaeger.Tag{Key: fmt.Sprintf("%s-blob-%02d", prefix, i+1), VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomBlob(vultureBlobSize))})
 	}
+	for i := 0; i < numInts; i++ {
+		tags = append(tags, &jaeger.Tag{Key: fmt.Sprintf("%s-int-%02d", prefix, i+1), VType: jaeger.TagType_LONG, VLong: int64Ptr(t.generateRandomInt(1, 1000000))})
+	}
+	return tags
 }
 
 func stringPtr(s string) *string { return &s }
 
 func int64Ptr(n int64) *int64 { return &n }
 
 func (t *TraceInfo) generateResourceWellKnownAttributes() []*jaeger.Tag {
+	if t.r.Intn(2) == 0 {
+		return nil
+	}
 	return []*jaeger.Tag{
 		{Key: "cluster", VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
 		{Key: "namespace", VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
@@ -258,6 +264,9 @@ func (t *TraceInfo) generateResourceWellKnownAttributes() []*jaeger.Tag {
 }
 
 func (t *TraceInfo) generateSpanWellKnownAttributes() []*jaeger.Tag {
+	if t.r.Intn(2) == 0 {
+		return nil
+	}
 	return []*jaeger.Tag{
 		{Key: "http.method", VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},
 		{Key: "http.url", VType: jaeger.TagType_STRING, VStr: stringPtr(t.generateRandomString())},

pkg/util/trace_info_test.go

@@ -100,8 +100,8 @@ func TestConstructTraceFromEpoch(t *testing.T) {
 
 	result, err := info.ConstructTraceFromEpoch()
 	require.NoError(t, err)
-	// Batch count is deterministic per seed; adding well-known attributes changes RNG sequence (now 10 batches for testSeed).
-	assert.Equal(t, 10, len(result.ResourceSpans))
+	// Batch count is deterministic per seed; adding well-known attributes changes RNG sequence (now 8 batches for testSeed).
+	assert.Equal(t, 8, len(result.ResourceSpans))
 
 	result2, err := info.ConstructTraceFromEpoch()
 	require.NoError(t, err)

tempodb/encoding/vparquet3/block_search_test.go

@@ -77,7 +77,7 @@ func TestBackendBlockSearch(t *testing.T) {
 									String02: strPtr("dedicated-span-attr-value-2"),
 									String03: strPtr("dedicated-span-attr-value-3"),
 									String04: strPtr("dedicated-span-attr-value-4"),
-									String05: strPtr("dedicated-span-attr-value-5"),
+									String05: strPtr(test.DedicatedBlobTestString()),
 								},
 							},
 						},
@@ -343,7 +343,7 @@ func makeTraces() ([]*Trace, map[string]string, map[string]string, map[string]st
 	spanAttrVals["dedicated.span.2"] = *dedicatedSpanAttrs.String02
 	spanAttrVals["dedicated.span.3"] = *dedicatedSpanAttrs.String03
 	spanAttrVals["dedicated.span.4"] = *dedicatedSpanAttrs.String04
-	spanAttrVals["dedicated.span.5"] = "dedicated-span-attr-value-5"
+	spanAttrVals["dedicated.span.5"] = test.DedicatedBlobTestString()
 
 	for i := 0; i < 10; i++ {
 		tr := &Trace{

tempodb/encoding/vparquet3/block_traceql_test.go

@@ -365,7 +365,7 @@ func TestBackendNilValueBlockSearchTraceQL(t *testing.T) {
 											String02: ptr("dedicated-span-attr-value-2"),
 											String03: ptr("dedicated-span-attr-value-3"),
 											String04: ptr("dedicated-span-attr-value-4"),
-											String05: ptr("dedicated-span-attr-value-5"),
+											String05: ptr(test.DedicatedBlobTestString()),
 										},
 										Attrs: []Attribute{
 											// BUG - at least one generic attr is required to satisfy
@@ -965,7 +965,7 @@ func fullyPopulatedTestTraceWithOption(id common.ID, parentIDTest bool) *Trace {
 									String02: ptr("dedicated-span-attr-value-2"),
 									String03: ptr("dedicated-span-attr-value-3"),
 									String04: ptr("dedicated-span-attr-value-4"),
-									String05: ptr("dedicated-span-attr-value-5"),
+									String05: ptr(test.DedicatedBlobTestString()),
 								},
 							},
 						},

tempodb/encoding/vparquet3/schema_test.go

@@ -170,7 +170,7 @@ func TestTraceToParquet(t *testing.T) {
 								{Key: "dedicated.span.2", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-2"}}},
 								{Key: "dedicated.span.3", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-3"}}},
 								{Key: "dedicated.span.4", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-4"}}},
-								{Key: "dedicated.span.5", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-5"}}},
+								{Key: "dedicated.span.5", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: test.DedicatedBlobTestString()}}},
 							},
 						}},
 					}},
@@ -215,7 +215,7 @@ func TestTraceToParquet(t *testing.T) {
 							HttpStatusCode: intPtr(201),
 							Attrs: []Attribute{
 								{Key: "span.attr", Value: strPtr("aaa")},
-								{Key: "dedicated.span.5", Value: strPtr("dedicated-span-attr-value-5")},
+								{Key: "dedicated.span.5", Value: strPtr(test.DedicatedBlobTestString())},
 							},
 							DedicatedAttributes: DedicatedAttributes{
 								String01: strPtr("dedicated-span-attr-value-1"),

tempodb/encoding/vparquet4/block_search_test.go

@@ -72,7 +72,7 @@ func TestBackendBlockSearch(t *testing.T) {
 									String02: ptr("dedicated-span-attr-value-2"),
 									String03: ptr("dedicated-span-attr-value-3"),
 									String04: ptr("dedicated-span-attr-value-4"),
-									String05: ptr("dedicated-span-attr-value-5"),
+									String05: ptr(test.DedicatedBlobTestString()),
 								},
 							},
 						},
@@ -336,7 +336,7 @@ func makeTraces() ([]*Trace, map[string]string, map[string]string, map[string]st
 	spanAttrVals["dedicated.span.2"] = *dedicatedSpanAttrs.String02
 	spanAttrVals["dedicated.span.3"] = *dedicatedSpanAttrs.String03
 	spanAttrVals["dedicated.span.4"] = *dedicatedSpanAttrs.String04
-	spanAttrVals["dedicated.span.5"] = "dedicated-span-attr-value-5"
+	spanAttrVals["dedicated.span.5"] = test.DedicatedBlobTestString()
 
 	for i := 0; i < 10; i++ {
 		tr := &Trace{

tempodb/encoding/vparquet4/block_traceql_test.go

@@ -420,7 +420,7 @@ func TestBackendNilValueBlockSearchTraceQL(t *testing.T) {
 											String02: ptr("dedicated-span-attr-value-2"),
 											String03: ptr("dedicated-span-attr-value-3"),
 											String04: ptr("dedicated-span-attr-value-4"),
-											String05: ptr("dedicated-span-attr-value-5"),
+											String05: ptr(test.DedicatedBlobTestString()),
 										},
 										Attrs: []Attribute{
 											// BUG - at least one generic attr is required to satisfy
@@ -1151,7 +1151,7 @@ func fullyPopulatedTestTraceWithOption(id common.ID, parentIDTest bool) *Trace {
 									String02: ptr("dedicated-span-attr-value-2"),
 									String03: ptr("dedicated-span-attr-value-3"),
 									String04: ptr("dedicated-span-attr-value-4"),
-									String05: ptr("dedicated-span-attr-value-5"),
+									String05: ptr(test.DedicatedBlobTestString()),
 								},
 							},
 						},

tempodb/encoding/vparquet4/schema_test.go

@@ -264,7 +264,7 @@ func TestTraceToParquet(t *testing.T) {
 								{Key: "dedicated.span.2", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-2"}}},
 								{Key: "dedicated.span.3", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-3"}}},
 								{Key: "dedicated.span.4", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-4"}}},
-								{Key: "dedicated.span.5", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "dedicated-span-attr-value-5"}}},
+								{Key: "dedicated.span.5", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: test.DedicatedBlobTestString()}}},
 								{Key: "span.string.array", Value: &v1.AnyValue{Value: &v1.AnyValue_ArrayValue{ArrayValue: &v1.ArrayValue{
 									Values: []*v1.AnyValue{
 										{Value: &v1.AnyValue_StringValue{StringValue: "one"}},
@@ -371,7 +371,7 @@ func TestTraceToParquet(t *testing.T) {
 							HttpStatusCode: ptr(int64(201)),
 							Attrs: []Attribute{
 								attr("span.attr", "aaa"),
-								attr("dedicated.span.5", "dedicated-span-attr-value-5"), // This is now a blob so falls back to generic area.
+								attr("dedicated.span.5", test.DedicatedBlobTestString()), // Blob column; vparquet4 stores this in generic attrs.
 								attr("span.string.array", []string{"one", "two"}),
 								attr("span.int.array", []int64{1, 2, 3}),
 								attr("span.double.array", []float64{1.1, 2.2}),