generator: add trace_id_label_name to user API

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## main / unreleased
 
+* [CHANGE] Expose metrics_generator.trace_id_label_name to user-configurable overrides API [#5972](https://github.com/grafana/tempo/pull/5972) (@carles-grafana)
 * [CHANGE] Remove remaining aws-sdk-go references and migrate tests to MinIO [#5856](https://github.com/grafana/tempo/pull/5856) (@anglerfishlyy)
 * [CHANGE] **BREAKING CHANGE** Validate tenant ID in frontend and distributor [#5786](https://github.com/grafana/tempo/pull/5786) (@carles-grafana)
 * [CHANGE] Remove busybox from Tempo image to make it more minimal and prevent future vulnerabilities [#5717](https://github.com/grafana/tempo/pull/5717) (@carles-grafana)

cmd/tempo/app/overrides_validation.go

@@ -91,5 +91,11 @@ func (v *overridesValidator) Validate(limits *client.Limits) error {
 		}
 	}
 
+	if traceIDLabelName, ok := limits.GetMetricsGenerator().GetTraceIDLabelName(); ok {
+		if err := validation.ValidateTraceIDLabelName(traceIDLabelName); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

cmd/tempo/app/overrides_validation_test.go

@@ -17,6 +17,10 @@ import (
 	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
 )
 
+func strPtr(s string) *string {
+	return &s
+}
+
 func Test_runtimeOverridesValidator(t *testing.T) {
 	testCases := []struct {
 		name      string
@@ -208,6 +212,25 @@ func Test_overridesValidator(t *testing.T) {
 			},
 			expErr: "metrics_generator.collection_interval \"10m0s\" is outside acceptable range of 15s to 5m",
 		},
+		{
+			name: "metrics_generator.trace_id_label_name empty is allowed",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					TraceIDLabelName: strPtr(""),
+				},
+			},
+		},
+		{
+			name: "metrics_generator.trace_id_label_name invalid",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					TraceIDLabelName: strPtr("trace-id"),
+				},
+			},
+			expErr: "trace_id_label_name \"trace-id\" is not a valid Prometheus label name",
+		},
 	}
 
 	for _, tc := range testCases {

docs/sources/tempo/operations/manage-advanced-systems/user-configurable-overrides.md

@@ -62,6 +62,7 @@ metrics_generator:
 
   [processors: <list of strings>]
   [collection_interval: <duration>]
+  [trace_id_label_name: <string>]
   [disable_collection: <bool> | default = false]
   [generate_native_histograms: <classic|native|both> | default = classic]
   [native_histogram_max_bucket_number: <int> | default = 100]

modules/generator/overrides_test.go

@@ -73,7 +73,7 @@ func (m *mockOverrides) MetricsGeneratorGenerateNativeHistograms(string) histogr
 	return m.nativeHistograms
 }
 
-func (m *mockOverrides) MetricsGenerationTraceIDLabelName(string) string {
+func (m *mockOverrides) MetricsGeneratorTraceIDLabelName(string) string {
 	return ""
 }
 

modules/generator/registry/overrides.go

@@ -13,7 +13,7 @@ type Overrides interface {
 	MetricsGeneratorCollectionInterval(userID string) time.Duration
 	MetricsGeneratorDisableCollection(userID string) bool
 	MetricsGeneratorGenerateNativeHistograms(userID string) histograms.HistogramMethod
-	MetricsGenerationTraceIDLabelName(userID string) string
+	MetricsGeneratorTraceIDLabelName(userID string) string
 	MetricsGeneratorNativeHistogramBucketFactor(userID string) float64
 	MetricsGeneratorNativeHistogramMaxBucketNumber(userID string) uint32
 	MetricsGeneratorNativeHistogramMinResetDuration(userID string) time.Duration

modules/generator/registry/registry.go

@@ -161,7 +161,7 @@ func (r *ManagedRegistry) NewCounter(name string) Counter {
 }
 
 func (r *ManagedRegistry) NewHistogram(name string, buckets []float64, histogramOverride HistogramMode) (h Histogram) {
-	traceIDLabelName := r.overrides.MetricsGenerationTraceIDLabelName(r.tenant)
+	traceIDLabelName := r.overrides.MetricsGeneratorTraceIDLabelName(r.tenant)
 
 	// TODO: Temporary switch: use the old implementation when native histograms
 	// are disabled, eventually the new implementation can handle all cases

modules/generator/registry/registry_test.go

@@ -480,7 +480,7 @@ func (m *mockOverrides) MetricsGeneratorGenerateNativeHistograms(_ string) histo
 	return m.generateNativeHistograms
 }
 
-func (m *mockOverrides) MetricsGenerationTraceIDLabelName(string) string {
+func (m *mockOverrides) MetricsGeneratorTraceIDLabelName(string) string {
 	return ""
 }
 

modules/generator/validation/fields.go

@@ -97,3 +97,10 @@ func ValidateDimensions(dimensions []string, intrinsicDimensions []string, dimen
 	}
 	return nil
 }
+
+func ValidateTraceIDLabelName(traceIDLabelName string) error {
+	if traceIDLabelName != SanitizeLabelName(traceIDLabelName) {
+		return fmt.Errorf("trace_id_label_name \"%s\" is not a valid Prometheus label name", traceIDLabelName)
+	}
+	return nil
+}

modules/overrides/interface.go

@@ -52,7 +52,7 @@ type Interface interface {
 	MetricsGeneratorCollectionInterval(userID string) time.Duration
 	MetricsGeneratorDisableCollection(userID string) bool
 	MetricsGeneratorGenerateNativeHistograms(userID string) histograms.HistogramMethod
-	MetricsGenerationTraceIDLabelName(userID string) string
+	MetricsGeneratorTraceIDLabelName(userID string) string
 	MetricsGeneratorRemoteWriteHeaders(userID string) map[string]string
 	MetricsGeneratorForwarderQueueSize(userID string) int
 	MetricsGeneratorForwarderWorkers(userID string) int

modules/overrides/runtime_config_overrides.go

@@ -460,9 +460,9 @@ func (o *runtimeConfigOverridesManager) MetricsGeneratorNativeHistogramMinResetD
 	return o.defaultLimits.MetricsGenerator.NativeHistogramMinResetDuration
 }
 
-// MetricsGenerationTraceIDLabelName is the label name used for the trace ID in metrics.
+// MetricsGeneratorTraceIDLabelName is the label name used for the trace ID in metrics.
 // "TraceID" is used if no value is provided.
-func (o *runtimeConfigOverridesManager) MetricsGenerationTraceIDLabelName(userID string) string {
+func (o *runtimeConfigOverridesManager) MetricsGeneratorTraceIDLabelName(userID string) string {
 	return o.getOverridesForUser(userID).MetricsGenerator.TraceIDLabelName
 }
 

modules/overrides/user_configurable_overrides.go

@@ -246,6 +246,13 @@ func (o *userConfigurableOverridesManager) MetricsGeneratorDisableCollection(use
 	return o.Interface.MetricsGeneratorDisableCollection(userID)
 }
 
+func (o *userConfigurableOverridesManager) MetricsGeneratorTraceIDLabelName(userID string) string {
+	if traceIDLabelName, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetTraceIDLabelName(); ok {
+		return traceIDLabelName
+	}
+	return o.Interface.MetricsGeneratorTraceIDLabelName(userID)
+}
+
 func (o *userConfigurableOverridesManager) MetricsGeneratorGenerateNativeHistograms(userID string) histograms.HistogramMethod {
 	if method, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetGenerateNativeHistograms(); ok {
 		return method

modules/overrides/user_configurable_overrides_test.go

@@ -109,6 +109,7 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 			Processors:                     map[string]struct{}{"service-graphs": {}},
 			DisableCollection:              boolPtr(true),
 			CollectionInterval:             &userconfigurableoverrides.Duration{Duration: 60 * time.Second},
+			TraceIDLabelName:               strPtr("trace_id"),
 			GenerateNativeHistograms:       (*histograms.HistogramMethod)(strPtr("native")),
 			NativeHistogramMaxBucketNumber: func(u uint32) *uint32 { return &u }(101),
 			Processor: userconfigurableoverrides.LimitsMetricsGeneratorProcessor{
@@ -160,6 +161,7 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 	assert.Equal(t, uint32(101), mgr.MetricsGeneratorNativeHistogramMaxBucketNumber(tenant1))
 	assert.Equal(t, []string{"sg-dimension"}, mgr.MetricsGeneratorProcessorServiceGraphsDimensions(tenant1))
 	assert.Equal(t, 60*time.Second, mgr.MetricsGeneratorCollectionInterval(tenant1))
+	assert.Equal(t, "trace_id", mgr.MetricsGeneratorTraceIDLabelName(tenant1))
 	assert.Equal(t, true, mgr.MetricsGeneratorProcessorServiceGraphsEnableClientServerPrefix(tenant1))
 	enableVirtualNodeLabelValue, enableVirtualNodeLabelIsSet := mgr.MetricsGeneratorProcessorServiceGraphsEnableVirtualNodeLabel(tenant1)
 	assert.Equal(t, true, enableVirtualNodeLabelValue)
@@ -415,7 +417,8 @@ func TestUserConfigOverridesManager_MergeRuntimeConfig(t *testing.T) {
 	mgr.tenantLimits[tenantID] = &userconfigurableoverrides.Limits{
 		Forwarders: &[]string{"my-other-forwarder"},
 		MetricsGenerator: userconfigurableoverrides.LimitsMetricsGenerator{
-			Processors: map[string]struct{}{"local-blocks": {}},
+			Processors:       map[string]struct{}{"local-blocks": {}},
+			TraceIDLabelName: strPtr("custom_trace_id"),
 		},
 	}
 
@@ -444,7 +447,7 @@ func TestUserConfigOverridesManager_MergeRuntimeConfig(t *testing.T) {
 	assert.Equal(t, mgr.MetricsGeneratorMaxActiveSeries(tenantID), baseMgr.MetricsGeneratorMaxActiveSeries(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorCollectionInterval(tenantID), baseMgr.MetricsGeneratorCollectionInterval(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorDisableCollection(tenantID), baseMgr.MetricsGeneratorDisableCollection(tenantID))
-	assert.Equal(t, mgr.MetricsGenerationTraceIDLabelName(tenantID), baseMgr.MetricsGenerationTraceIDLabelName(tenantID))
+	assert.Equal(t, "custom_trace_id", mgr.MetricsGeneratorTraceIDLabelName(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorForwarderQueueSize(tenantID), baseMgr.MetricsGeneratorForwarderQueueSize(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorForwarderWorkers(tenantID), baseMgr.MetricsGeneratorForwarderWorkers(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID), baseMgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID))

modules/overrides/userconfigurable/api/limits.go

@@ -17,6 +17,7 @@ func limitsFromOverrides(overrides overrides.Interface, userID string) *client.L
 			Processors:                     overrides.MetricsGeneratorProcessors(userID),
 			DisableCollection:              boolPtr(overrides.MetricsGeneratorDisableCollection(userID)),
 			CollectionInterval:             timePtr(overrides.MetricsGeneratorCollectionInterval(userID)),
+			TraceIDLabelName:               strPtr(overrides.MetricsGeneratorTraceIDLabelName(userID)),
 			GenerateNativeHistograms:       histogramModePtr(overrides.MetricsGeneratorGenerateNativeHistograms(userID)),
 			NativeHistogramMaxBucketNumber: uint32Ptr(overrides.MetricsGeneratorNativeHistogramMaxBucketNumber(userID)),
 			Processor: client.LimitsMetricsGeneratorProcessor{

modules/overrides/userconfigurable/api/limits_test.go

@@ -23,6 +23,7 @@ func Test_limitsFromOverrides(t *testing.T) {
 				Processors:                     map[string]struct{}{"service-graphs": {}},
 				CollectionInterval:             15 * time.Second,
 				DisableCollection:              true,
+				TraceIDLabelName:               "trace_id",
 				GenerateNativeHistograms:       histograms.HistogramMethodBoth,
 				NativeHistogramMaxBucketNumber: 160,
 				Processor: overrides.ProcessorOverrides{
@@ -78,6 +79,7 @@ func Test_limitsFromOverrides(t *testing.T) {
     ],
     "disable_collection": true,
     "collection_interval": "15s",
+    "trace_id_label_name": "trace_id",
     "generate_native_histograms": "both",
     "native_histogram_max_bucket_number": 160,
     "processor": {

modules/overrides/userconfigurable/client/limits.go

@@ -39,6 +39,7 @@ type LimitsMetricsGenerator struct {
 	Processors                     listtomap.ListToMap         `yaml:"processors,omitempty" json:"processors,omitempty"`
 	DisableCollection              *bool                       `yaml:"disable_collection,omitempty" json:"disable_collection,omitempty"`
 	CollectionInterval             *Duration                   `yaml:"collection_interval,omitempty" json:"collection_interval,omitempty"`
+	TraceIDLabelName               *string                     `yaml:"trace_id_label_name,omitempty" json:"trace_id_label_name,omitempty"`
 	GenerateNativeHistograms       *histograms.HistogramMethod `yaml:"generate_native_histograms" json:"generate_native_histograms,omitempty"`
 	NativeHistogramMaxBucketNumber *uint32                     `yaml:"native_histogram_max_bucket_number,omitempty" json:"native_histogram_max_bucket_number,omitempty"`
 
@@ -87,6 +88,13 @@ func (l *LimitsMetricsGenerator) GetCollectionInterval() (time.Duration, bool) {
 	return 0, false
 }
 
+func (l *LimitsMetricsGenerator) GetTraceIDLabelName() (string, bool) {
+	if l != nil && l.TraceIDLabelName != nil {
+		return *l.TraceIDLabelName, true
+	}
+	return "", false
+}
+
 type LimitsMetricsGeneratorProcessor struct {
 	ServiceGraphs LimitsMetricsGeneratorProcessorServiceGraphs `yaml:"service_graphs,omitempty" json:"service_graphs,omitempty"`
 	SpanMetrics   LimitsMetricsGeneratorProcessorSpanMetrics   `yaml:"span_metrics,omitempty" json:"span_metrics,omitempty"`

modules/overrides/userconfigurable/client/limits_test.go

@@ -32,6 +32,7 @@ func TestLimits_parseJson(t *testing.T) {
   "metrics_generator": {
     "processors": ["service-graphs"],
     "collection_interval": "30s",
+    "trace_id_label_name": "my_trace_id",
 		"native_histogram_max_bucket_number": 101,
 		"generate_native_histograms": "native",
     "processor": {
@@ -54,6 +55,7 @@ func TestLimits_parseJson(t *testing.T) {
 				MetricsGenerator: LimitsMetricsGenerator{
 					Processors:                     map[string]struct{}{"service-graphs": {}},
 					CollectionInterval:             &Duration{Duration: 30 * time.Second},
+					TraceIDLabelName:               strPtr("my_trace_id"),
 					GenerateNativeHistograms:       (*histograms.HistogramMethod)(strPtr("native")),
 					NativeHistogramMaxBucketNumber: func(u uint32) *uint32 { return &u }(101),
 					Processor: LimitsMetricsGeneratorProcessor{