generator: add native histogram settings to user API

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * [CHANGE] Expose metrics_generator.trace_id_label_name to user-configurable overrides API [#5972](https://github.com/grafana/tempo/pull/5972) (@carles-grafana)
 * [CHANGE] Expose metrics_generator.ingestion_time_range_slack to user-configurable overrides API [#5958](https://github.com/grafana/tempo/pull/5958) (@carles-grafana)
+* [CHANGE] Expose metrics_generator.native_histogram_bucket_factor and native_histogram_min_reset_duration to user-configurable overrides API [#5973](https://github.com/grafana/tempo/pull/5973) (@carles-grafana)
 * [CHANGE] Remove remaining aws-sdk-go references and migrate tests to MinIO [#5856](https://github.com/grafana/tempo/pull/5856) (@anglerfishlyy)
 * [CHANGE] **BREAKING CHANGE** Validate tenant ID in frontend and distributor [#5786](https://github.com/grafana/tempo/pull/5786) (@carles-grafana)
 * [CHANGE] Remove busybox from Tempo image to make it more minimal and prevent future vulnerabilities [#5717](https://github.com/grafana/tempo/pull/5717) (@carles-grafana)

docs/sources/tempo/operations/manage-advanced-systems/user-configurable-overrides.md

@@ -67,6 +67,8 @@ metrics_generator:
   [disable_collection: <bool> | default = false]
   [generate_native_histograms: <classic|native|both> | default = classic]
   [native_histogram_max_bucket_number: <int> | default = 100]
+  [native_histogram_bucket_factor: <float> | default = 1.1]
+  [native_histogram_min_reset_duration: <duration> | default = 15m]
 
   processor:
 

modules/overrides/user_configurable_overrides.go

@@ -274,6 +274,20 @@ func (o *userConfigurableOverridesManager) MetricsGeneratorNativeHistogramMaxBuc
 	return o.Interface.MetricsGeneratorNativeHistogramMaxBucketNumber(userID)
 }
 
+func (o *userConfigurableOverridesManager) MetricsGeneratorNativeHistogramBucketFactor(userID string) float64 {
+	if factor, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetNativeHistogramBucketFactor(); ok {
+		return factor
+	}
+	return o.Interface.MetricsGeneratorNativeHistogramBucketFactor(userID)
+}
+
+func (o *userConfigurableOverridesManager) MetricsGeneratorNativeHistogramMinResetDuration(userID string) time.Duration {
+	if minReset, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetNativeHistogramMinResetDuration(); ok {
+		return minReset
+	}
+	return o.Interface.MetricsGeneratorNativeHistogramMinResetDuration(userID)
+}
+
 func (o *userConfigurableOverridesManager) MetricsGeneratorCollectionInterval(userID string) time.Duration {
 	if collectionInterval, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetCollectionInterval(); ok {
 		return collectionInterval

modules/overrides/user_configurable_overrides_test.go

@@ -107,13 +107,15 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 	mgr.tenantLimits[tenant1] = &userconfigurableoverrides.Limits{
 		Forwarders: &[]string{"my-forwarder"},
 		MetricsGenerator: userconfigurableoverrides.LimitsMetricsGenerator{
-			Processors:                     map[string]struct{}{"service-graphs": {}},
-			DisableCollection:              boolPtr(true),
-			CollectionInterval:             &userconfigurableoverrides.Duration{Duration: 60 * time.Second},
-			TraceIDLabelName:               strPtr("trace_id"),
-			IngestionSlack:                 &userconfigurableoverrides.Duration{Duration: 45 * time.Second},
-			GenerateNativeHistograms:       (*histograms.HistogramMethod)(strPtr("native")),
-			NativeHistogramMaxBucketNumber: func(u uint32) *uint32 { return &u }(101),
+			Processors:                      map[string]struct{}{"service-graphs": {}},
+			DisableCollection:               boolPtr(true),
+			CollectionInterval:              &userconfigurableoverrides.Duration{Duration: 60 * time.Second},
+			TraceIDLabelName:                strPtr("trace_id"),
+			IngestionSlack:                  &userconfigurableoverrides.Duration{Duration: 45 * time.Second},
+			GenerateNativeHistograms:        (*histograms.HistogramMethod)(strPtr("native")),
+			NativeHistogramMaxBucketNumber:  func(u uint32) *uint32 { return &u }(101),
+			NativeHistogramBucketFactor:     func(f float64) *float64 { return &f }(1.4),
+			NativeHistogramMinResetDuration: &userconfigurableoverrides.Duration{Duration: 5 * time.Minute},
 			Processor: userconfigurableoverrides.LimitsMetricsGeneratorProcessor{
 				ServiceGraphs: userconfigurableoverrides.LimitsMetricsGeneratorProcessorServiceGraphs{
 					Dimensions:               &[]string{"sg-dimension"},
@@ -161,6 +163,8 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 	assert.Equal(t, true, mgr.MetricsGeneratorDisableCollection(tenant1))
 	assert.Equal(t, histograms.HistogramMethodNative, mgr.MetricsGeneratorGenerateNativeHistograms(tenant1))
 	assert.Equal(t, uint32(101), mgr.MetricsGeneratorNativeHistogramMaxBucketNumber(tenant1))
+	assert.Equal(t, 1.4, mgr.MetricsGeneratorNativeHistogramBucketFactor(tenant1))
+	assert.Equal(t, 5*time.Minute, mgr.MetricsGeneratorNativeHistogramMinResetDuration(tenant1))
 	assert.Equal(t, []string{"sg-dimension"}, mgr.MetricsGeneratorProcessorServiceGraphsDimensions(tenant1))
 	assert.Equal(t, 60*time.Second, mgr.MetricsGeneratorCollectionInterval(tenant1))
 	assert.Equal(t, "trace_id", mgr.MetricsGeneratorTraceIDLabelName(tenant1))
@@ -420,9 +424,13 @@ func TestUserConfigOverridesManager_MergeRuntimeConfig(t *testing.T) {
 	mgr.tenantLimits[tenantID] = &userconfigurableoverrides.Limits{
 		Forwarders: &[]string{"my-other-forwarder"},
 		MetricsGenerator: userconfigurableoverrides.LimitsMetricsGenerator{
-			Processors:       map[string]struct{}{"local-blocks": {}},
-			TraceIDLabelName: strPtr("custom_trace_id"),
-			IngestionSlack:   &userconfigurableoverrides.Duration{Duration: time.Minute},
+			Processors:                  map[string]struct{}{"local-blocks": {}},
+			TraceIDLabelName:            strPtr("custom_trace_id"),
+			IngestionSlack:              &userconfigurableoverrides.Duration{Duration: time.Minute},
+			NativeHistogramBucketFactor: func(f float64) *float64 { return &f }(2.1),
+			NativeHistogramMinResetDuration: &userconfigurableoverrides.Duration{
+				Duration: 2 * time.Minute,
+			},
 		},
 	}
 
@@ -449,6 +457,10 @@ func TestUserConfigOverridesManager_MergeRuntimeConfig(t *testing.T) {
 	assert.Equal(t, time.Minute, mgr.MetricsGeneratorIngestionSlack(tenantID))
 	assert.NotEqual(t, mgr.MetricsGeneratorIngestionSlack(tenantID), baseMgr.MetricsGeneratorIngestionSlack(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorRingSize(tenantID), baseMgr.MetricsGeneratorRingSize(tenantID))
+	assert.Equal(t, 2.1, mgr.MetricsGeneratorNativeHistogramBucketFactor(tenantID))
+	assert.NotEqual(t, mgr.MetricsGeneratorNativeHistogramBucketFactor(tenantID), baseMgr.MetricsGeneratorNativeHistogramBucketFactor(tenantID))
+	assert.Equal(t, 2*time.Minute, mgr.MetricsGeneratorNativeHistogramMinResetDuration(tenantID))
+	assert.NotEqual(t, mgr.MetricsGeneratorNativeHistogramMinResetDuration(tenantID), baseMgr.MetricsGeneratorNativeHistogramMinResetDuration(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorMaxActiveSeries(tenantID), baseMgr.MetricsGeneratorMaxActiveSeries(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorCollectionInterval(tenantID), baseMgr.MetricsGeneratorCollectionInterval(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorDisableCollection(tenantID), baseMgr.MetricsGeneratorDisableCollection(tenantID))

modules/overrides/userconfigurable/api/limits.go

@@ -14,13 +14,15 @@ func limitsFromOverrides(overrides overrides.Interface, userID string) *client.L
 	return &client.Limits{
 		Forwarders: strArrPtr(overrides.Forwarders(userID)),
 		MetricsGenerator: client.LimitsMetricsGenerator{
-			Processors:                     overrides.MetricsGeneratorProcessors(userID),
-			DisableCollection:              boolPtr(overrides.MetricsGeneratorDisableCollection(userID)),
-			CollectionInterval:             timePtr(overrides.MetricsGeneratorCollectionInterval(userID)),
-			TraceIDLabelName:               strPtr(overrides.MetricsGeneratorTraceIDLabelName(userID)),
-			IngestionSlack:                 timePtr(overrides.MetricsGeneratorIngestionSlack(userID)),
-			GenerateNativeHistograms:       histogramModePtr(overrides.MetricsGeneratorGenerateNativeHistograms(userID)),
-			NativeHistogramMaxBucketNumber: uint32Ptr(overrides.MetricsGeneratorNativeHistogramMaxBucketNumber(userID)),
+			Processors:                      overrides.MetricsGeneratorProcessors(userID),
+			DisableCollection:               boolPtr(overrides.MetricsGeneratorDisableCollection(userID)),
+			CollectionInterval:              timePtr(overrides.MetricsGeneratorCollectionInterval(userID)),
+			TraceIDLabelName:                strPtr(overrides.MetricsGeneratorTraceIDLabelName(userID)),
+			IngestionSlack:                  timePtr(overrides.MetricsGeneratorIngestionSlack(userID)),
+			GenerateNativeHistograms:        histogramModePtr(overrides.MetricsGeneratorGenerateNativeHistograms(userID)),
+			NativeHistogramMaxBucketNumber:  uint32Ptr(overrides.MetricsGeneratorNativeHistogramMaxBucketNumber(userID)),
+			NativeHistogramBucketFactor:     floatPtr(overrides.MetricsGeneratorNativeHistogramBucketFactor(userID)),
+			NativeHistogramMinResetDuration: timePtr(overrides.MetricsGeneratorNativeHistogramMinResetDuration(userID)),
 			Processor: client.LimitsMetricsGeneratorProcessor{
 				ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
 					Dimensions:               strArrPtr(overrides.MetricsGeneratorProcessorServiceGraphsDimensions(userID)),
@@ -71,6 +73,10 @@ func floatArrPtr(f []float64) *[]float64 {
 	return &f
 }
 
+func floatPtr(f float64) *float64 {
+	return &f
+}
+
 func filterPoliciesPtr(p []config.FilterPolicy) *[]config.FilterPolicy {
 	return &p
 }

modules/overrides/userconfigurable/api/limits_test.go

@@ -20,13 +20,15 @@ func Test_limitsFromOverrides(t *testing.T) {
 		Defaults: overrides.Overrides{
 			Forwarders: []string{"my-forwarder"},
 			MetricsGenerator: overrides.MetricsGeneratorOverrides{
-				Processors:                     map[string]struct{}{"service-graphs": {}},
-				CollectionInterval:             15 * time.Second,
-				IngestionSlack:                 time.Minute,
-				DisableCollection:              true,
-				TraceIDLabelName:               "trace_id",
-				GenerateNativeHistograms:       histograms.HistogramMethodBoth,
-				NativeHistogramMaxBucketNumber: 160,
+				Processors:                      map[string]struct{}{"service-graphs": {}},
+				CollectionInterval:              15 * time.Second,
+				IngestionSlack:                  time.Minute,
+				DisableCollection:               true,
+				TraceIDLabelName:                "trace_id",
+				GenerateNativeHistograms:        histograms.HistogramMethodBoth,
+				NativeHistogramMaxBucketNumber:  160,
+				NativeHistogramBucketFactor:     1.2,
+				NativeHistogramMinResetDuration: 10 * time.Minute,
 				Processor: overrides.ProcessorOverrides{
 					ServiceGraphs: overrides.ServiceGraphsOverrides{
 						HistogramBuckets:         []float64{0.1, 0.2, 0.5},
@@ -84,6 +86,8 @@ func Test_limitsFromOverrides(t *testing.T) {
     "ingestion_time_range_slack": "1m0s",
     "generate_native_histograms": "both",
     "native_histogram_max_bucket_number": 160,
+    "native_histogram_bucket_factor": 1.2,
+    "native_histogram_min_reset_duration": "10m0s",
     "processor": {
       "service_graphs": {
         "dimensions": [

modules/overrides/userconfigurable/client/limits.go

@@ -36,13 +36,15 @@ func (l *Limits) GetCostAttribution() *CostAttribution {
 }
 
 type LimitsMetricsGenerator struct {
-	Processors                     listtomap.ListToMap         `yaml:"processors,omitempty" json:"processors,omitempty"`
-	DisableCollection              *bool                       `yaml:"disable_collection,omitempty" json:"disable_collection,omitempty"`
-	CollectionInterval             *Duration                   `yaml:"collection_interval,omitempty" json:"collection_interval,omitempty"`
-	TraceIDLabelName               *string                     `yaml:"trace_id_label_name,omitempty" json:"trace_id_label_name,omitempty"`
-	IngestionSlack                 *Duration                   `yaml:"ingestion_time_range_slack,omitempty" json:"ingestion_time_range_slack,omitempty"`
-	GenerateNativeHistograms       *histograms.HistogramMethod `yaml:"generate_native_histograms" json:"generate_native_histograms,omitempty"`
-	NativeHistogramMaxBucketNumber *uint32                     `yaml:"native_histogram_max_bucket_number,omitempty" json:"native_histogram_max_bucket_number,omitempty"`
+	Processors                      listtomap.ListToMap         `yaml:"processors,omitempty" json:"processors,omitempty"`
+	DisableCollection               *bool                       `yaml:"disable_collection,omitempty" json:"disable_collection,omitempty"`
+	CollectionInterval              *Duration                   `yaml:"collection_interval,omitempty" json:"collection_interval,omitempty"`
+	TraceIDLabelName                *string                     `yaml:"trace_id_label_name,omitempty" json:"trace_id_label_name,omitempty"`
+	IngestionSlack                  *Duration                   `yaml:"ingestion_time_range_slack,omitempty" json:"ingestion_time_range_slack,omitempty"`
+	GenerateNativeHistograms        *histograms.HistogramMethod `yaml:"generate_native_histograms" json:"generate_native_histograms,omitempty"`
+	NativeHistogramMaxBucketNumber  *uint32                     `yaml:"native_histogram_max_bucket_number,omitempty" json:"native_histogram_max_bucket_number,omitempty"`
+	NativeHistogramBucketFactor     *float64                    `yaml:"native_histogram_bucket_factor,omitempty" json:"native_histogram_bucket_factor,omitempty"`
+	NativeHistogramMinResetDuration *Duration                   `yaml:"native_histogram_min_reset_duration,omitempty" json:"native_histogram_min_reset_duration,omitempty"`
 
 	Processor LimitsMetricsGeneratorProcessor `yaml:"processor,omitempty" json:"processor,omitempty"`
 }
@@ -75,6 +77,20 @@ func (l *LimitsMetricsGenerator) GetNativeHistogramMaxBucketNumber() (uint32, bo
 	return 0, false
 }
 
+func (l *LimitsMetricsGenerator) GetNativeHistogramBucketFactor() (float64, bool) {
+	if l != nil && l.NativeHistogramBucketFactor != nil {
+		return *l.NativeHistogramBucketFactor, true
+	}
+	return 0, false
+}
+
+func (l *LimitsMetricsGenerator) GetNativeHistogramMinResetDuration() (time.Duration, bool) {
+	if l != nil && l.NativeHistogramMinResetDuration != nil {
+		return l.NativeHistogramMinResetDuration.Duration, true
+	}
+	return 0, false
+}
+
 func (l *LimitsMetricsGenerator) GetProcessor() *LimitsMetricsGeneratorProcessor {
 	if l != nil {
 		return &l.Processor

modules/overrides/userconfigurable/client/limits_test.go

@@ -34,6 +34,8 @@ func TestLimits_parseJson(t *testing.T) {
     "collection_interval": "30s",
     "trace_id_label_name": "my_trace_id",
     "ingestion_time_range_slack": "45s",
+    "native_histogram_bucket_factor": 1.2,
+    "native_histogram_min_reset_duration": "10m",
 		"native_histogram_max_bucket_number": 101,
 		"generate_native_histograms": "native",
     "processor": {
@@ -54,12 +56,14 @@ func TestLimits_parseJson(t *testing.T) {
 			Limits{
 				Forwarders: &[]string{"dev"},
 				MetricsGenerator: LimitsMetricsGenerator{
-					Processors:                     map[string]struct{}{"service-graphs": {}},
-					CollectionInterval:             &Duration{Duration: 30 * time.Second},
-					TraceIDLabelName:               strPtr("my_trace_id"),
-					IngestionSlack:                 &Duration{Duration: 45 * time.Second},
-					GenerateNativeHistograms:       (*histograms.HistogramMethod)(strPtr("native")),
-					NativeHistogramMaxBucketNumber: func(u uint32) *uint32 { return &u }(101),
+					Processors:                      map[string]struct{}{"service-graphs": {}},
+					CollectionInterval:              &Duration{Duration: 30 * time.Second},
+					TraceIDLabelName:                strPtr("my_trace_id"),
+					IngestionSlack:                  &Duration{Duration: 45 * time.Second},
+					NativeHistogramBucketFactor:     func(f float64) *float64 { return &f }(1.2),
+					NativeHistogramMinResetDuration: &Duration{Duration: 10 * time.Minute},
+					GenerateNativeHistograms:        (*histograms.HistogramMethod)(strPtr("native")),
+					NativeHistogramMaxBucketNumber:  func(u uint32) *uint32 { return &u }(101),
 					Processor: LimitsMetricsGeneratorProcessor{
 						ServiceGraphs: LimitsMetricsGeneratorProcessorServiceGraphs{
 							Dimensions: &[]string{"cluster"},