Support exemplars with native-only histograms

integration/e2e/deployments/config-all-in-one-local.yaml

@@ -48,7 +48,7 @@ storage:
   trace:
     backend: local
     local:
-      path: /var/tempo
+      path: /var/tempo/traces
     pool:
       max_workers: 10
       queue_depth: 100

modules/generator/registry/native_histogram.go

@@ -19,7 +19,7 @@ import (
 type nativeHistogram struct {
 	metricName string
 
-	// TODO we can also switch to a HistrogramVec and let prometheus handle the labels. This would remove the series map
+	// TODO: we can also switch to a HistrogramVec and let prometheus handle the labels. This would remove the series map
 	//  and all locking around it.
 	//  Downside: you need to list labels at creation time while our interfaces only pass labels at observe time, this
 	//  will requires a bigger refactor, maybe something for a second pass?
@@ -133,19 +133,40 @@ func (h *nativeHistogram) ObserveWithExemplar(labelValueCombo *LabelValueCombo,
 }
 
 func (h *nativeHistogram) newSeries(labelValueCombo *LabelValueCombo, value float64, traceID string, multiplier float64) *nativeHistogramSeries {
+	// Configure histogram based on mode
+	//
+	// Native-only mode sets buckets to nil, and uses the histogram.Exemplars slice as the native exemplar format.
+	// Hybrid mode uses classic buckets and bucket.Exemplar format for compatibility.
+
+	var buckets []float64
+
+	// The native histogram only uses the static buckets when the classic histograms are enabled.
+	hasClassic := hasClassicHistograms(h.histogramOverride)
+	if hasClassic {
+		// Hybrid "both" mode: include classic buckets for compatibility
+		buckets = h.buckets
+	}
+
+	// Configure native histogram options based on mode
+	nativeOpts := prometheus.HistogramOpts{
+		Name:    h.name(),
+		Help:    "Native histogram for metric " + h.name(),
+		Buckets: buckets, // nil for pure native, h.buckets for hybrid
+		// Native histogram parameters
+		NativeHistogramBucketFactor:     1.1,
+		NativeHistogramMaxBucketNumber:  100,
+		NativeHistogramMinResetDuration: 15 * time.Minute,
+	}
+
+	if hasClassic {
+		// Hybrid mode: let Prometheus decide defaults for compatibility
+		nativeOpts.NativeHistogramMaxExemplars = -1 // Use default
+	}
+
 	newSeries := &nativeHistogramSeries{
-		promHistogram: prometheus.NewHistogram(prometheus.HistogramOpts{
-			Name:                            h.name(),
-			Help:                            "Native histogram for metric " + h.name(),
-			Buckets:                         h.buckets,
-			NativeHistogramBucketFactor:     1.1,
-			NativeHistogramMaxBucketNumber:  100,
-			NativeHistogramMinResetDuration: 15 * time.Minute,
-			// TODO enable examplars on native histograms
-			NativeHistogramMaxExemplars: -1,
-		}),
-		lastUpdated: 0,
-		firstSeries: atomic.NewBool(true),
+		promHistogram: prometheus.NewHistogram(nativeOpts),
+		lastUpdated:   0,
+		firstSeries:   atomic.NewBool(true),
 	}
 
 	h.updateSeries(newSeries, value, traceID, multiplier)
@@ -179,12 +200,16 @@ func (h *nativeHistogram) newSeries(labelValueCombo *LabelValueCombo, value floa
 }
 
 func (h *nativeHistogram) updateSeries(s *nativeHistogramSeries, value float64, traceID string, multiplier float64) {
+	// Use Prometheus native exemplar handling
+	exemplarObserver := s.promHistogram.(prometheus.ExemplarObserver)
+
+	labels := prometheus.Labels{h.traceIDLabelName: traceID}
+
 	for i := 0.0; i < multiplier; i++ {
-		s.promHistogram.(prometheus.ExemplarObserver).ObserveWithExemplar(
-			value,
-			map[string]string{h.traceIDLabelName: traceID},
-		)
+		// Let Prometheus handle exemplars natively
+		exemplarObserver.ObserveWithExemplar(value, labels)
 	}
+
 	s.lastUpdated = time.Now().UnixMilli()
 }
 
@@ -233,23 +258,6 @@ func (h *nativeHistogram) collectMetrics(appender storage.Appender, timeMs int64
 
 		// TODO: impact on active series from appending a histogram?
 		activeSeries += 0
-
-		if len(s.histogram.Exemplars) > 0 {
-			for _, encodedExemplar := range s.histogram.Exemplars {
-
-				// TODO are we appending the same exemplar twice?
-				e := exemplar.Exemplar{
-					Labels: convertLabelPairToLabels(encodedExemplar.Label),
-					Value:  encodedExemplar.GetValue(),
-					Ts:     convertTimestampToMs(encodedExemplar.GetTimestamp()),
-					HasTs:  true,
-				}
-				_, err = appender.AppendExemplar(0, s.labels, e)
-				if err != nil {
-					return activeSeries, err
-				}
-			}
-		}
 	}
 
 	return
@@ -267,7 +275,7 @@ func (h *nativeHistogram) removeStaleSeries(staleTimeMs int64) {
 }
 
 func (h *nativeHistogram) activeSeriesPerHistogramSerie() uint32 {
-	// TODO can we estimate this?
+	// TODO: can we estimate this?
 	return 1
 }
 
@@ -301,11 +309,60 @@ func (h *nativeHistogram) nativeHistograms(appender storage.Appender, lbls label
 	}
 	hist.NegativeBuckets = s.histogram.NegativeDelta
 
-	_, err = appender.AppendHistogram(0, lbls, timeMs, &hist, nil)
+	// Append the native histogram
+	ref, err := appender.AppendHistogram(0, lbls, timeMs, &hist, nil)
 	if err != nil {
 		return err
 	}
 
+	// NOTE: two exemplar formats are used:
+	// Native exemplars use the histogram.Exemplars slice, which is the native format.
+	// Bucket exemplars use the bucket.Exemplar field, which is the classic format.
+	//
+	// Use native exemplars when available, falling back to bucket exemplars
+
+	for _, ex := range s.histogram.Exemplars {
+		if ex != nil && len(ex.Label) > 0 {
+			_, err = appender.AppendExemplar(ref, lbls, exemplar.Exemplar{
+				Labels: convertLabelPairToLabels(ex.GetLabel()),
+				Value:  ex.GetValue(),
+				Ts:     timeMs,
+			})
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// NOTE: We clear the native exemplar slice to prevent accumulation, but the
+	// client_golang package handles the expiration of exemplars internally, and
+	// we don't have control over clearing the native histogram exemplars in the
+	// same way we do for the class histogram exemplars.
+	if len(s.histogram.Exemplars) > 0 {
+		clear(s.histogram.Exemplars)
+		s.histogram.Exemplars = s.histogram.Exemplars[:0]
+	}
+
+	// For pure native mode, never emit bucket exemplars - only native ones
+	// For hybrid mode, fallback to bucket exemplars if no native exemplars available
+	isHybridMode := hasClassicHistograms(h.histogramOverride)
+	if isHybridMode && len(s.histogram.Exemplars) == 0 {
+		// Hybrid mode fallback: use bucket exemplars if no native exemplars
+		for _, bucket := range s.histogram.Bucket {
+			if bucket.Exemplar != nil && len(bucket.Exemplar.Label) > 0 {
+				_, err = appender.AppendExemplar(ref, lbls, exemplar.Exemplar{
+					Labels: convertLabelPairToLabels(bucket.Exemplar.GetLabel()),
+					Value:  bucket.Exemplar.GetValue(),
+					Ts:     timeMs,
+				})
+				if err != nil {
+					return err
+				}
+				// Don't clear bucket exemplars here as they might be needed for classic emission
+			}
+		}
+	}
+
 	return
 }
 
@@ -360,6 +417,7 @@ func (h *nativeHistogram) classicHistograms(appender storage.Appender, timeMs in
 		}
 		activeSeries++
 
+		// Check for exemplars from prometheus histogram
 		if bucket.Exemplar != nil && len(bucket.Exemplar.Label) > 0 {
 			_, err = appender.AppendExemplar(ref, s.lb.Labels(), exemplar.Exemplar{
 				Labels: convertLabelPairToLabels(bucket.Exemplar.GetLabel()),
@@ -383,7 +441,7 @@ func (h *nativeHistogram) classicHistograms(appender storage.Appender, timeMs in
 				return activeSeries, err
 			}
 		}
-		_, err = appender.Append(0, s.lb.Labels(), timeMs, getIfGreaterThenZeroOr(s.histogram.GetSampleCountFloat(), s.histogram.GetSampleCount()))
+		_, err := appender.Append(0, s.lb.Labels(), timeMs, getIfGreaterThenZeroOr(s.histogram.GetSampleCountFloat(), s.histogram.GetSampleCount()))
 		if err != nil {
 			return activeSeries, err
 		}

modules/generator/registry/native_histogram_test.go

@@ -587,3 +587,86 @@ func expectedSeriesLen(samples []sample) int {
 	}
 	return len(series)
 }
+
+// Test specifically for native-only mode to ensure exemplars work
+func Test_NativeOnlyExemplars(t *testing.T) {
+	buckets := []float64{1, 2}
+	collectionTimeMs := time.Now().UnixMilli()
+
+	t.Run("native_only_with_exemplars", func(t *testing.T) {
+		onAdd := func(uint32) bool { return true }
+		// Use HistogramModeNative to test native-only behavior
+		h := newNativeHistogram("test_native_histogram", buckets, onAdd, nil, "trace_id", HistogramModeNative, nil)
+
+		// Add some observations with exemplars
+		lvc := newLabelValueCombo([]string{"service"}, []string{"test-service"})
+		h.ObserveWithExemplar(lvc, 1.5, "trace-123", 1.0)
+		h.ObserveWithExemplar(lvc, 0.5, "trace-456", 1.0)
+
+		// Collect metrics
+		appender := &capturingAppender{}
+		activeSeries, err := h.collectMetrics(appender, collectionTimeMs)
+		require.NoError(t, err)
+
+		t.Logf("Active series: %d", activeSeries)
+		t.Logf("Captured samples: %d", len(appender.samples))
+		t.Logf("Captured exemplars: %d", len(appender.exemplars))
+
+		for i, ex := range appender.exemplars {
+			t.Logf("Exemplar %d: labels=%v, value=%f, trace=%v", i, ex.l, ex.e.Value, ex.e.Labels)
+		}
+
+		// We should have exemplars
+		require.Greater(t, len(appender.exemplars), 0, "Native-only mode should capture exemplars")
+
+		// Verify that exemplars have the correct format for native histograms
+		for _, ex := range appender.exemplars {
+			// Exemplars should be attached to the main histogram metric, not bucket metrics
+			require.Equal(t, "test_native_histogram", ex.l.Get(labels.MetricName), "Exemplar should be attached to main histogram metric")
+			require.Contains(t, ex.e.Labels.String(), "trace_id", "Exemplar should contain trace_id")
+			require.Greater(t, ex.e.Value, 0.0, "Exemplar should have a value")
+		}
+
+		require.Len(t, appender.exemplars, 2, "Should have exactly 2 exemplars for 2 observations")
+
+		require.Equal(t, "trace-456", appender.exemplars[0].e.Labels.Get("trace_id"))
+		require.Equal(t, 0.5, appender.exemplars[0].e.Value)
+		require.Equal(t, "trace-123", appender.exemplars[1].e.Labels.Get("trace_id"))
+		require.Equal(t, 1.5, appender.exemplars[1].e.Value)
+	})
+
+	t.Run("native_only_histogram_exemplars", func(t *testing.T) {
+		onAdd := func(uint32) bool { return true }
+
+		// Create a native histogram with empty buckets to force native-only mode
+		h := newNativeHistogram("test_native_only", []float64{}, onAdd, nil, "trace_id", HistogramModeNative, nil)
+
+		// Add some observations with exemplars
+		lvc := newLabelValueCombo([]string{"service"}, []string{"native-only-xyz"})
+		h.ObserveWithExemplar(lvc, 1.5, "trace-native-123", 1.0)
+		h.ObserveWithExemplar(lvc, 0.5, "trace-native-456", 1.0)
+
+		// Collect metrics
+		appender := &capturingAppender{}
+		activeSeries, err := h.collectMetrics(appender, collectionTimeMs)
+		require.NoError(t, err)
+
+		t.Logf("Native-only - Active series: %d", activeSeries)
+		t.Logf("Native-only - Captured samples: %d", len(appender.samples))
+		t.Logf("Native-only - Captured exemplars: %d", len(appender.exemplars))
+
+		// This test might still use bucket exemplars due to Prometheus's default behavior,
+		// but it helps us understand the difference
+
+		require.Len(t, appender.exemplars, 2, "Should have exactly 2 exemplars for 2 observations")
+
+		for i, ex := range appender.exemplars {
+			t.Logf("Native-only exemplar %d: labels=%v, value=%f, trace=%v", i, ex.l, ex.e.Value, ex.e.Labels)
+		}
+
+		require.Equal(t, "trace-native-456", appender.exemplars[0].e.Labels.Get("trace_id"))
+		require.Equal(t, 0.5, appender.exemplars[0].e.Value)
+		require.Equal(t, "trace-native-123", appender.exemplars[1].e.Labels.Get("trace_id"))
+		require.Equal(t, 1.5, appender.exemplars[1].e.Value)
+	})
+}

modules/generator/storage/config_util.go

@@ -50,8 +50,7 @@ func generateTenantRemoteWriteConfigs(inputs []prometheus_config.RemoteWriteConf
 		}
 
 		output.SendNativeHistograms = sendNativeHistograms
-		// TODO: enable exemplars
-		// cloneCfg.SendExemplars = sendExemplars
+		output.SendExemplars = true
 
 		outputs = append(outputs, output)
 	}