[vParquet3] Add dedicated columns to overrides module

modules/overrides/interface.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/grafana/tempo/tempodb/backend"
+
 	"github.com/grafana/dskit/services"
 	"github.com/prometheus/client_golang/prometheus"
 
@@ -55,4 +57,5 @@ type Interface interface {
 	MetricsGeneratorProcessorServiceGraphsEnableClientServerPrefix(userID string) bool
 	BlockRetention(userID string) time.Duration
 	MaxSearchDuration(userID string) time.Duration
+	DedicatedColumns(userID string) []backend.DedicatedColumn
 }

modules/overrides/limits.go

@@ -4,6 +4,8 @@ import (
 	"flag"
 	"time"
 
+	"github.com/grafana/tempo/tempodb/backend"
+
 	"github.com/grafana/tempo/pkg/sharedconfig"
 	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
 	"github.com/prometheus/client_golang/prometheus"
@@ -98,6 +100,9 @@ type Limits struct {
 	//  is not used when doing a trace by id lookup.
 	MaxBytesPerTrace int `yaml:"max_bytes_per_trace" json:"max_bytes_per_trace"`
 
+	// tempodb limits
+	DedicatedColumns []backend.DedicatedColumn `yaml:"dedicated_columns" json:"dedicated_columns"`
+
 	// Configuration for overrides, convenient if it goes here.
 	PerTenantOverrideConfig string         `yaml:"per_tenant_override_config" json:"per_tenant_override_config"`
 	PerTenantOverridePeriod model.Duration `yaml:"per_tenant_override_period" json:"per_tenant_override_period"`

modules/overrides/overrides.go

@@ -7,6 +7,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/grafana/tempo/tempodb/backend"
+
 	"github.com/grafana/dskit/runtimeconfig"
 	"github.com/grafana/dskit/services"
 
@@ -394,6 +396,10 @@ func (o *overrides) MaxSearchDuration(userID string) time.Duration {
 	return time.Duration(o.getOverridesForUser(userID).MaxSearchDuration)
 }
 
+func (o *overrides) DedicatedColumns(userID string) []backend.DedicatedColumn {
+	return o.getOverridesForUser(userID).DedicatedColumns
+}
+
 func (o *overrides) getOverridesForUser(userID string) *Limits {
 	if tenantOverrides := o.tenantOverrides(); tenantOverrides != nil {
 		l := tenantOverrides.forUser(userID)

modules/overrides/overrides_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/tempo/tempodb/backend"
+
 	"github.com/grafana/dskit/services"
 	"github.com/grafana/tempo/pkg/sharedconfig"
 	"github.com/prometheus/client_golang/prometheus"
@@ -329,3 +331,73 @@ func TestMetricsGeneratorOverrides(t *testing.T) {
 		})
 	}
 }
+
+func TestTempoDBOverrides(t *testing.T) {
+
+	tests := []struct {
+		name                     string
+		limits                   Limits
+		overrides                *perTenantOverrides
+		expectedDedicatedColumns map[string][]backend.DedicatedColumn
+	}{
+		{
+			name: "limits",
+			limits: Limits{
+				DedicatedColumns: []backend.DedicatedColumn{
+					{Scope: "resource", Name: "namespace", Type: "string"},
+				},
+			},
+			expectedDedicatedColumns: map[string][]backend.DedicatedColumn{
+				"user1": {{Scope: "resource", Name: "namespace", Type: "string"}},
+				"user2": {{Scope: "resource", Name: "namespace", Type: "string"}},
+			},
+		},
+		{
+			name: "basic overrides",
+			limits: Limits{
+				DedicatedColumns: []backend.DedicatedColumn{
+					{Scope: "resource", Name: "namespace", Type: "string"},
+				},
+			},
+			overrides: &perTenantOverrides{TenantLimits: map[string]*Limits{
+				"user2": {
+					DedicatedColumns: []backend.DedicatedColumn{{Scope: "span", Name: "http.status", Type: "int"}},
+				},
+			}},
+			expectedDedicatedColumns: map[string][]backend.DedicatedColumn{
+				"user1": {{Scope: "resource", Name: "namespace", Type: "string"}},
+				"user2": {{Scope: "span", Name: "http.status", Type: "int"}},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.overrides != nil {
+				overridesFile := filepath.Join(t.TempDir(), "overrides.yaml")
+
+				buff, err := yaml.Marshal(tc.overrides)
+				require.NoError(t, err)
+
+				err = os.WriteFile(overridesFile, buff, os.ModePerm)
+				require.NoError(t, err)
+
+				tc.limits.PerTenantOverrideConfig = overridesFile
+				tc.limits.PerTenantOverridePeriod = model.Duration(time.Hour)
+			}
+
+			prometheus.DefaultRegisterer = prometheus.NewRegistry() // have to overwrite the registry or test panics with multiple metric reg
+			overrides, err := NewOverrides(tc.limits)
+			require.NoError(t, err)
+			err = services.StartAndAwaitRunning(context.TODO(), overrides)
+			require.NoError(t, err)
+
+			for user, expected := range tc.expectedDedicatedColumns {
+				assert.Equal(t, expected, overrides.DedicatedColumns(user))
+			}
+
+			err = services.StopAndAwaitTerminated(context.TODO(), overrides)
+			require.NoError(t, err)
+		})
+	}
+}

tempodb/backend/block_meta.go

@@ -56,11 +56,11 @@ type BlockMeta struct {
 // be stored in a dedicated column instead of the generic attribute column.
 type DedicatedColumn struct {
 	// The Scope of the attribute: can be 'resource' or 'span'
-	Scope string `json:"scope"`
+	Scope string `yaml:"scope" json:"scope"`
 	// The Name of the attribute stored in the dedicated column
-	Name string `json:"name"`
+	Name string `yaml:"name" json:"name"`
 	// The Type of attribute value: only 'string' supported
-	Type string `json:"type"`
+	Type string `yaml:"type" json:"type"`
 }
 
 func NewBlockMeta(tenantID string, blockID uuid.UUID, version string, encoding Encoding, dataEncoding string) *BlockMeta {