Add span metrics filter policies to user-configurable overrides api

CHANGELOG.md

@@ -16,6 +16,7 @@
 * [ENHANCEMENT] Add `TempoUserConfigurableOverridesReloadFailing` alert [#2784](https://github.com/grafana/tempo/pull/2784) (@kvrhdn)
 * [ENHANCEMENT] Add RootSpanName and RootServiceName to log about discarded spans [#2816](https://github.com/grafana/tempo/pull/2816) (@marcinginszt)
 * [ENHANCEMENT] Add `UserID` to log message about rate limiting [#2850](https://github.com/grafana/tempo/pull/2850) (@lshippy)
+* [ENHANCEMENT] Add span metrics filter policies to user-configurable overrides [#2906](https://github.com/grafana/tempo/pull/2906) (@rlankfo)
 * [BUGFIX] Fix panic in metrics summary api [#2738](https://github.com/grafana/tempo/pull/2738) (@mdisibio)
 * [BUGFIX] Only search ingester blocks that fall within the request time range. [#2783](https://github.com/grafana/tempo/pull/2783) (@joe-elliott)
 * [BUGFIX] Align tempo_query_frontend_queries_total and tempo_query_frontend_queries_within_slo_total. [#2840](https://github.com/grafana/tempo/pull/2840) (@joe-elliott)

cmd/tempo/app/overrides_validation.go

@@ -8,6 +8,7 @@ import (
 	"github.com/grafana/tempo/modules/generator"
 	"github.com/grafana/tempo/modules/overrides/userconfigurable/api"
 	"github.com/grafana/tempo/modules/overrides/userconfigurable/client"
+	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
 )
 
 type overridesValidator struct {
@@ -48,5 +49,13 @@ func (v *overridesValidator) Validate(limits *client.Limits) error {
 		}
 	}
 
+	if filterPolicies, ok := limits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetFilterPolicies(); ok {
+		for _, fp := range filterPolicies {
+			if err := filterconfig.ValidateFilterPolicy(fp); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }

cmd/tempo/app/overrides_validation_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/grafana/tempo/modules/distributor/forwarder"
 	"github.com/grafana/tempo/modules/generator"
 	"github.com/grafana/tempo/modules/overrides/userconfigurable/client"
+	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
 )
 
 func Test_overridesValidator(t *testing.T) {
@@ -62,6 +63,39 @@ func Test_overridesValidator(t *testing.T) {
 			},
 			expErr: fmt.Sprintf("metrics_generator.processor \"span-span\" is not a known processor, valid values: %v", generator.SupportedProcessors),
 		},
+		{
+			name: "filter policies",
+			cfg:  Config{},
+			limits: client.Limits{
+				Forwarders: &[]string{},
+				MetricsGenerator: &client.LimitsMetricsGenerator{Processor: &client.LimitsMetricsGeneratorProcessor{SpanMetrics: &client.LimitsMetricsGeneratorProcessorSpanMetrics{FilterPolicies: &[]filterconfig.FilterPolicy{{
+					Include: &filterconfig.PolicyMatch{
+						MatchType: filterconfig.Strict,
+						Attributes: []filterconfig.MatchPolicyAttribute{
+							{
+								Key:   "span.kind",
+								Value: "SPAN_KIND_SERVER",
+							},
+						},
+					},
+				}}}}},
+			},
+		},
+		{
+			name: "filter policies - invalid",
+			cfg:  Config{},
+			limits: client.Limits{
+				Forwarders: &[]string{},
+				MetricsGenerator: &client.LimitsMetricsGenerator{Processor: &client.LimitsMetricsGeneratorProcessor{SpanMetrics: &client.LimitsMetricsGeneratorProcessorSpanMetrics{FilterPolicies: &[]filterconfig.FilterPolicy{
+					{
+						Include: &filterconfig.PolicyMatch{
+							MatchType: "invalid",
+						},
+					},
+				}}}},
+			},
+			expErr: "invalid include policy: invalid match type: invalid",
+		},
 	}
 
 	for _, tc := range testCases {

modules/overrides/user_configurable_overrides.go

@@ -21,6 +21,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	userconfigurableoverrides "github.com/grafana/tempo/modules/overrides/userconfigurable/client"
+	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
 	tempo_log "github.com/grafana/tempo/pkg/util/log"
 	"github.com/grafana/tempo/tempodb/backend"
 )
@@ -249,6 +250,13 @@ func (o *userConfigurableOverridesManager) MetricsGeneratorProcessorSpanMetricsE
 	return o.Interface.MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(userID)
 }
 
+func (o *userConfigurableOverridesManager) MetricsGeneratorProcessorSpanMetricsFilterPolicies(userID string) []filterconfig.FilterPolicy {
+	if filterPolicies, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetFilterPolicies(); ok {
+		return filterPolicies
+	}
+	return o.Interface.MetricsGeneratorProcessorSpanMetricsFilterPolicies(userID)
+}
+
 // statusUserConfigurableOverrides used to marshal userconfigurableoverrides.Limits for tenants
 type statusUserConfigurableOverrides struct {
 	TenantLimits tenantLimits `yaml:"user_configurable_overrides" json:"user_configurable_overrides"`

modules/overrides/user_configurable_overrides_test.go

@@ -12,6 +12,7 @@ import (
 
 	userconfigurableoverrides "github.com/grafana/tempo/modules/overrides/userconfigurable/client"
 	tempo_api "github.com/grafana/tempo/pkg/api"
+	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/grafana/tempo/tempodb/backend/local"
 )
@@ -76,6 +77,7 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 	assert.Empty(t, mgr.MetricsGeneratorProcessorServiceGraphsPeerAttributes(tenant1))
 	assert.Empty(t, mgr.MetricsGeneratorProcessorSpanMetricsDimensions(tenant1))
 	assert.Equal(t, false, mgr.MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(tenant1))
+	assert.Empty(t, mgr.MetricsGeneratorProcessorSpanMetricsFilterPolicies(tenant1))
 
 	// Inject user-configurable overrides
 	mgr.tenantLimits[tenant1] = &userconfigurableoverrides.Limits{
@@ -92,6 +94,28 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 				SpanMetrics: &userconfigurableoverrides.LimitsMetricsGeneratorProcessorSpanMetrics{
 					Dimensions:       &[]string{"sm-dimension"},
 					EnableTargetInfo: boolPtr(true),
+					FilterPolicies: &[]filterconfig.FilterPolicy{
+						{
+							Include: &filterconfig.PolicyMatch{
+								MatchType: filterconfig.Strict,
+								Attributes: []filterconfig.MatchPolicyAttribute{
+									{
+										Key:   "span.kind",
+										Value: "SPAN_KIND_SERVER",
+									},
+								},
+							},
+							Exclude: &filterconfig.PolicyMatch{
+								MatchType: filterconfig.Strict,
+								Attributes: []filterconfig.MatchPolicyAttribute{
+									{
+										Key:   "span.kind",
+										Value: "SPAN_KIND_CONSUMER",
+									},
+								},
+							},
+						},
+					},
 				},
 			},
 		},
@@ -106,6 +130,15 @@ func TestUserConfigOverridesManager_allFields(t *testing.T) {
 	assert.Equal(t, []string{"attribute"}, mgr.MetricsGeneratorProcessorServiceGraphsPeerAttributes(tenant1))
 	assert.Equal(t, []string{"sm-dimension"}, mgr.MetricsGeneratorProcessorSpanMetricsDimensions(tenant1))
 	assert.Equal(t, true, mgr.MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(tenant1))
+
+	filterPolicies := mgr.MetricsGeneratorProcessorSpanMetricsFilterPolicies(tenant1)
+	assert.NotEmpty(t, filterPolicies)
+	assert.Equal(t, filterconfig.Strict, filterPolicies[0].Include.MatchType)
+	assert.Equal(t, filterconfig.Strict, filterPolicies[0].Exclude.MatchType)
+	assert.Equal(t, "span.kind", filterPolicies[0].Include.Attributes[0].Key)
+	assert.Equal(t, "span.kind", filterPolicies[0].Exclude.Attributes[0].Key)
+	assert.Equal(t, "SPAN_KIND_SERVER", filterPolicies[0].Include.Attributes[0].Value)
+	assert.Equal(t, "SPAN_KIND_CONSUMER", filterPolicies[0].Exclude.Attributes[0].Value)
 }
 
 func TestUserConfigOverridesManager_populateFromBackend(t *testing.T) {

modules/overrides/userconfigurable/client/limits.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
 	"github.com/grafana/tempo/pkg/util/listtomap"
 )
 
@@ -99,8 +100,9 @@ func (l *LimitsMetricsGeneratorProcessorServiceGraphs) GetPeerAttributes() ([]st
 }
 
 type LimitsMetricsGeneratorProcessorSpanMetrics struct {
-	Dimensions       *[]string `json:"dimensions,omitempty"`
-	EnableTargetInfo *bool     `json:"enable_target_info,omitempty"`
+	Dimensions       *[]string                    `json:"dimensions,omitempty"`
+	EnableTargetInfo *bool                        `json:"enable_target_info,omitempty"`
+	FilterPolicies   *[]filterconfig.FilterPolicy `json:"filter_policies,omitempty"`
 }
 
 func (l *LimitsMetricsGeneratorProcessorSpanMetrics) GetDimensions() ([]string, bool) {
@@ -116,3 +118,10 @@ func (l *LimitsMetricsGeneratorProcessorSpanMetrics) GetEnableTargetInfo() (bool
 	}
 	return false, false
 }
+
+func (l *LimitsMetricsGeneratorProcessorSpanMetrics) GetFilterPolicies() ([]filterconfig.FilterPolicy, bool) {
+	if l != nil && l.FilterPolicies != nil {
+		return *l.FilterPolicies, true
+	}
+	return nil, true
+}

pkg/spanfilter/config/config.go

@@ -8,8 +8,8 @@ import (
 )
 
 type FilterPolicy struct {
-	Include *PolicyMatch `yaml:"include"`
-	Exclude *PolicyMatch `yaml:"exclude"`
+	Include *PolicyMatch `yaml:"include" json:"include,omitempty"`
+	Exclude *PolicyMatch `yaml:"exclude" json:"exclude,omitempty"`
 }
 
 type MatchType string
@@ -26,13 +26,13 @@ var supportedIntrinsics = []traceql.Intrinsic{
 }
 
 type PolicyMatch struct {
-	MatchType  MatchType              `yaml:"match_type"`
-	Attributes []MatchPolicyAttribute `yaml:"attributes"`
+	MatchType  MatchType              `yaml:"match_type" json:"match_type"`
+	Attributes []MatchPolicyAttribute `yaml:"attributes" json:"attributes"`
 }
 
 type MatchPolicyAttribute struct {
-	Key   string      `yaml:"key"`
-	Value interface{} `yaml:"value"`
+	Key   string      `yaml:"key" json:"key"`
+	Value interface{} `yaml:"value" json:"value"`
 }
 
 func ValidateFilterPolicy(policy FilterPolicy) error {