[TraceQL Metrics] Use new per-tenant max_metrics_duration and fix duration check

CHANGELOG.md

@@ -4,6 +4,7 @@
 * [ENHANCEMENT] Add new (unsafe) query hints for metrics queries [#3396](https://github.com/grafana/tempo/pull/3396) (@mdisibio)
 * [BUGFIX] Fix metrics query results when filtering and rating on the same attribute [#3428](https://github.com/grafana/tempo/issues/3428) (@mdisibio)
 * [BUGFIX] Fix metrics query results when series contain empty strings or nil values [#3429](https://github.com/grafana/tempo/issues/3429) (@mdisibio)
+* [BUGFIX] Fix metrics query duration check, add per-tenant override for max metrics query duration [#3479](https://github.com/grafana/tempo/issues/3479) (@mdisibio)
 * [BUGFIX] Return unfiltered results when a bad TraceQL query is provided in autocomplete. [#3426](https://github.com/grafana/tempo/pull/3426) (@mapno)
 * [BUGFIX] Correctly handle 429s in GRPC search streaming. [#3469](https://github.com/grafana/tempo/pull/3469) (@joe-ellitot)
 * [BUGFIX] Correctly cancel GRPC and HTTP contexts in the frontend to prevent having to rely on http write timeout. [#3443](https://github.com/grafana/tempo/pull/3443) (@joe-elliott)

docs/sources/tempo/configuration/_index.md

@@ -517,6 +517,26 @@ query_frontend:
         # If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
         # Query is within SLO if it returned 200 within duration_slo seconds.
         [duration_slo: <duration> | default = 0s ]
+
+    # Metrics query configuration
+    metrics:
+        # The number of concurrent jobs to execute when querying the backend.
+        [concurrent_jobs: <int> | default = 1000 ]
+
+        # The target number of bytes for each job to handle when querying the backend.
+        [target_bytes_per_job: <int> | default = 100MiB ]
+
+        # The maximum allowed time range for a metrics query.
+        # 0 disables this limit.
+        [max_duration: <duration> | default = 3h ]
+
+        # query_backend_after controls where the query-frontend searches for traces.
+        # Time ranges older than query_backend_after will be searched in the backend/object storage only.
+        # Time ranges between query_backend_after and now will be queried from the metrics-generators.
+        [query_backend_after: <duration> | default = 30m ]
+
+        # The target length of time for each job to handle when querying the backend. 
+        [interval: <duration> | default = 5m ]
 ```
 
 ## Querier
@@ -1248,6 +1268,10 @@ overrides:
       #  in the front-end configuration is used.
       [max_search_duration: <duration> | default = 0s]
 
+      # Per-user max duration for metrics queries. If this value is set to 0 (default), then metrics max_duration
+      #  in the front-end configuration is used.
+      [max_metrics_duration: <duration> | default = 0s]
+
     # Compaction related overrides
     compaction:
       # Per-user block retention. If this value is set to 0 (default),

modules/frontend/config.go

@@ -88,9 +88,10 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) {
 	}
 	cfg.Metrics = MetricsConfig{
 		Sharder: QueryRangeSharderConfig{
-			QueryBackendAfter:     time.Hour,
+			MaxDuration:           3 * time.Hour,
+			QueryBackendAfter:     30 * time.Minute,
 			ConcurrentRequests:    defaultConcurrentRequests,
-			TargetBytesPerRequest: 100 * 1024 * 1024,
+			TargetBytesPerRequest: defaultTargetBytesPerRequest,
 			Interval:              5 * time.Minute,
 		},
 	}

modules/frontend/query_range_sharding.go

@@ -105,7 +105,7 @@ func (s queryRangeSharder) RoundTrip(r *http.Request) (*http.Response, error) {
 
 	// calculate and enforce max search duration
 	maxDuration := s.maxDuration(tenantID)
-	if maxDuration != 0 && time.Duration(queryRangeReq.End-queryRangeReq.Start)*time.Second > maxDuration {
+	if maxDuration != 0 && time.Duration(queryRangeReq.End-queryRangeReq.Start)*time.Nanosecond > maxDuration {
 		return &http.Response{
 			StatusCode: http.StatusBadRequest,
 			Body:       io.NopCloser(strings.NewReader(fmt.Sprintf("range specified by start and end exceeds %s. received start=%d end=%d", maxDuration, queryRangeReq.Start, queryRangeReq.End))),
@@ -434,7 +434,7 @@ func (s *queryRangeSharder) toUpstreamRequest(ctx context.Context, req tempopb.Q
 // maxDuration returns the max search duration allowed for this tenant.
 func (s *queryRangeSharder) maxDuration(tenantID string) time.Duration {
 	// check overrides first, if no overrides then grab from our config
-	maxDuration := s.overrides.MaxSearchDuration(tenantID)
+	maxDuration := s.overrides.MaxMetricsDuration(tenantID)
 	if maxDuration != 0 {
 		return maxDuration
 	}

modules/overrides/config.go

@@ -142,7 +142,8 @@ type ReadOverrides struct {
 	MaxBlocksPerTagValuesQuery int `yaml:"max_blocks_per_tag_values_query,omitempty" json:"max_blocks_per_tag_values_query,omitempty"`
 
 	// QueryFrontend enforced overrides
-	MaxSearchDuration model.Duration `yaml:"max_search_duration,omitempty" json:"max_search_duration,omitempty"`
+	MaxSearchDuration  model.Duration `yaml:"max_search_duration,omitempty" json:"max_search_duration,omitempty"`
+	MaxMetricsDuration model.Duration `yaml:"max_metrics_duration,omitempty" json:"max_metrics_duration,omitempty"`
 
 	UnsafeQueryHints bool `yaml:"unsafe_query_hints,omitempty" json:"unsafe_query_hints,omitempty"`
 }

modules/overrides/config_legacy.go

@@ -117,8 +117,9 @@ type LegacyOverrides struct {
 	MaxBlocksPerTagValuesQuery int `yaml:"max_blocks_per_tag_values_query" json:"max_blocks_per_tag_values_query"`
 
 	// QueryFrontend enforced limits
-	MaxSearchDuration model.Duration `yaml:"max_search_duration" json:"max_search_duration"`
-	UnsafeQueryHints  bool           `yaml:"unsafe_query_hints" json:"unsafe_query_hints"`
+	MaxSearchDuration  model.Duration `yaml:"max_search_duration" json:"max_search_duration"`
+	MaxMetricsDuration model.Duration `yaml:"max_metrics_duration" json:"max_metrics_duration"`
+	UnsafeQueryHints   bool           `yaml:"unsafe_query_hints" json:"unsafe_query_hints"`
 
 	// MaxBytesPerTrace is enforced in the Ingester, Compactor, Querier (Search) and Serverless (Search). It
 	//  is not used when doing a trace by id lookup.
@@ -141,6 +142,7 @@ func (l *LegacyOverrides) toNewLimits() Overrides {
 			MaxBytesPerTagValuesQuery:  l.MaxBytesPerTagValuesQuery,
 			MaxBlocksPerTagValuesQuery: l.MaxBlocksPerTagValuesQuery,
 			MaxSearchDuration:          l.MaxSearchDuration,
+			MaxMetricsDuration:         l.MaxMetricsDuration,
 			UnsafeQueryHints:           l.UnsafeQueryHints,
 		},
 		Compaction: CompactionOverrides{

modules/overrides/interface.go

@@ -68,6 +68,7 @@ type Interface interface {
 	MetricsGeneratorProcessorSpanMetricsTargetInfoExcludedDimensions(userID string) []string
 	BlockRetention(userID string) time.Duration
 	MaxSearchDuration(userID string) time.Duration
+	MaxMetricsDuration(userID string) time.Duration
 	DedicatedColumns(userID string) backend.DedicatedColumns
 	UnsafeQueryHints(userID string) bool
 

modules/overrides/runtime_config_overrides.go

@@ -335,6 +335,10 @@ func (o *runtimeConfigOverridesManager) MaxSearchDuration(userID string) time.Du
 	return time.Duration(o.getOverridesForUser(userID).Read.MaxSearchDuration)
 }
 
+func (o *runtimeConfigOverridesManager) MaxMetricsDuration(userID string) time.Duration {
+	return time.Duration(o.getOverridesForUser(userID).Read.MaxMetricsDuration)
+}
+
 // MetricsGeneratorIngestionSlack is the max amount of time passed since a span's end time
 // for the span to be considered in metrics generation
 func (o *runtimeConfigOverridesManager) MetricsGeneratorIngestionSlack(userID string) time.Duration {