Coalesce read configuration for recent data cutoff

CHANGELOG.md

@@ -9,6 +9,7 @@
 * [CHANGE] Remove all traces of ingesters from the dashboards [#6352](https://github.com/grafana/tempo/pull/6352) (@javiermolinar)
 * [CHANGE] **BREAKING CHANGE** tempo-cli: Support relative time (now, now-1h) for start/end args and standardize on RFC3339 in all commands. [#6458](https://github.com/grafana/tempo/pull/6458) (@electron0zero)
   `query search` command no longer accepts timestamps without timezone (e.g. `2024-01-01T00:00:00`), use RFC3339 (e.g. `2024-01-01T00:00:00Z`) or relative time instead.
+* [CHANGE] **BREAKING CHANGE** Consolidate read configuration for recent data cutoff. `query_frontend.search.query_ingesters_until` is removed in favor of only `query_frontend.search.query_backend_after`. [#](https://github.com/grafana/tempo/pull/) (@mapno)
 * [FEATURE] Add new include_any filter policy for spanmetrics filter [#6392](https://github.com/grafana/tempo/pull/6392) (@javiermolinar)
 * [FEATURE] Add span_multiplier_key to overrides. This allows tenants to specify the attribute key used for span multiplier values to compensate for head-based sampling. [#6260](https://github.com/grafana/tempo/pull/6260) (@carles-grafana)
 * [FEATURE] **BREAKING CHANGE** Optimize TraceQL AST by rewriting conditions on the same attribute to their array equivalent [#6353](https://github.com/grafana/tempo/pull/6353) (@stoewer)

build/tools.mk

@@ -16,8 +16,17 @@ TOOLS_IMAGE_TAG ?= main-36bd1c7
 
 GOTOOLS ?= $(shell cd $(TOOL_DIR) && go list -e -f '{{ .Imports }}' -tags tools |tr -d '[]')
 
-TOOLS_CMD = docker run --rm -t -v ${PWD}:/tools $(TOOLS_IMAGE):$(TOOLS_IMAGE_TAG)
-LINT_CMD =  docker run --rm -t -v ${PWD}:/tools -v ${PWD}/.cache/golangci-lint:/root/.cache/golangci-lint $(TOOLS_IMAGE):$(TOOLS_IMAGE_TAG)
+# Mount the git common directory to the tools container.
+# This is needed when using git worktrees.
+GIT_COMMON_DIR := $(shell git rev-parse --git-common-dir 2>/dev/null)
+ifneq ($(strip $(GIT_COMMON_DIR)),)
+  ifneq ($(GIT_COMMON_DIR),.git)
+    WORKTREE_DOCKER_MOUNT := -v $(GIT_COMMON_DIR):$(GIT_COMMON_DIR)
+  endif
+endif
+
+TOOLS_CMD = docker run --rm -t -v ${PWD}:/tools $(WORKTREE_DOCKER_MOUNT) $(TOOLS_IMAGE):$(TOOLS_IMAGE_TAG)
+LINT_CMD =  docker run --rm -t -v ${PWD}:/tools $(WORKTREE_DOCKER_MOUNT) -v ${PWD}/.cache/golangci-lint:/root/.cache/golangci-lint $(TOOLS_IMAGE):$(TOOLS_IMAGE_TAG)
 
 .PHONY: tools-image-build
 tools-image-build:

docs/sources/tempo/configuration/_index.md

@@ -744,17 +744,12 @@ query_frontend:
         # (default: 168h)
         [max_duration: <duration>]
 
-        # query_backend_after and query_ingesters_until together control where the query-frontend searches for traces.
-        # Time ranges before query_ingesters_until will be searched in the ingesters only.
-        # Time ranges after query_backend_after will be searched in the backend/object storage only.
-        # Time ranges from query_backend_after through query_ingesters_until will be queried from both locations.
-        # query_backend_after must be less than or equal to query_ingesters_until.
+        # query_backend_after controls where the query-frontend searches for traces.
+        # Time ranges newer than query_backend_after will be searched in the live-stores only.
+        # Time ranges older than query_backend_after will be searched in the backend/object storage only.
         # (default: 15m)
         [query_backend_after: <duration>]
 
-        # (default: 30m)
-        [query_ingesters_until: <duration>]
-
         # If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
         # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
         # NOTE: Requires `duration_slo` AND `throughput_bytes_slo` to be configured.

docs/sources/tempo/configuration/manifest.md

@@ -400,7 +400,6 @@ query_frontend:
         max_result_limit: 0
         max_duration: 168h0m0s
         query_backend_after: 15m0s
-        query_ingesters_until: 30m0s
         ingester_shards: 3
         most_recent_shards: 200
         default_spans_per_span_set: 3

integration/util/config-query-backend.yaml

@@ -5,6 +5,5 @@
 query_frontend:
   search:
     query_backend_after: 0s
-    query_ingesters_until: 0s
   metrics:
     query_backend_after: 0s

modules/frontend/config.go

@@ -93,7 +93,6 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) {
 	cfg.Search = SearchConfig{
 		Sharder: SearchSharderConfig{
 			QueryBackendAfter:      15 * time.Minute,
-			QueryIngestersUntil:    30 * time.Minute,
 			DefaultLimit:           20,
 			MaxLimit:               0,
 			MaxDuration:            168 * time.Hour, // 1 week

modules/frontend/frontend.go

@@ -102,10 +102,6 @@ func New(cfg Config, next pipeline.RoundTripper, o overrides.Interface, reader t
 		return nil, fmt.Errorf("frontend search target bytes per request should be greater than 0")
 	}
 
-	if cfg.Search.Sharder.QueryIngestersUntil < cfg.Search.Sharder.QueryBackendAfter {
-		return nil, fmt.Errorf("query backend after should be less than or equal to query ingester until")
-	}
-
 	if cfg.Search.Sharder.MostRecentShards <= 0 {
 		return nil, fmt.Errorf("most recent shards must be greater than 0")
 	}

modules/frontend/frontend_test.go

@@ -132,25 +132,6 @@ func TestFrontendBadConfigFails(t *testing.T) {
 	assert.EqualError(t, err, "frontend search target bytes per request should be greater than 0")
 	assert.Nil(t, f)
 
-	f, err = New(Config{
-		TraceByID: TraceByIDConfig{
-			QueryShards: maxQueryShards,
-			SLO:         testSLOcfg,
-		},
-		Search: SearchConfig{
-			Sharder: SearchSharderConfig{
-				ConcurrentRequests:    defaultConcurrentRequests,
-				TargetBytesPerRequest: defaultTargetBytesPerRequest,
-				MostRecentShards:      defaultMostRecentShards,
-				QueryIngestersUntil:   time.Minute,
-				QueryBackendAfter:     time.Hour,
-			},
-			SLO: testSLOcfg,
-		},
-	}, nil, nil, nil, nil, "", fakeHTTPAuthMiddleware, nil, log.NewNopLogger(), nil)
-	assert.EqualError(t, err, "query backend after should be less than or equal to query ingester until")
-	assert.Nil(t, f)
-
 	f, err = New(Config{
 		TraceByID: TraceByIDConfig{
 			QueryShards: maxQueryShards,

modules/frontend/search_handlers_test.go

@@ -494,7 +494,6 @@ func TestSearchFailurePropagatesFromQueriers(t *testing.T) {
 					ConcurrentRequests:    defaultConcurrentRequests,
 					TargetBytesPerRequest: defaultTargetBytesPerRequest,
 					MostRecentShards:      defaultMostRecentShards,
-					QueryIngestersUntil:   100 * time.Hour,
 					QueryBackendAfter:     100 * time.Hour,
 					IngesterShards:        1,
 				},
@@ -545,7 +544,6 @@ func TestSearchFailurePropagatesFromQueriers(t *testing.T) {
 					ConcurrentRequests:    defaultConcurrentRequests,
 					TargetBytesPerRequest: defaultTargetBytesPerRequest,
 					MostRecentShards:      defaultMostRecentShards,
-					QueryIngestersUntil:   100 * time.Hour,
 					QueryBackendAfter:     100 * time.Hour,
 					IngesterShards:        1,
 				},
@@ -881,7 +879,6 @@ func frontendWithSettings(t require.TestingT, next pipeline.RoundTripper, rdr te
 					TargetBytesPerRequest: defaultTargetBytesPerRequest,
 					MostRecentShards:      defaultMostRecentShards,
 					QueryBackendAfter:     30 * time.Minute,
-					QueryIngestersUntil:   30 * time.Minute,
 					IngesterShards:        1,
 				},
 

modules/frontend/search_sharder.go

@@ -36,7 +36,6 @@ type SearchSharderConfig struct {
 	MaxDuration           time.Duration `yaml:"max_duration"`
 	// QueryBackendAfter determines when to query backend storage vs ingesters only.
 	QueryBackendAfter      time.Duration `yaml:"query_backend_after,omitempty"`
-	QueryIngestersUntil    time.Duration `yaml:"query_ingesters_until,omitempty"`
 	IngesterShards         int           `yaml:"ingester_shards,omitempty"`
 	MostRecentShards       int           `yaml:"most_recent_shards,omitempty"`
 	DefaultSpansPerSpanSet uint32        `yaml:"default_spans_per_span_set,omitempty"`
@@ -115,7 +114,7 @@ func (s asyncSearchSharder) RoundTrip(pipelineRequest pipeline.Request) (pipelin
 	// buffer of shards+1 allows us to insert ingestReq and metrics
 	reqCh := make(chan pipeline.Request, s.cfg.IngesterShards+1)
 
-	// build request to search ingesters based on query_ingesters_until config and time range
+	// build request to search ingesters based on query_backend_after config and time range
 	// pass subCtx in requests so we can cancel and exit early
 	jobMetrics, err := s.ingesterRequests(tenantID, pipelineRequest, *searchReq, reqCh)
 	if err != nil {
@@ -203,7 +202,7 @@ func (s *asyncSearchSharder) ingesterRequests(tenantID string, parent pipeline.R
 		return resp, buildIngesterRequest(tenantID, parent, &searchReq, reqCh)
 	}
 
-	ingesterUntil := uint32(time.Now().Add(-s.cfg.QueryIngestersUntil).Unix())
+	ingesterUntil := uint32(time.Now().Add(-s.cfg.QueryBackendAfter).Unix())
 
 	// if there's no overlap between the query and ingester range just return nil
 	if searchReq.End < ingesterUntil {