Support OR conditions for search tags and search tag values v2

CHANGELOG.md

@@ -1,6 +1,8 @@
 ## main / unreleased
 
 * [CHANGE] **BREAKING CHANGE** Remove duplicate "compaction" prefix from CompactorConfig CLI flags. Affected flags: `compaction.block-retention`, `compaction.max-objects-per-block`, `compaction.max-block-bytes`, `compaction.compaction-window`. [#6909](https://github.com/grafana/tempo/pull/6909) (@electron0zero)
+* [ENHANCEMENT] Support OR conditions for tag name and tag value autocomplete (search tags v2) [#6827](https://github.com/grafana/tempo/pull/6827) (@ie-pham)
+* [ENHANCEMENT] Expose MinIO retry settings via S3 config [#6561](https://github.com/grafana/tempo/pull/6561) (@rwhitty)
 * [CHANGE] **BREAKING CHANGE** Centralize block and WAL config: `block_builder` and `live_store` now always use `storage.trace.block` settings; per-module block config fields are removed. [#6647](https://github.com/grafana/tempo/pull/6647) (@stoewer)
 * [CHANGE] **BREAKING CHANGE** Remove Opencensus receiver [#6523](https://github.com/grafana/tempo/pull/6523) (@javiermolinar)
 * [CHANGE] Upgrade Tempo to Go 1.26.0 [#6443](https://github.com/grafana/tempo/pull/6443) (@stoewer)

docs/sources/tempo/api_docs/_index.md

@@ -411,7 +411,7 @@ Parameters:
   Specifies the scope of the tags, this is an optional parameter, if not specified it means all scopes.
   Default = `all`
 - `q = (traceql query)`
-  Optional. A TraceQL query to filter tag names by. Currently only works for a single spanset of `&&`ed conditions. For example: `{ span.foo = "bar" && resource.baz = "bat" ...}`. See also [Filtered tag values](#filtered-tag-values).
+  Optional. A TraceQL query to filter tag names by. Supports `&&` and `||` operators within a single spanset. For example: `{ span.foo = "bar" && resource.baz = "bat" }` or `{ span.foo = "bar" || resource.baz = "bat" }`. See also [Filtered tag values](#filtered-tag-values).
 - `start = (unix epoch seconds)`
   Optional. Along with `end` define a time range from which tags should be returned.
 - `end = (unix epoch seconds)`
@@ -608,7 +608,7 @@ Parameters:
 - `end = (unix epoch seconds)`
   Optional. Along with `start`, defines a time range from which tags values should be returned. Providing both `start` and `end` includes blocks for the specified time range only.
 - `q = (traceql query)`
-  Optional. A TraceQL query to filter tag values by. Currently only works for a single spanset of `&&`ed conditions. For example: `{ span.foo = "bar" && resource.baz = "bat" ...}`. Refer to [Filtered tag values](#filtered-tag-values).
+  Optional. A TraceQL query to filter tag values by. Supports `&&` and `||` operators within a single spanset. For example: `{ span.foo = "bar" && resource.baz = "bat" }` or `{ span.foo = "bar" || resource.baz = "bat" }`. Refer to [Filtered tag values](#filtered-tag-values).
 - `limit = (integer)`
   Optional. Limits the maximum number of tags values
 - `maxStaleValues = (integer)`
@@ -632,12 +632,13 @@ Tempo extracts only the valid matchers and builds a valid query.
 If an input is invalid, Tempo doesn't provide an error. Instead,
 you'll see the whole list when a failure of parsing input. This behavior helps with backwards compatibility.
 
-Only queries with a single selector `{}` and AND `&&` operators are supported.
+Queries must use a single spanset selector `{}`. Both `&&` and `||` operators are supported within the selector.
 
 - Example supported: `{ resource.cluster = "us-east-1" && resource.service = "frontend" }`
-- Example unsupported: `{ resource.cluster = "us-east-1" || resource.service = "frontend" } && { resource.cluster = "us-east-2" }`
+- Example supported: `{ resource.cluster = "us-east-1" || resource.service = "frontend" }`
+- Example unsupported: `{ resource.cluster = "us-east-1" } && { resource.cluster = "us-east-2" }`
 
-Unscoped attributes aren't supported for filtered tag values.
+Unscoped attributes are supported for filtered tag values. When you use an unscoped attribute name, Tempo looks for matches in both span and resource scopes.
 
 The following request returns all discovered service names on spans with `span.http.method=GET`:
 

docs/sources/tempo/configuration/manifest.md

@@ -692,6 +692,7 @@ overrides:
             retry_info_enabled: true
         read:
             max_bytes_per_tag_values_query: 1000000
+            max_condition_groups_per_tag_query: 100
         metrics_generator:
             generate_native_histograms: classic
             native_histogram_bucket_factor: 1.1

integration/api/api_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"slices"
 	"sort"
 	"strings"
@@ -42,6 +43,9 @@ func TestTagEndpoints(t *testing.T) {
 			{spanCount: 2, name: "baz", resourceAttr: "secondRes", resourceAttVal: "qux", SpanAttr: "secondSpan", spanAttVal: "qux"},
 			{spanCount: 2, name: "foo", resourceAttr: "twoRes", resourceAttVal: "bar", SpanAttr: "twoSpan", spanAttVal: "bar"},
 			{spanCount: 2, name: "baz", resourceAttr: "twoRes", resourceAttVal: "qux", SpanAttr: "twoSpan", spanAttVal: "qux"},
+			{spanCount: 2, name: "foo", resourceAttr: "secondRes", resourceAttVal: "qux", SpanAttr: "secondSpan", spanAttVal: "bar"},
+			{spanCount: 2, name: "boo", resourceAttr: "sameAttr", resourceAttVal: "qux", SpanAttr: "someSpan", spanAttVal: "bar"},
+			{spanCount: 2, name: "boo", resourceAttr: "someRes", resourceAttVal: "qux", SpanAttr: "sameAttr", spanAttVal: "bar"},
 		}
 
 		for _, b := range batches {
@@ -69,7 +73,7 @@ func TestTagEndpoints(t *testing.T) {
 
 		// Test V1 API endpoints (backwards compatibility)
 		t.Run("tags_v1_wal", func(t *testing.T) {
-			expectedTags := []string{"firstRes", "firstSpan", "secondRes", "secondSpan", "service.name", "twoRes", "twoSpan"}
+			expectedTags := []string{"firstRes", "firstSpan", "sameAttr", "secondRes", "secondSpan", "service.name", "someRes", "someSpan", "twoRes", "twoSpan"}
 			callSearchTagsAndAssert(t, h, expectedTags, 0, 0)
 		})
 
@@ -101,7 +105,7 @@ func TestTagEndpoints(t *testing.T) {
 
 		// Test V1 API endpoints on backend (backwards compatibility)
 		t.Run("tags_v1_backend", func(t *testing.T) {
-			expectedTags := []string{"firstRes", "firstSpan", "secondRes", "secondSpan", "service.name", "twoRes", "twoSpan"}
+			expectedTags := []string{"firstRes", "firstSpan", "sameAttr", "secondRes", "secondSpan", "service.name", "someRes", "someSpan", "twoRes", "twoSpan"}
 			callSearchTagsAndAssert(t, h, expectedTags, start.Unix(), end.Unix())
 		})
 
@@ -132,11 +136,11 @@ func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
 				Scopes: []*tempopb.SearchTagsV2Scope{
 					{
 						Name: "span",
-						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+						Tags: []string{batches[0].SpanAttr, batches[6].SpanAttr, batches[1].SpanAttr, batches[5].SpanAttr, batches[2].SpanAttr},
 					},
 					{
 						Name: "resource",
-						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+						Tags: []string{batches[0].resourceAttr, batches[5].resourceAttr, batches[1].resourceAttr, batches[6].resourceAttr, batches[2].resourceAttr, "service.name"},
 					},
 				},
 			},
@@ -150,11 +154,11 @@ func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
 				Scopes: []*tempopb.SearchTagsV2Scope{
 					{
 						Name: "span",
-						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+						Tags: []string{batches[0].SpanAttr, batches[6].SpanAttr, batches[1].SpanAttr, batches[5].SpanAttr, batches[2].SpanAttr},
 					},
 					{
 						Name: "resource",
-						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+						Tags: []string{batches[0].resourceAttr, batches[5].resourceAttr, batches[1].resourceAttr, batches[6].resourceAttr, batches[2].resourceAttr, "service.name"},
 					},
 				},
 			},
@@ -167,7 +171,7 @@ func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
 				Scopes: []*tempopb.SearchTagsV2Scope{
 					{
 						Name: "resource",
-						Tags: []string{batches[0].resourceAttr, batches[2].resourceAttr, "service.name"},
+						Tags: []string{batches[0].resourceAttr, batches[2].resourceAttr, batches[4].resourceAttr, "service.name"},
 					},
 				},
 			},
@@ -246,11 +250,11 @@ func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
 				Scopes: []*tempopb.SearchTagsV2Scope{
 					{
 						Name: "span",
-						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+						Tags: []string{batches[0].SpanAttr, batches[6].SpanAttr, batches[1].SpanAttr, batches[5].SpanAttr, batches[2].SpanAttr},
 					},
 					{
 						Name: "resource",
-						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+						Tags: []string{batches[0].resourceAttr, batches[5].resourceAttr, batches[1].resourceAttr, batches[6].resourceAttr, batches[2].resourceAttr, "service.name"},
 					},
 				},
 			},
@@ -263,11 +267,11 @@ func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
 				Scopes: []*tempopb.SearchTagsV2Scope{
 					{
 						Name: "span",
-						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+						Tags: []string{batches[0].SpanAttr, batches[6].SpanAttr, batches[1].SpanAttr, batches[5].SpanAttr, batches[2].SpanAttr},
 					},
 					{
 						Name: "resource",
-						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+						Tags: []string{batches[0].resourceAttr, batches[5].resourceAttr, batches[1].resourceAttr, batches[6].resourceAttr, batches[2].resourceAttr, "service.name"},
 					},
 				},
 			},
@@ -280,11 +284,11 @@ func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
 				Scopes: []*tempopb.SearchTagsV2Scope{
 					{
 						Name: "span",
-						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+						Tags: []string{batches[0].SpanAttr, batches[6].SpanAttr, batches[1].SpanAttr, batches[5].SpanAttr, batches[2].SpanAttr},
 					},
 					{
 						Name: "resource",
-						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+						Tags: []string{batches[0].resourceAttr, batches[5].resourceAttr, batches[1].resourceAttr, batches[6].resourceAttr, batches[2].resourceAttr, "service.name"},
 					},
 				},
 			},
@@ -297,11 +301,59 @@ func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
 				Scopes: []*tempopb.SearchTagsV2Scope{
 					{
 						Name: "span",
-						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+						Tags: []string{batches[0].SpanAttr, batches[6].SpanAttr, batches[1].SpanAttr, batches[5].SpanAttr, batches[2].SpanAttr},
 					},
 					{
 						Name: "resource",
-						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+						Tags: []string{batches[0].resourceAttr, batches[5].resourceAttr, batches[1].resourceAttr, batches[6].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		// OR conditions
+		{
+			name:  "OR - two resource attrs from different batches",
+			query: fmt.Sprintf(`{ resource.%s="%s" || resource.%s="%s" }`, batches[0].resourceAttr, batches[0].resourceAttVal, batches[1].resourceAttr, batches[1].resourceAttVal),
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "OR - same resource attr two values",
+			query: fmt.Sprintf(`{ resource.%s="%s" || resource.%s="%s" }`, batches[2].resourceAttr, batches[2].resourceAttVal, batches[3].resourceAttr, batches[3].resourceAttVal),
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[2].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "OR - left branch matches only",
+			query: fmt.Sprintf(`{ resource.%s="%s" || resource.nonexistent="nope" }`, batches[0].resourceAttr, batches[0].resourceAttVal),
+			scope: "resource",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, "service.name"},
 					},
 				},
 			},
@@ -414,7 +466,7 @@ func buildSearchTagValuesV2TestCases(batches []batchTmpl) []struct {
 			query:   `{ resource.service.name="my-service"}`,
 			tagName: "name",
 			expected: &tempopb.SearchTagValuesV2Response{
-				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[3].name}, {Type: "string", Value: batches[2].name}},
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[3].name}, {Type: "string", Value: batches[5].name}, {Type: "string", Value: batches[2].name}},
 			},
 		},
 		{
@@ -433,6 +485,45 @@ func buildSearchTagValuesV2TestCases(batches []batchTmpl) []struct {
 				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}, {Type: "string", Value: batches[3].spanAttVal}},
 			},
 		},
+		// OR conditions
+		{
+			name:    "OR - both branches match different values",
+			query:   fmt.Sprintf(`{ resource.%s="%s" || resource.%s="%s" }`, batches[2].resourceAttr, batches[2].resourceAttVal, batches[3].resourceAttr, batches[3].resourceAttVal),
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}, {Type: "string", Value: batches[3].spanAttVal}},
+			},
+		},
+		{
+			name:    "OR - only left branch matches",
+			query:   fmt.Sprintf(`{ resource.%s="%s" || resource.nonexistent="nope" }`, batches[2].resourceAttr, batches[2].resourceAttVal),
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}},
+			},
+		},
+		{
+			name:     "OR - neither branch matches",
+			query:    fmt.Sprintf(`{ resource.%s="nope" || resource.%s="nope" }`, batches[2].resourceAttr, batches[3].resourceAttr),
+			tagName:  "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{},
+		},
+		{
+			name:    "OR - different attributes each matching a different batch",
+			query:   fmt.Sprintf(`{ span:name="%s" || resource.%s="%s" }`, batches[1].name, batches[4].resourceAttr, batches[4].resourceAttVal),
+			tagName: "span.secondSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[4].spanAttVal}, {Type: "string", Value: batches[1].spanAttVal}},
+			},
+		},
+		{
+			name:    "OR - unscoped tag with query having different attributes each matching a different batch",
+			query:   fmt.Sprintf(`{ span.%s="%s" || resource.%s="%s" }`, batches[5].SpanAttr, batches[5].spanAttVal, batches[6].resourceAttr, batches[6].resourceAttVal),
+			tagName: ".sameAttr",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[6].spanAttVal}, {Type: "string", Value: batches[5].resourceAttVal}},
+			},
+		},
 	}
 }
 
@@ -546,6 +637,71 @@ func TestSearchTagValuesV2_badRequest(t *testing.T) {
 	})
 }
 
+const configMaxConditionGroups = "./config-max-condition-groups.yaml"
+
+// TestSearchTags_maxConditionGroupsExceeded verifies that both the HTTP and gRPC tag search
+// endpoints return a 400 / InvalidArgument error when a query expands into more OR condition
+// groups than the configured limit.
+func TestSearchTags_maxConditionGroupsExceeded(t *testing.T) {
+	// Set max_condition_groups_per_tag_query to 1 so any OR query with two branches exceeds it.
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configMaxConditionGroups,
+	}, func(h *util.TempoHarness) {
+		// A query with two OR branches produces 2 condition groups, exceeding the limit of 1.
+		overLimitQuery := `{ resource.service.name="foo" || resource.service.name="bar" }`
+
+		// --- SearchTagsV2 HTTP ---
+		req, err := http.NewRequest(http.MethodGet,
+			fmt.Sprintf("%s/api/v2/search/tags?q=%s", h.BaseURL(), url.QueryEscape(overLimitQuery)), nil)
+		require.NoError(t, err)
+
+		res, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusBadRequest, res.StatusCode)
+
+		// --- SearchTagsV2 gRPC ---
+		grpcClient, ctx, err := h.APIClientGRPC("")
+		require.NoError(t, err)
+
+		tagsStream, err := grpcClient.SearchTagsV2(ctx, &tempopb.SearchTagsRequest{
+			Query: overLimitQuery,
+		})
+		require.NoError(t, err)
+
+		_, err = tagsStream.Recv()
+		require.Error(t, err)
+
+		st, ok := status.FromError(err)
+		require.True(t, ok)
+		require.Equal(t, codes.InvalidArgument, st.Code())
+
+		// --- SearchTagValuesV2 HTTP ---
+		req, err = http.NewRequest(http.MethodGet,
+			fmt.Sprintf("%s/api/v2/search/tag/resource.service.name/values?q=%s", h.BaseURL(), url.QueryEscape(overLimitQuery)), nil)
+		require.NoError(t, err)
+
+		res, err = http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusBadRequest, res.StatusCode)
+
+		// --- SearchTagValuesV2 gRPC ---
+		valuesStream, err := grpcClient.SearchTagValuesV2(ctx, &tempopb.SearchTagValuesRequest{
+			TagName: "resource.service.name",
+			Query:   overLimitQuery,
+		})
+		require.NoError(t, err)
+
+		_, err = valuesStream.Recv()
+		require.Error(t, err)
+
+		st, ok = status.FromError(err)
+		require.True(t, ok)
+		require.Equal(t, codes.InvalidArgument, st.Code())
+	})
+}
+
 func callSearchTagValuesV2AndAssert(t *testing.T, h *util.TempoHarness, tagName, query string, expected *tempopb.SearchTagValuesV2Response, start, end int64) {
 	apiClient := h.APIClientHTTP("")
 	response, err := apiClient.SearchTagValuesV2WithRange(tagName, query, start, end)

integration/api/config-max-condition-groups.yaml

@@ -0,0 +1,4 @@
+overrides:
+  defaults:
+    read:
+      max_condition_groups_per_tag_query: 1

modules/frontend/mcp_tools.go

@@ -298,10 +298,10 @@ func (s *MCPServer) handleGetAttributeValues(ctx context.Context, request mcp.Ca
 	}
 
 	query := request.GetString("filter-query", "")
-	if query != "" {
-		extractedReq := traceql.ExtractFetchRequest(query)
-		if extractedReq == nil || !extractedReq.AllConditions {
-			return mcp.NewToolResultError("filter-query invalid. It can only have one spanset and only &&'ed conditions like { <cond> && <cond> && ... }"), nil
+	if !traceql.IsEmptyQuery(query) {
+		conditionGroups, _ := traceql.ExtractConditionGroups(query, traceql.DefaultMaxConditionGroupsPerTagQuery)
+		if len(conditionGroups) == 0 {
+			return mcp.NewToolResultError("filter-query invalid. It must have a single spanset filter with &&/|| conditions like { <cond> && <cond> } or { <cond> || <cond> }"), nil
 		}
 	}
 

modules/frontend/mcp_tools_test.go

@@ -423,7 +423,7 @@ func TestHandleGetAttributeValues(t *testing.T) {
 				"filter-query": "{ foo bar baz }",
 			}),
 			expected: expectedResult{
-				err: "filter-query invalid. It can only have one spanset and only &&'ed conditions like { <cond> && <cond> && ... }",
+				err: "filter-query invalid. It must have a single spanset filter with &&/|| conditions like { <cond> && <cond> } or { <cond> || <cond> }",
 			},
 		},
 	}