Remove v2 search

CHANGELOG.md

@@ -3,6 +3,15 @@
 * [FEATURE] Add flag to check configuration [#2131](https://github.com/grafana/tempo/issues/2131) (@robertscherbarth @agrib-01)
 * [FEATURE] Add flag to optionally enable all available Go runtime metrics [#2005](https://github.com/grafana/tempo/pull/2005) (@andreasgerstmayr)
 * [CHANGE] Add support for s3 session token in static config [#2093](https://github.com/grafana/tempo/pull/2093) (@farodin91)
+* [CHANGE] **Breaking Change** Remove support for search on v2 blocks. [#2159](https://github.com/grafana/tempo/pull/2062) (@joe-elliott)
+  Removed config options:
+  ```
+  overrides:
+    max_search_bytes_per_trace:
+    search_tags_allow_list:
+  ```
+  Removed metrics:
+  `tempo_ingester_trace_search_bytes_discarded_total`
 * [CHANGE] Stop caching parquet files for search [#2164](https://github.com/grafana/tempo/pull/2164) (@mapno)
 * [ENHANCEMENT] Metrics generator to make use of counters earlier [#2068](https://github.com/grafana/tempo/pull/2068) (@zalegrala)
 * [ENHANCEMENT] Log when a trace is too large to compact [#2105](https://github.com/grafana/tempo/pull/2105) (@scalalang2)

Makefile

@@ -12,7 +12,6 @@ GORELEASER := $(GOPATH)/bin/goreleaser
 
 # Build Images
 DOCKER_PROTOBUF_IMAGE ?= otel/build-protobuf:0.14.0
-FLATBUFFERS_IMAGE ?= neomantra/flatbuffers
 LOKI_BUILD_IMAGE ?= grafana/loki-build-image:0.21.0
 DOCS_IMAGE ?= grafana/docs-base:latest
 
@@ -218,14 +217,6 @@ gen-proto:
 
 	rm -rf $(PROTO_INTERMEDIATE_DIR)
 
-# ##############
-# Gen Flatbuffer
-# ##############
-.PHONY: gen-flat
-gen-flat:
-	# -o /pkg generates into same folder as tempo.fbs for simpler imports.
-	docker run --rm -v${PWD}:/opt/src ${FLATBUFFERS_IMAGE} flatc --go -o /opt/src/pkg /opt/src/pkg/tempofb/tempo.fbs
-
 # ##############
 # Gen Traceql
 # ##############
@@ -239,8 +230,8 @@ gen-traceql-local:
 
 ### Check vendored and generated files are up to date
 .PHONY: vendor-check
-vendor-check: gen-proto gen-flat update-mod gen-traceql
-	git diff --exit-code -- **/go.sum **/go.mod vendor/ pkg/tempopb/ pkg/tempofb/ pkg/traceql/
+vendor-check: gen-proto update-mod gen-traceql
+	git diff --exit-code -- **/go.sum **/go.mod vendor/ pkg/tempopb/ pkg/traceql/
 
 ### Tidy dependencies for tempo and tempo-serverless modules
 .PHONY: update-mod

cmd/tempo-serverless/cloud-run/go.mod

@@ -50,7 +50,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v22.10.26+incompatible // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect

cmd/tempo-serverless/cloud-run/go.sum

@@ -236,8 +236,6 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM=
-github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=

cmd/tempo-serverless/lambda/go.mod

@@ -52,7 +52,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v22.10.26+incompatible // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect

cmd/tempo-serverless/lambda/go.sum

@@ -240,8 +240,6 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM=
-github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=

cmd/tempo/app/modules.go

@@ -39,7 +39,6 @@ import (
 	"github.com/grafana/tempo/tempodb/backend/gcs"
 	"github.com/grafana/tempo/tempodb/backend/local"
 	"github.com/grafana/tempo/tempodb/backend/s3"
-	v2 "github.com/grafana/tempo/tempodb/encoding/v2"
 )
 
 // The various modules that make up tempo.
@@ -186,9 +185,6 @@ func (t *App) initDistributor() (services.Service, error) {
 }
 
 func (t *App) initIngester() (services.Service, error) {
-	// always use flatbuffer search if we're using the v2 blocks. todo: in 2.1 remove flatbuffer search altogether
-	t.cfg.Ingester.UseFlatbufferSearch = (t.cfg.StorageConfig.Trace.Block.Version == v2.VersionString)
-
 	t.cfg.Ingester.LifecyclerConfig.ListenPort = t.cfg.Server.GRPCListenPort
 	ingester, err := ingester.New(t.cfg.Ingester, t.store, t.overrides, prometheus.DefaultRegisterer)
 	if err != nil {

docs/sources/tempo/api_docs/_index.md

@@ -134,7 +134,8 @@ but if it can also send OpenTelemetry proto if `Accept: application/protobuf` is
 
 ### Search
 
-Tempo's Search API finds traces based on span and process attributes (tags and values).
+Tempo's Search API finds traces based on span and process attributes (tags and values). Note that search functionality is **not** available on
+[v2 blocks]({{< relref "../configuration/parquet#disable-parquet" >}}).
 
 When performing a search, Tempo does a massively parallel search over the given time range, and takes the first N results. Even identical searches will differ due to things like machine load and network latency. TraceQL follows the same behavior.
 

docs/sources/tempo/configuration/_index.md

@@ -179,11 +179,6 @@ distributor:
     # Disables write extension with inactive ingesters. Use this along with ingester.lifecycler.unregister_on_shutdown = true
     #  note that setting these two config values reduces tolerance to failures on rollout b/c there is always one guaranteed to be failing replica
     [extend_writes: <bool>]
-
-    # Optional.
-    # List of tags that will **not** be extracted from trace data for search lookups
-    # This is a global config that will apply to all tenants
-    [search_tags_deny_list: <list of string> | default = ]
 ```
 
 ## Ingester
@@ -1127,11 +1122,6 @@ overrides:
     # This override limit is used by the ingester.
     [max_traces_per_user: <int> | default = 10000]
 
-    # Maximum size of search data for a single trace in bytes. A value of 0
-    # disables the check. From an operational perspective, the size of search
-    # data is proportional to the total size of all tags in a trace.
-    [max_search_bytes_per_trace: <int> | default = 5000]
-
     # Maximum size in bytes of a tag-values query. Tag-values query is used mainly
     # to populate the autocomplete dropdown. This limit protects the system from
     # tags with high cardinality or large values such as HTTP URLs or SQL queries.

docs/sources/tempo/configuration/manifest.md

@@ -24,7 +24,7 @@ http_api_prefix: ""
 server:
   http_listen_network: tcp
   http_listen_address: ""
-  http_listen_port: 3100
+  http_listen_port: 3200
   http_listen_conn_limit: 0
   grpc_listen_network: tcp
   grpc_listen_address: ""
@@ -64,6 +64,50 @@ server:
   log_source_ips_regex: ""
   log_request_at_info_level_enabled: false
   http_path_prefix: ""
+internal_server:
+  http_listen_network: tcp
+  http_listen_address: localhost
+  http_listen_port: 3101
+  http_listen_conn_limit: 0
+  grpc_listen_network: ""
+  grpc_listen_address: ""
+  grpc_listen_port: 0
+  grpc_listen_conn_limit: 0
+  tls_cipher_suites: ""
+  tls_min_version: ""
+  http_tls_config:
+    cert_file: ""
+    key_file: ""
+    client_auth_type: ""
+    client_ca_file: ""
+  grpc_tls_config:
+    cert_file: ""
+    key_file: ""
+    client_auth_type: ""
+    client_ca_file: ""
+  register_instrumentation: false
+  graceful_shutdown_timeout: 30s
+  http_server_read_timeout: 30s
+  http_server_write_timeout: 30s
+  http_server_idle_timeout: 2m0s
+  grpc_server_max_recv_msg_size: 0
+  grpc_server_max_send_msg_size: 0
+  grpc_server_max_concurrent_streams: 0
+  grpc_server_max_connection_idle: 0s
+  grpc_server_max_connection_age: 0s
+  grpc_server_max_connection_age_grace: 0s
+  grpc_server_keepalive_time: 0s
+  grpc_server_keepalive_timeout: 0s
+  grpc_server_min_time_between_pings: 0s
+  grpc_server_ping_without_stream_allowed: false
+  log_format: ""
+  log_level: ""
+  log_source_ips_enabled: false
+  log_source_ips_header: ""
+  log_source_ips_regex: ""
+  log_request_at_info_level_enabled: false
+  http_path_prefix: ""
+  enable: false
 distributor:
   ring:
     kvstore:
@@ -98,7 +142,7 @@ distributor:
         mirror_timeout: 2s
     heartbeat_period: 5s
     heartbeat_timeout: 5m0s
-    instance_id: joe
+    instance_id: hostname
     instance_interface_names:
       - eth0
       - en0
@@ -109,12 +153,12 @@ distributor:
   log_received_traces: false
   forwarders: []
   extend_writes: true
-  search_tags_deny_list: []
 ingester_client:
   pool_config:
     checkinterval: 15s
     healthcheckenabled: true
     healthchecktimeout: 1s
+    maxconcurrenthealthchecks: 0
   remote_timeout: 5s
   grpc_client_config:
     max_recv_msg_size: 104857600
@@ -140,6 +184,7 @@ metrics_generator_client:
     checkinterval: 15s
     healthcheckenabled: true
     healthchecktimeout: 1s
+    maxconcurrenthealthchecks: 0
   remote_timeout: 5s
   grpc_client_config:
     max_recv_msg_size: 104857600
@@ -312,21 +357,8 @@ ingester:
     join_after: 0s
     min_ready_duration: 15s
     interface_names:
-      - wlp2s0
-      - br-39d728775b03
-      - br-d5846bf66182
-      - br-ea48fef4186e
-      - br-f163873defd4
-      - br-24f5062c6edd
-      - br-3b836e91bc36
-      - br-9cef180f0356
-      - br-a5544df3e712
-      - br-14ab1fbc2f0e
-      - br-16536cce4aa3
-      - docker0
-      - br-721c7a5d3933
-      - br-dd28551f2dbd
-      - br-d3d1776850a0
+      - eth0
+      - en0
     final_sleep: 0s
     tokens_file_path: ""
     availability_zone: ""
@@ -500,6 +532,7 @@ storage:
       hedge_requests_up_to: 2
       signature_v2: false
       forcepathstyle: false
+      bucket_lookup_type: 0
       tags: {}
       storage_class: ""
       metadata: {}
@@ -526,10 +559,8 @@ overrides:
   ingestion_rate_strategy: local
   ingestion_rate_limit_bytes: 15000000
   ingestion_burst_size_bytes: 20000000
-  search_tags_allow_list: null
   max_traces_per_user: 10000
   max_global_traces_per_user: 0
-  max_search_bytes_per_trace: 5000
   forwarders: []
   metrics_generator_ring_size: 0
   metrics_generator_processors: null

docs/sources/tempo/configuration/parquet.md

@@ -7,7 +7,7 @@ weight: 75
 # Apache Parquet block format
 
 
-Tempo has a default columnar block format based on Apache Parquet. [TraceQL]({{< relref "../traceql" >}}), the query language for traces, requires Parquet. In a future release, Parquet will be required to use traces search.
+Tempo has a default columnar block format based on Apache Parquet. Tags based search as well as [TraceQL]({{< relref "../traceql" >}}), the query language for traces, requires Parquet.
 
 A columnar block format may result in improved search performance and also enables a large ecosystem of tools access to the underlying trace data. For example, you can use `parquet-tools` to [query Parquet data]({{< relref "../operations/parquet" >}}).
 
@@ -19,11 +19,11 @@ If you install using the new Helm charts, then Parquet is enabled by default.
 
 The new Parquet block is enabled by default in Tempo 2.0. No data conversion or upgrade process is necessary. As soon as the Parquet format is enabled, Tempo starts writing data in that format, leaving existing data as-is.
 
-The new Parquet block format requires more CPU and memory resources than the previous v2 format but provides faster search and the new TraceQL.
+The new Parquet block format requires more CPU and memory resources than the previous v2 format but provides search and TraceQL functionality.
 
 ## Disable Parquet
 
-It is possible to disable Parquet and use the previous v2 block format. This disables TraceQL and will result in reduced search performance, but also reduces resource consumption, and may be desired for a high-throughput cluster that does not need these capabilities. Set the block format option to v2 in the Storage section of the configuration file.
+It is possible to disable Parquet and use the previous v2 block format. This disables all forms of search, but also reduces resource consumption, and may be desired for a high-throughput cluster that does not need these capabilities. Set the block format option to v2 in the Storage section of the configuration file.
 
 ```yaml
 # block format version. options: v2, vParquet

docs/sources/tempo/getting-started/tempo-in-grafana.md

@@ -29,14 +29,10 @@ You can run a TraceQL query either by issuing it to Tempo’s `q` parameter of t
 
 For details about how queries are constructed, read the [TraceQL documentation]({{< relref "../traceql" >}}).
 
-## Find traces using Tempo search
+## Find traces using Tempo tags search
 
 Search for traces using common dimensions such as time range, duration, span tags, service names, and more. Use the trace view to quickly diagnose errors and high-latency events in your system.
 
-Tempo includes the ability to search recent traces held in ingesters.
-Traces can be searched for data originating from a specific service,
-duration range, span, or process-level attributes included in your application's instrumentation, such as HTTP status code and customer ID.
-
 ### Non-deterministic search
 
 Most search functions are deterministic: using the same search criteria results in the same results.
@@ -46,34 +42,6 @@ If you perform the same search twice, you’ll get different lists, assuming the
 
 When performing a search, Tempo does a massively parallel search over the given time range, and takes the first N results. Even identical searches will differ due to things like machine load and network latency. This approach values speed over predictability and is quite simple; enforcing that the search results are consistent would introduce additional complexity (and increase the time the user spends waiting for results). TraceQL follows the same behavior.
 
-### Search of recent traces
-
-Search of recent traces is disabled by default.
-Ingesters default to storing the last 15 minutes of traces.
-
-To enable recent traces search:
-
--  Run Tempo, enabling search in the YAML configuration.
-Refer to the [search]({{< relref "../configuration#search" >}}) configuration documentation.
--  Run Grafana 8.2 or a more recent version. Enable the `tempoSearch` [feature toggle](https://github.com/grafana/tempo/blob/main/example/docker-compose/tempo-search/grafana.ini).
-
-<p align="center"><img src="../assets/tempo-search.png" alt="Tempo Search"></p>
-
-### Search of the backend datastore
-
-Tempo includes the the ability to search the entire backend datastore.
-
-To enable search of the backend datastore:
-
--  Run Tempo, enabling search in the YAML configuration.
-Refer to the [search]({{< relref "../configuration#search" >}}) configuration documentation.
-Further configuration information is in [backend search]({{< relref "../operations/backend_search" >}}).
-The Tempo configuration is the same for searching recent traces or
-for search of the backend datastore.
-
--  Run Grafana 8.3.6 or a more recent version. Enable the `tempoBackendSearch` [feature toggle](https://github.com/grafana/tempo/blob/main/example/docker-compose/tempo-search/grafana.ini). This will cause Grafana to pass the `start` and `end` parameters necessary for the backend datastore search.
-
-
 ## Service graph view
 
 Grafana provides a built-in service graph view available in Grafana Cloud and Grafana 9.1.

docs/sources/tempo/operations/backend_search.md

@@ -12,6 +12,9 @@ some suggestions for tuning your backend to improve performance.
 General advice is to scale your compactors and queriers. Additional queriers can more effectively run jobs in parallel
 while additional compactors will more aggressively reduce the length of your blocklist and copies of data (if using RF=3).
 
+**Note:** All forms of search (TraceQL and tags based) are only supported on the `vParquet` and forward blocks. [v2 blocks]({{< relref "../configuration/parquet#disable-parquet" >}})
+can only be used for trace by id lookup.
+
 ## Configuration
 
 Queriers and query frontends have additional configuration related

docs/sources/tempo/setup/tanka.md

@@ -303,7 +303,6 @@ Install the `k.libsonnet`, Jsonnet, and Memcachd libraries.
                },
            },
            overrides+: {
-               max_search_bytes_per_trace: 5000000,
                metrics_generator_processors: ['service-graphs', 'span-metrics'],
            },
        },

example/helm/microservices-tempo-values.yaml

@@ -43,7 +43,6 @@ config: |
     join_members:
       - tempo-tempo-distributed-gossip-ring
   overrides:
-    max_search_bytes_per_trace: 0
     per_tenant_override_config: /conf/overrides.yaml
   server:
     http_listen_port: 3100

go.mod

@@ -28,7 +28,6 @@ require (
 	github.com/gogo/status v1.1.1
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4
-	github.com/google/flatbuffers v22.10.26+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -430,8 +430,6 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM=
-github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=

integration/e2e/config-all-in-one-azurite.yaml

@@ -38,7 +38,4 @@ storage:
       storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
     pool:
       max_workers: 10
-      queue_depth: 100
-
-overrides:
-  max_search_bytes_per_trace: 50_000
+      queue_depth: 100