Improve defaults for worker pool

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * [CHANGE] Vulture now exercises search at any point during the block retention to test full backend search. 
   **BREAKING CHANGE** Dropped `tempo-search-retention-duration` parameter.  [#1297](https://github.com/grafana/tempo/pull/1297) (@joe-elliott)
+* [CHANGE] Updated storage.trace.pool.queue_depth default from 200->10000. [#1345](https://github.com/grafana/tempo/pull/1345) (@joe-elliott)
 * [FEATURE]: v2 object encoding added. This encoding adds a start/end timestamp to every record to reduce proto marshalling and increase search speed.  
   **BREAKING CHANGE** After this rollout the distributors will use a new API on the ingesters. As such you must rollout all ingesters before rolling the 
   distributors. Also, during this period, the ingesters will use considerably more resources and as such should be scaled up (or incoming traffic should be

docs/tempo/website/configuration/_index.md

@@ -642,7 +642,7 @@ storage:
         # the worker pool is used primarily when finding traces by id, but is also used by other
         pool:
 
-            # total number of workers pulling jobs from the queue (default: 30)
+            # total number of workers pulling jobs from the queue (default: 50)
             [max_workers: <int>] 
 
             # length of job queue. imporatant for querier as it queues a job for every block it has to search

docs/tempo/website/configuration/manifest.md

@@ -414,7 +414,7 @@ storage:
   trace:
     pool:
       max_workers: 50
-      queue_depth: 200
+      queue_depth: 10000
     wal:
       path: /tmp/tempo/wal
       completedfilepath: /tmp/tempo/wal/completed

modules/storage/config.go

@@ -82,5 +82,5 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 
 	cfg.Trace.Pool = &pool.Config{}
 	f.IntVar(&cfg.Trace.Pool.MaxWorkers, util.PrefixConfig(prefix, "trace.pool.max-workers"), 50, "Workers in the worker pool.")
-	f.IntVar(&cfg.Trace.Pool.QueueDepth, util.PrefixConfig(prefix, "trace.pool.queue-depth"), 200, "Work item queue depth.")
+	f.IntVar(&cfg.Trace.Pool.QueueDepth, util.PrefixConfig(prefix, "trace.pool.queue-depth"), 10000, "Work item queue depth.")
 }