Fix tempo-cli list block encoding and add some helpful things

CHANGELOG.md

@@ -5,6 +5,7 @@
 * [CHANGE] New compactor setting for max block size data instead of traces. [#520](https://github.com/grafana/tempo/pull/520)
 * [CHANGE] Change default ingester_client compression from gzip to snappy. [#522](https://github.com/grafana/tempo/pull/522)
 * [CHANGE/BUGFIX] Rename `tempodb_compaction_objects_written` and `tempodb_compaction_bytes_written` metrics to `tempodb_compaction_objects_written_total` and `tempodb_compaction_bytes_written_total`. [#524](https://github.com/grafana/tempo/pull/524)
+* [CHANGE] Replace tempo-cli `list block` `--check-dupes` option with `--scan` and collect additional stats [#534](https://github.com/grafana/tempo/pull/534) 
 * [FEATURE] Added block compression.  This is a **breaking change** b/c some configuration fields moved. [#504](https://github.com/grafana/tempo/pull/504)
 * [ENHANCEMENT] Serve config at the "/config" endpoint. [#446](https://github.com/grafana/tempo/pull/446)
 * [ENHANCEMENT] Switch blocklist polling and retention to different concurrency mechanism, add configuration options. [#475](https://github.com/grafana/tempo/issues/475)

cmd/tempo-cli/cmd-list-block.go

@@ -5,6 +5,9 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -16,9 +19,9 @@ import (
 type listBlockCmd struct {
 	backendOptions
 
-	TenantID   string `arg:"" help:"tenant-id within the bucket"`
-	BlockID    string `arg:"" help:"block ID to list"`
-	CheckDupes bool   `help:"check contents of block for duplicate trace IDs (warning, can be intense)"`
+	TenantID string `arg:"" help:"tenant-id within the bucket"`
+	BlockID  string `arg:"" help:"block ID to list"`
+	Scan     bool   `help:"scan contents of block for duplicate trace IDs and other info (warning, can be intense)"`
 }
 
 func (cmd *listBlockCmd) Run(ctx *globalOptions) error {
@@ -27,10 +30,10 @@ func (cmd *listBlockCmd) Run(ctx *globalOptions) error {
 		return err
 	}
 
-	return dumpBlock(r, c, cmd.TenantID, time.Hour, cmd.BlockID, cmd.CheckDupes)
+	return dumpBlock(r, c, cmd.TenantID, time.Hour, cmd.BlockID, cmd.Scan)
 }
 
-func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID string, windowRange time.Duration, blockID string, checkDupes bool) error {
+func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID string, windowRange time.Duration, blockID string, scan bool) error {
 	id := uuid.MustParse(blockID)
 
 	meta, err := r.BlockMeta(context.TODO(), id, tenantID)
@@ -62,10 +65,16 @@ func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID s
 	fmt.Println("Duration      : ", fmt.Sprint(unifiedMeta.end.Sub(unifiedMeta.start).Round(time.Second)))
 	fmt.Println("Age           : ", fmt.Sprint(time.Since(unifiedMeta.end).Round(time.Second)))
 
-	if checkDupes {
-		fmt.Println("Searching for dupes ...")
+	if scan {
+		fmt.Println("Scanning block contents.  Press CRTL+C to quit ...")
+
+		en, err := tempodb_backend.ParseEncoding(unifiedMeta.encoding)
+		if err != nil {
+			return err
+		}
 
 		block, err := encoding.NewBackendBlock(&tempodb_backend.BlockMeta{
+			Encoding: en,
 			Version:  unifiedMeta.version,
 			TenantID: tenantID,
 			BlockID:  id,
@@ -74,23 +83,53 @@ func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID s
 			return err
 		}
 
-		iter, err := block.Iterator(10 * 1024 * 1024)
+		iter, err := block.Iterator(uint32(2 * 1024 * 1024))
 		if err != nil {
 			return err
 		}
 		defer iter.Close()
 
+		// Scanning stats
 		i := 0
 		dupe := 0
+		maxObjSize := 0
+		minObjSize := 0
+
+		printStats := func() {
+			fmt.Println()
+			fmt.Println("Scanning results:")
+			fmt.Println("Objects scanned : ", i)
+			fmt.Println("Duplicates      : ", dupe)
+			fmt.Println("Smallest object : ", humanize.Bytes(uint64(minObjSize)))
+			fmt.Println("Largest object  : ", humanize.Bytes(uint64(maxObjSize)))
+		}
+
+		// Print stats on ctrl+c
+		c := make(chan os.Signal)
+		signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+		go func() {
+			<-c
+			printStats()
+			os.Exit(0)
+		}()
+
 		prevID := make([]byte, 16)
 		for {
-			objID, _, err := iter.Next()
+			objID, obj, err := iter.Next()
 			if err == io.EOF {
 				break
 			} else if err != nil {
 				return err
 			}
 
+			if len(obj) > maxObjSize {
+				maxObjSize = len(obj)
+			}
+
+			if len(obj) < minObjSize || minObjSize == 0 {
+				minObjSize = len(obj)
+			}
+
 			if bytes.Equal(objID, prevID) {
 				dupe++
 			}
@@ -102,8 +141,7 @@ func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID s
 			}
 		}
 
-		fmt.Println("total: ", i)
-		fmt.Println("dupes: ", dupe)
+		printStats()
 	}
 
 	return nil

docs/tempo/website/cli/_index.md

@@ -97,7 +97,7 @@ tempo-cli list blocks -c ./tempo.yaml single-tenant
 ```
 
 ## List Block
-Lists information about a single block, and optionally, perform an integrity check for duplicate records.
+Lists information about a single block, and optionally, scan its contents.
 
 ```bash
 tempo-cli list block <tenant-id> <block-id>
@@ -108,7 +108,7 @@ Arguments:
 - `block-id` The block ID as UUID string.
 
 Options:
-- `--check-dupes` Also load the block data and perform integrity check for duplicates. **Note:** can be intense.
+- `--scan` Also load the block data, perform integrity check for duplicates, and collect statistics. **Note:** can be intense.
 
 **Example:**
 ```bash