Skip to content

Drop MinID/MaxID from BlockMeta #3950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zalegrala merged 2 commits into from
Aug 9, 2024
Merged

Drop MinID/MaxID from BlockMeta #3950

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
24924ca
Drop MinID/MaxID from BlockMeta
zalegrala Aug 9, 2024
d0c0d8a
Update changelog
zalegrala Aug 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions tempodb/backend/block_meta.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package backend

import (
"bytes"
"encoding/json"
"errors"
"fmt"
Expand Down Expand Up @@ -100,10 +99,6 @@ type BlockMeta struct {
Version string `json:"format"`
// BlockID is a unique identifier of the block.
BlockID uuid.UUID `json:"blockID"`
// MinID is the smallest object id stored in this block.
MinID []byte `json:"minID"`
// MaxID is the largest object id stored in this block.
MaxID []byte `json:"maxID"`
// A TenantID that defines the tenant to which this block belongs.
TenantID string `json:"tenantID"`
// StartTime roughly matches when the first obj was written to this block. It is used to determine block.
Expand Down Expand Up @@ -187,8 +182,6 @@ func NewBlockMetaWithDedicatedColumns(tenantID string, blockID uuid.UUID, versio
b := &BlockMeta{
Version: version,
BlockID: blockID,
MinID: []byte{},
MaxID: []byte{},
TenantID: tenantID,
Encoding: encoding,
DataEncoding: dataEncoding,
Expand All @@ -215,13 +208,6 @@ func (b *BlockMeta) ObjectAdded(id []byte, start, end uint32) {
}
}

if len(b.MinID) == 0 || bytes.Compare(id, b.MinID) == -1 {
b.MinID = id
}
if len(b.MaxID) == 0 || bytes.Compare(id, b.MaxID) == 1 {
b.MaxID = id
}

b.TotalObjects++
}

Expand Down
16 changes: 0 additions & 16 deletions tempodb/backend/block_meta_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/stretchr/testify/require"

"github.com/grafana/tempo/pkg/tempopb"
"github.com/grafana/tempo/pkg/util"
)

const (
Expand Down Expand Up @@ -40,8 +39,6 @@ func TestBlockMetaObjectAdded(t *testing.T) {
ids [][]byte
starts []uint32
ends []uint32
expectedMaxID []byte
expectedMinID []byte
expectedStart time.Time
expectedEnd time.Time
expectedObjects int
Expand All @@ -57,8 +54,6 @@ func TestBlockMetaObjectAdded(t *testing.T) {
ends: []uint32{
uint32(now.Add(time.Minute).Unix()),
},
expectedMaxID: []byte{0x01},
expectedMinID: []byte{0x01},
expectedStart: now,
expectedEnd: now.Add(time.Minute),
expectedObjects: 1,
Expand All @@ -76,8 +71,6 @@ func TestBlockMetaObjectAdded(t *testing.T) {
uint32(now.Add(time.Hour).Unix()),
uint32(now.Add(time.Minute).Unix()),
},
expectedMaxID: []byte{0x02},
expectedMinID: []byte{0x01},
expectedStart: now.Add(-time.Minute),
expectedEnd: now.Add(time.Hour),
expectedObjects: 2,
Expand All @@ -91,8 +84,6 @@ func TestBlockMetaObjectAdded(t *testing.T) {
b.ObjectAdded(tc.ids[i], tc.starts[i], tc.ends[i])
}

assert.Equal(t, tc.expectedMaxID, b.MaxID)
assert.Equal(t, tc.expectedMinID, b.MinID)
assert.Equal(t, tc.expectedStart, b.StartTime)
assert.Equal(t, tc.expectedEnd, b.EndTime)
assert.Equal(t, tc.expectedObjects, b.TotalObjects)
Expand All @@ -106,14 +97,9 @@ func TestBlockMetaParsing(t *testing.T) {
return date
}

minID, _ := util.HexStringToTraceID("00203ff2da512a3b4fab11d7243ac1cc")
maxID, _ := util.HexStringToTraceID("f12b7a1ad3115ff207734fab0d0ab235")

meta := BlockMeta{
Version: "vParquet3",
BlockID: uuid.MustParse("00000000-0000-0000-0000-000000000000"),
MinID: minID,
MaxID: maxID,
TenantID: "single-tenant",
StartTime: timeParse("2021-01-01T00:00:00.0000000Z"),
EndTime: timeParse("2021-01-02T00:00:00.0000000Z"),
Expand All @@ -136,8 +122,6 @@ func TestBlockMetaParsing(t *testing.T) {
expectedJSON := `{
"format": "vParquet3",
"blockID": "00000000-0000-0000-0000-000000000000",
"minID": "ACA/8tpRKjtPqxHXJDrBzA==",
"maxID": "8St6GtMRX/IHc0+rDQqyNQ==",
"tenantID": "single-tenant",
"startTime": "2021-01-01T00:00:00Z",
"endTime": "2021-01-02T00:00:00Z",
Expand Down
4 changes: 0 additions & 4 deletions tempodb/encoding/v2/streaming_block_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,6 @@ func TestStreamingBlockAddObject(t *testing.T) {

assert.Equal(t, time.Unix(10000, 0), meta.StartTime)
assert.Equal(t, time.Unix(25000, 0), meta.EndTime)
assert.Equal(t, minID, common.ID(meta.MinID))
assert.Equal(t, maxID, common.ID(meta.MaxID))
assert.Equal(t, testTenantID, meta.TenantID)
assert.Equal(t, numObjects, meta.TotalObjects)
assert.Greater(t, meta.Size, uint64(0))
Expand Down Expand Up @@ -281,8 +279,6 @@ func streamingBlock(t *testing.T, cfg *common.BlockConfig, w backend.Writer) (*S
require.NoError(t, err)

// test downsample config
require.True(t, bytes.Equal(block.BlockMeta().MinID, minID))
require.True(t, bytes.Equal(block.BlockMeta().MaxID, maxID))
require.Equal(t, originatingMeta.StartTime, block.BlockMeta().StartTime)
require.Equal(t, originatingMeta.EndTime, block.BlockMeta().EndTime)
require.Equal(t, originatingMeta.TenantID, block.BlockMeta().TenantID)
Expand Down
3 changes: 1 addition & 2 deletions tempodb/encoding/vparquet2/block_findtracebyid_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,7 @@ func BenchmarkFindTraceByID(b *testing.B) {
meta, err := rr.BlockMeta(ctx, blockID, tenantID)
require.NoError(b, err)

traceID := meta.MinID

traceID := []byte{}
block := newBackendBlock(meta, rr)

// idx := genIndex(b, block)
Expand Down
2 changes: 1 addition & 1 deletion tempodb/encoding/vparquet3/block_findtracebyid_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func BenchmarkFindTraceByID(b *testing.B) {
meta, err := rr.BlockMeta(ctx, blockID, tenantID)
require.NoError(b, err)

traceID := meta.MinID
traceID := []byte{}
block := newBackendBlock(meta, rr)

// index := genIndex(b, block)
Expand Down
5 changes: 1 addition & 4 deletions tempodb/encoding/vparquet3/block_traceql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2657,10 +2657,7 @@ func (b *backendBlock) rowGroupsForShard(ctx context.Context, pf *parquet.File,
matches := []parquet.RowGroup{}
for i := 0; i < len(index.RowGroups); i++ {
if i == 0 {
// The index contains the max trace ID for each row
// group. So to determine the min/max for the first
// entry we use the minimum ID from block meta.
if testRange(m.MinID, index.RowGroups[i]) {
if testRange([]byte{}, index.RowGroups[i]) {
matches = append(matches, rgs[i])
}
} else {
Expand Down
2 changes: 1 addition & 1 deletion tempodb/encoding/vparquet4/block_findtracebyid_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func BenchmarkFindTraceByID(b *testing.B) {
meta, err := rr.BlockMeta(ctx, blockID, tenantID)
require.NoError(b, err)

traceID := meta.MinID
traceID := []byte{}
block := newBackendBlock(meta, rr)

// index := genIndex(b, block)
Expand Down
5 changes: 1 addition & 4 deletions tempodb/encoding/vparquet4/block_traceql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3164,10 +3164,7 @@ func (b *backendBlock) rowGroupsForShard(ctx context.Context, pf *parquet.File,
matches := []parquet.RowGroup{}
for i := 0; i < len(index.RowGroups); i++ {
if i == 0 {
// The index contains the max trace ID for each row
// group. So to determine the min/max for the first
// entry we use the minimum ID from block meta.
if testRange(m.MinID, index.RowGroups[i]) {
if testRange([]byte{}, index.RowGroups[i]) {
matches = append(matches, rgs[i])
}
} else {
Expand Down
30 changes: 0 additions & 30 deletions tempodb/tempodb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse(BlockIDMax),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
start: 0,
end: 0,
Expand All @@ -293,8 +291,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse(BlockIDMax),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
start: 0,
end: 0,
Expand All @@ -307,8 +303,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse(BlockIDMax),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
start: 0,
end: 0,
Expand All @@ -321,8 +315,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse(BlockIDMax),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
start: 0,
end: 0,
Expand All @@ -335,8 +327,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
StartTime: time.Unix(10000, 0),
EndTime: time.Unix(20000, 0),
},
Expand All @@ -351,8 +341,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
StartTime: time.Unix(1650285326, 0),
EndTime: time.Unix(1650288990, 0),
},
Expand All @@ -367,8 +355,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x05},
MaxID: []byte{0x05},
},
start: 0,
end: 0,
Expand All @@ -382,8 +368,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse("51000000-0000-0000-0000-000000000000"),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("52000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
},
// todo: restore when this is fixed: https://github.com/grafana/tempo/issues/1903
Expand All @@ -394,8 +378,6 @@ func TestIncludeBlock(t *testing.T) {
// blockEnd: uuid.MustParse(BlockIDMax),
// meta: &backend.BlockMeta{
// BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
// MinID: []byte{0x01},
// MaxID: []byte{0x10},
// },
// },
// {
Expand All @@ -405,8 +387,6 @@ func TestIncludeBlock(t *testing.T) {
// blockEnd: uuid.MustParse(BlockIDMax),
// meta: &backend.BlockMeta{
// BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
// MinID: []byte{0x01},
// MaxID: []byte{0x10},
// },
// },
{
Expand All @@ -416,8 +396,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse("51000000-0000-0000-0000-000000000000"),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("4FFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
},
{
Expand All @@ -427,8 +405,6 @@ func TestIncludeBlock(t *testing.T) {
blockEnd: uuid.MustParse("51000000-0000-0000-0000-000000000000"),
meta: &backend.BlockMeta{
BlockID: uuid.MustParse("51000000-0000-0000-0000-000000000001"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
},
}
Expand Down Expand Up @@ -467,8 +443,6 @@ func TestIncludeCompactedBlock(t *testing.T) {
meta: &backend.CompactedBlockMeta{
BlockMeta: backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
CompactedTime: time.Now().Add(-(1 * blocklistPoll)),
},
Expand All @@ -484,8 +458,6 @@ func TestIncludeCompactedBlock(t *testing.T) {
meta: &backend.CompactedBlockMeta{
BlockMeta: backend.BlockMeta{
BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
CompactedTime: time.Now().Add(-(3 * blocklistPoll)),
},
Expand All @@ -501,8 +473,6 @@ func TestIncludeCompactedBlock(t *testing.T) {
meta: &backend.CompactedBlockMeta{
BlockMeta: backend.BlockMeta{
BlockID: uuid.MustParse("51000000-0000-0000-0000-000000000000"),
MinID: []byte{0x00},
MaxID: []byte{0x10},
},
CompactedTime: time.Now().Add(-(1 * blocklistPoll)),
},
Expand Down