From 3c7f827fb946b005beb20e716532145ea129bab5 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 19 Sep 2024 16:27:52 -0400 Subject: [PATCH 1/3] shutdown generators only after they have been removed from the ring Signed-off-by: Joe Elliott --- modules/generator/generator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/generator/generator.go b/modules/generator/generator.go index 96618790407..102af3b8c22 100644 --- a/modules/generator/generator.go +++ b/modules/generator/generator.go @@ -162,9 +162,6 @@ func (g *Generator) running(ctx context.Context) error { } func (g *Generator) stopping(_ error) error { - // Mark as read-only - g.stopIncomingRequests() - if g.subservices != nil { err := services.StopManagerAndAwaitStopped(context.Background(), g.subservices) if err != nil { @@ -172,6 +169,9 @@ func (g *Generator) stopping(_ error) error { } } + // Mark as read-only after we have removed ourselves from the ring + g.stopIncomingRequests() + var wg sync.WaitGroup wg.Add(len(g.instances)) From 97007a5253a44de45a2536997e9b4f1cd2151917 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 19 Sep 2024 17:20:50 -0400 Subject: [PATCH 2/3] wait for a little bit Signed-off-by: Joe Elliott --- modules/generator/generator.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/generator/generator.go b/modules/generator/generator.go index 102af3b8c22..1b76317aacc 100644 --- a/modules/generator/generator.go +++ b/modules/generator/generator.go @@ -7,6 +7,7 @@ import ( "os" "path" "sync" + "time" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -169,6 +170,8 @@ func (g *Generator) stopping(_ error) error { } } + time.Sleep(5 * time.Second) // let the ring propagate the shutdown + // Mark as read-only after we have removed ourselves from the ring g.stopIncomingRequests() From 619997bf7b83e34fc9d0a30ac43829cf716ec227 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 20 Sep 2024 08:48:12 -0400 Subject: [PATCH 3/3] changelog Signed-off-by: Joe Elliott --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c1542f0bb..64bb2af52d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * [ENHANCEMENT] Add disk caching in ingester SearchTagValuesV2 for completed blocks [#4069](https://github.com/grafana/tempo/pull/4069) (@electron0zero) * [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4063](https://github.com/grafana/tempo/pull/4063) [#4078](https://github.com/grafana/tempo/pull/4078) (@galalen) +* [BUGFIX] Metrics generators: Correctly drop from the ring before stopping ingestion to reduce drops during a rollout. [#4101](https://github.com/grafana/tempo/pull/4101) (@joe-elliott) * [CHANGE] TraceByID: don't allow concurrent_shards greater than query_shards. [#4074](https://github.com/grafana/tempo/pull/4074) (@electron0zero) * **BREAKING CHANGE** tempo-query is no longer a jaeger instance with grpcPlugin. Its now a standalone server. Serving a grpc api for jaeger on `0.0.0.0:7777` by default. [#3840](https://github.com/grafana/tempo/issues/3840) (@frzifus) * [CHANGE] **BREAKING CHANGE** The dynamic injection of X-Scope-OrgID header for metrics generator remote-writes is changed. If the header is aleady set in per-tenant overrides or global tempo configuration, then it is honored and not overwritten. [#4021](https://github.com/grafana/tempo/pull/4021) (@mdisibio)