Add /status/overrides/{tenant} endpoint

CHANGELOG.md

@@ -30,6 +30,7 @@
 * [ENHANCEMENT] Added a `frontend-search` cache role for job search caching. [#3225](https://github.com/grafana/tempo/pull/3225) (@joe-elliott)
 * [ENHANCEMENT] Added a `parquet-page` cache role for page level caching. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)
 * [ENHANCEMENT] Update opentelemetry-collector-contrib dependency to the latest version, v0.89.0 [#3148](https://github.com/grafana/tempo/pull/3148) (@gebn)
+* [ENHANCEMENT] Add /status/overrides/{tenant} endpoint [#3244](https://github.com/grafana/tempo/pull/3244) (@kvrhdn)
 * [BUGFIX] Prevent building parquet iterators that would loop forever. [#3159](https://github.com/grafana/tempo/pull/3159) (@mapno)
 * [BUGFIX] Sanitize name in mapped dimensions in span-metrics processor [#3171](https://github.com/grafana/tempo/pull/3171) (@mapno)
 * [BUGFIX] Fixed an issue where cached footers were requested then ignored. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)

cmd/tempo/app/app.go

@@ -197,6 +197,7 @@ func (t *App) Run() error {
 	t.Server.HTTP.Path("/ready").Handler(t.readyHandler(sm))
 	t.Server.HTTP.Path("/status").Handler(t.statusHandler()).Methods("GET")
 	t.Server.HTTP.Path("/status/{endpoint}").Handler(t.statusHandler()).Methods("GET")
+	t.Server.HTTP.Path("/status/overrides/{tenant}").Handler(t.tenantOverridesHandler()).Methods("GET")
 	grpc_health_v1.RegisterHealthServer(t.Server.GRPC, grpcutil.NewHealthCheck(sm))
 
 	// Let's listen for events from this manager, and log them.
@@ -369,6 +370,7 @@ func (t *App) statusHandler() http.HandlerFunc {
 			msg.WriteString("GET /status/" + endpoint + "\n")
 
 			switch endpoint {
+			// TODO should we change this endpoint to /status/overrides?
 			case "runtime_config":
 				err := t.writeRuntimeConfig(&msg, r)
 				if err != nil {
@@ -518,6 +520,25 @@ func (t *App) writeStatusEndpoints(w io.Writer) error {
 	return nil
 }
 
+func (t *App) tenantOverridesHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+
+		tenant, ok := vars["tenant"]
+		if !ok {
+			http.Error(w, "must specify tenant ID", http.StatusBadRequest)
+			return
+		}
+
+		err := t.Overrides.WriteTenantOverrides(w, r, tenant)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			level.Error(log.Logger).Log("msg", "error writing response", "endpoint", r.URL.String(), "err", err)
+			return
+		}
+	}
+}
+
 func (t *App) buildinfoHandler() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")

docs/sources/tempo/api_docs/_index.md

@@ -560,6 +560,12 @@ Displays the override configuration.
 Query parameter:
 - `mode = (diff)`: Show the difference between defaults and overrides.
 
+```
+GET /status/overrides/{tenant}
+```
+
+Displays the override configuration for the specified tenant.
+
 ```
 GET /status/usage-stats
 ```
@@ -638,4 +644,4 @@ message SearchMetrics {
   uint32 totalJobs = 5;
   uint64 totalBlockBytes = 6;
 }
-```
+```

modules/overrides/interface.go

@@ -21,6 +21,10 @@ type Service interface {
 type Interface interface {
 	prometheus.Collector
 
+	// GetRuntimeOverridesFor returns the runtime overrides set for the given user excluding
+	// overrides from the user-configurable overrides, if enabled.
+	GetRuntimeOverridesFor(userID string) *Overrides
+
 	// Config
 	IngestionRateStrategy() string
 	MaxLocalTracesPerUser(userID string) int
@@ -62,6 +66,7 @@ type Interface interface {
 	MaxSearchDuration(userID string) time.Duration
 	DedicatedColumns(userID string) backend.DedicatedColumns
 
-	// API
+	// Management API
 	WriteStatusRuntimeConfig(w io.Writer, r *http.Request) error
+	WriteTenantOverrides(w io.Writer, r *http.Request, tenant string) error
 }

modules/overrides/runtime_config_overrides.go

@@ -241,6 +241,22 @@ func (o *runtimeConfigOverridesManager) WriteStatusRuntimeConfig(w io.Writer, r
 	return nil
 }
 
+func (o *runtimeConfigOverridesManager) GetRuntimeOverridesFor(userID string) *Overrides {
+	return o.getOverridesForUser(userID)
+}
+
+func (o *runtimeConfigOverridesManager) WriteTenantOverrides(w io.Writer, _ *http.Request, userID string) error {
+	overrides := o.getOverridesForUser(userID)
+
+	out, err := yaml.Marshal(overrides)
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(out)
+	return err
+}
+
 // IngestionRateStrategy returns whether the ingestion rate limit should be individually applied
 // to each distributor instance (local) or evenly shared across the cluster (global).
 func (o *runtimeConfigOverridesManager) IngestionRateStrategy() string {

modules/overrides/user_configurable_overrides.go

@@ -316,6 +316,26 @@ func (o *userConfigurableOverridesManager) WriteStatusRuntimeConfig(w io.Writer,
 	return nil
 }
 
+type statusTenantOverrides struct {
+	UserConfigurableLimits *userconfigurableoverrides.Limits `yaml:"user_configurable_limits"`
+	RuntimeOverrides       *Overrides                        `yaml:"runtime_overrides"`
+}
+
+func (o *userConfigurableOverridesManager) WriteTenantOverrides(w io.Writer, _ *http.Request, userID string) error {
+	overrides := statusTenantOverrides{
+		UserConfigurableLimits: o.getTenantLimits(userID),
+		RuntimeOverrides:       o.GetRuntimeOverridesFor(userID),
+	}
+
+	out, err := yaml.Marshal(overrides)
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(out)
+	return err
+}
+
 func (o *userConfigurableOverridesManager) Describe(ch chan<- *prometheus.Desc) {
 	// TODO for now just pass along to the underlying overrides, in the future we should export
 	//  the user-config overrides as well