tempo-cli: rewrite migrate overrides-config and add per-tenant migration command #6793
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,107 +1,159 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "flag" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "reflect" | ||
|
|
||
| "go.yaml.in/yaml/v2" | ||
|
|
||
| "github.com/grafana/tempo/cmd/tempo/app" | ||
| ) | ||
|
|
||
| type migrateOverridesConfigCmd struct { | ||
| ConfigFile string `arg:"" help:"Path to the full tempo config file"` | ||
|
|
||
| ConfigDest string `type:"path" short:"d" help:"Path to write the migrated overrides section. If not specified, output to stdout"` | ||
| } | ||
|
|
||
| func (cmd *migrateOverridesConfigCmd) Run(*globalOptions) error { | ||
| cfg := app.Config{} | ||
| cfg.RegisterFlagsAndApplyDefaults("", &flag.FlagSet{}) | ||
|
|
||
| // Build the default overrides config for comparison from a separate instance | ||
| // to avoid shared pointers between defaultOverrides and cfg.Overrides. | ||
| defaultCfg := app.Config{} | ||
| defaultCfg.RegisterFlagsAndApplyDefaults("", &flag.FlagSet{}) | ||
| defaultOverrides := defaultCfg.Overrides | ||
|
|
||
| buff, err := os.ReadFile(cmd.ConfigFile) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to read configFile %s: %w", cmd.ConfigFile, err) | ||
| } | ||
|
|
||
| // Legacy overrides are automatically converted to a new format during unmarshaling | ||
| // via Config.UnmarshalYAML. | ||
| if err := yaml.UnmarshalStrict(buff, &cfg); err != nil { | ||
|
electron0zero marked this conversation as resolved.
|
||
| return fmt.Errorf("failed to parse configFile %s: %w", cmd.ConfigFile, err) | ||
| } | ||
|
|
||
| // Marshal both configs to maps so we can diff them. | ||
| loadedMap, err := toMap(cfg.Overrides) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to convert loaded overrides to map: %w", err) | ||
| } | ||
|
|
||
| defaultMap, err := toMap(defaultOverrides) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to convert default overrides to map: %w", err) | ||
| } | ||
|
|
||
| // Remove keys that match the defaults so we only output user-set values. | ||
| removeDefaults(loadedMap, defaultMap) | ||
|
|
||
| result := map[string]interface{}{ | ||
| "overrides": loadedMap, | ||
|
electron0zero marked this conversation as resolved.
|
||
| } | ||
|
|
||
| outputBytes, err := yaml.Marshal(result) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to marshal overrides: %w", err) | ||
| } | ||
|
|
||
| printWarnings(os.Stderr) | ||
|
|
||
| if cmd.ConfigDest != "" { | ||
| if err := os.WriteFile(cmd.ConfigDest, outputBytes, 0o600); err != nil { | ||
| return fmt.Errorf("failed to write output file: %w", err) | ||
| } | ||
| fmt.Fprintf(os.Stderr, "Migrated overrides written to %s\n", cmd.ConfigDest) | ||
| fmt.Fprintln(os.Stderr, "Replace the overrides section in your config file with the contents of this file.") | ||
| } else { | ||
| fmt.Fprintln(os.Stderr, "Migrated overrides config. Replace the overrides section in your config file with the output below:") | ||
| fmt.Fprintln(os.Stderr, "---") | ||
| fmt.Print(string(outputBytes)) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // toMap converts a struct to a normalized map[string]interface{} via yaml round-trip. | ||
| // All nested maps are normalized to map[string]interface{}. | ||
| func toMap(v interface{}) (map[string]interface{}, error) { | ||
| b, err := yaml.Marshal(v) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| var m map[string]interface{} | ||
| if err := yaml.Unmarshal(b, &m); err != nil { | ||
| return nil, err | ||
| } | ||
| return normalizeMap(m), nil | ||
| } | ||
|
|
||
| // normalizeMap recursively converts all map[interface{}]interface{} to map[string]interface{}. | ||
| func normalizeMap(m map[string]interface{}) map[string]interface{} { | ||
| for k, v := range m { | ||
| m[k] = normalizeValue(v) | ||
| } | ||
| return m | ||
| } | ||
|
|
||
| func normalizeValue(v interface{}) interface{} { | ||
| switch val := v.(type) { | ||
| case map[interface{}]interface{}: | ||
| out := make(map[string]interface{}, len(val)) | ||
| for k, v := range val { | ||
| out[fmt.Sprintf("%v", k)] = normalizeValue(v) | ||
| } | ||
| return out | ||
| case map[string]interface{}: | ||
| return normalizeMap(val) | ||
| case []interface{}: | ||
| for i, item := range val { | ||
| val[i] = normalizeValue(item) | ||
| } | ||
| return val | ||
| default: | ||
| return v | ||
| } | ||
| } | ||
|
|
||
| // removeDefaults recursively removes keys from loaded where the value matches the default. | ||
| func removeDefaults(loaded, defaults map[string]interface{}) { | ||
| for key, defaultVal := range defaults { | ||
| loadedVal, ok := loaded[key] | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| loadedChild, loadedIsMap := loadedVal.(map[string]interface{}) | ||
| defaultChild, defaultIsMap := defaultVal.(map[string]interface{}) | ||
|
|
||
| if loadedIsMap && defaultIsMap { | ||
| removeDefaults(loadedChild, defaultChild) | ||
| if len(loadedChild) == 0 { | ||
| delete(loaded, key) | ||
| } | ||
| continue | ||
| } | ||
|
|
||
| if reflect.DeepEqual(loadedVal, defaultVal) { | ||
| delete(loaded, key) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func printWarnings(w io.Writer) { | ||
| fmt.Fprintln(w, "WARNING: Please verify the migrated output carefully before using it.") | ||
| fmt.Fprintln(w, "- Fields set to Go zero values (false, 0, \"\") may be silently dropped") | ||
| fmt.Fprintln(w, " due to omitempty tags. Compare against your original config to ensure nothing is lost.") | ||
| fmt.Fprintln(w, "- Secret values (e.g. remote_write_headers) are masked as '<secret>' in the output.") | ||
| fmt.Fprintln(w, " You must manually restore the original values.") | ||
| fmt.Fprintln(w, "- Some struct fields without omitempty may appear with zero values (e.g. 'exclude: null')") | ||
| fmt.Fprintln(w, " that were not in your original config. These can be removed.") | ||
| fmt.Fprintln(w, "NOTE: This tool is provided for convenience only. Always double check the output against your original config.") | ||
| fmt.Fprintln(w, "For full details on overrides configuration, see: https://grafana.com/docs/tempo/latest/configuration/#overrides") | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "os" | ||
| "sort" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "go.yaml.in/yaml/v2" | ||
| ) | ||
|
|
||
| func TestMigrateOverridesConfig(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| inputFile string | ||
| expectedFile string | ||
| }{ | ||
| { | ||
| name: "legacy config is migrated to new scoped format", | ||
| inputFile: "test-data/migrate-overrides-config/legacy-input.yaml", | ||
| expectedFile: "test-data/migrate-overrides-config/legacy-expected.yaml", | ||
| }, | ||
| { | ||
| name: "new format config is passed through unchanged", | ||
| inputFile: "test-data/migrate-overrides-config/new-format-input.yaml", | ||
| expectedFile: "test-data/migrate-overrides-config/new-format-expected.yaml", | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| expectedBytes, err := os.ReadFile(tt.expectedFile) | ||
| require.NoError(t, err) | ||
|
|
||
| outputPath := t.TempDir() + "/output.yaml" | ||
| cmd := migrateOverridesConfigCmd{ | ||
| ConfigFile: tt.inputFile, | ||
| ConfigDest: outputPath, | ||
| } | ||
|
|
||
| err = cmd.Run(nil) | ||
| require.NoError(t, err) | ||
|
|
||
| actualBytes, err := os.ReadFile(outputPath) | ||
| require.NoError(t, err) | ||
|
|
||
| // Compare as parsed YAML maps to avoid flaky failures from | ||
| // non-deterministic map iteration order (e.g. ListToMap processors). | ||
| var expectedMap, actualMap interface{} | ||
| require.NoError(t, yaml.Unmarshal(expectedBytes, &expectedMap)) | ||
| require.NoError(t, yaml.Unmarshal(actualBytes, &actualMap)) | ||
|
|
||
| sortStringSlices(expectedMap) | ||
| sortStringSlices(actualMap) | ||
|
|
||
| require.Equal(t, expectedMap, actualMap) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // sortStringSlices recursively sorts []interface{} slices that contain only strings | ||
| // to handle non-deterministic ordering from map-backed types like ListToMap. | ||
| func sortStringSlices(v interface{}) { | ||
| switch val := v.(type) { | ||
| case map[interface{}]interface{}: | ||
| for _, v := range val { | ||
| sortStringSlices(v) | ||
| } | ||
| case []interface{}: | ||
| allStrings := true | ||
| for _, item := range val { | ||
| if _, ok := item.(string); !ok { | ||
| allStrings = false | ||
| } | ||
| sortStringSlices(item) | ||
| } | ||
| if allStrings && len(val) > 0 { | ||
| sort.Slice(val, func(i, j int) bool { | ||
| return val[i].(string) < val[j].(string) | ||
| }) | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "go.yaml.in/yaml/v2" | ||
|
|
||
| "github.com/grafana/tempo/modules/overrides" | ||
| ) | ||
|
|
||
| type migrateOverridesPerTenantCmd struct { | ||
| OverridesFile string `arg:"" help:"Path to the per-tenant overrides file"` | ||
|
|
||
| OutputDest string `type:"path" short:"d" help:"Path to write the migrated per-tenant overrides. If not specified, output to stdout"` | ||
| } | ||
|
|
||
| func (cmd *migrateOverridesPerTenantCmd) Run(*globalOptions) error { | ||
| buff, err := os.ReadFile(cmd.OverridesFile) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to read overrides file %s: %w", cmd.OverridesFile, err) | ||
| } | ||
|
|
||
| // Legacy per-tenant overrides are automatically converted to the new format | ||
| // during unmarshaling via perTenantOverrides.UnmarshalYAML. | ||
| tenantOverrides, err := overrides.UnmarshalPerTenantOverrides(buff) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to parse overrides file %s: %w", cmd.OverridesFile, err) | ||
| } | ||
|
|
||
| // Diff each tenant's overrides against a zero-value Overrides to strip defaults. | ||
| defaultMap, err := toMap(overrides.Overrides{}) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to convert default overrides to map: %w", err) | ||
| } | ||
|
|
||
| result := make(map[string]interface{}, len(tenantOverrides)) | ||
| for tenant, o := range tenantOverrides { | ||
| if o == nil { | ||
| continue | ||
| } | ||
| tenantMap, err := toMap(o) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to convert overrides for tenant %s to map: %w", tenant, err) | ||
| } | ||
| removeDefaults(tenantMap, defaultMap) | ||
| if len(tenantMap) > 0 { | ||
| result[tenant] = tenantMap | ||
| } | ||
| } | ||
|
|
||
| output := map[string]interface{}{ | ||
| "overrides": result, | ||
| } | ||
|
|
||
| outputBytes, err := yaml.Marshal(output) | ||
|
electron0zero marked this conversation as resolved.
|
||
| if err != nil { | ||
| return fmt.Errorf("failed to marshal per-tenant overrides: %w", err) | ||
| } | ||
|
|
||
| printWarnings(os.Stderr) | ||
|
|
||
| if cmd.OutputDest != "" { | ||
| if err := os.WriteFile(cmd.OutputDest, outputBytes, 0o600); err != nil { | ||
| return fmt.Errorf("failed to write output file: %w", err) | ||
| } | ||
| fmt.Fprintf(os.Stderr, "Migrated per-tenant overrides written to %s\n", cmd.OutputDest) | ||
| } else { | ||
| fmt.Fprintln(os.Stderr, "Migrated per-tenant overrides. Replace your per-tenant overrides file with the output below:") | ||
| fmt.Fprintln(os.Stderr, "---") | ||
| fmt.Print(string(outputBytes)) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.