diff --git a/CHANGELOG.md b/CHANGELOG.md index b748b3d5dba..1664234f29f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [CHANGE] Enable Compactor and Alertmanager in target all. #6204 * [FEATURE] Ruler: Experimental: Add `ruler.frontend-address` to allow query to query frontends instead of ingesters. #6151 * [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129 * [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188 diff --git a/integration/api_endpoints_test.go b/integration/api_endpoints_test.go index 9cd9a79aa53..6da0971b49e 100644 --- a/integration/api_endpoints_test.go +++ b/integration/api_endpoints_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -22,10 +23,19 @@ func TestIndexAPIEndpoint(t *testing.T) { require.NoError(t, err) defer s.Close() + configOverrides := map[string]string{ + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // Start Cortex in single binary mode, reading the config from file. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) - cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) + cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) // GET / should succeed @@ -44,10 +54,19 @@ func TestConfigAPIEndpoint(t *testing.T) { require.NoError(t, err) defer s.Close() + configOverrides := map[string]string{ + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // Start Cortex in single binary mode, reading the config from file. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) - cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) + cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) // Get config from /config API endpoint. @@ -62,6 +81,7 @@ func TestConfigAPIEndpoint(t *testing.T) { // Start again Cortex in single binary with the exported config // and ensure it starts (pass the readiness probe). require.NoError(t, writeFileToSharedDir(s, cortexConfigFile, body)) - cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, nil, "", 9009, 9095) + configOverrides["-alertmanager.cluster.peers"] = cortex1.HTTPEndpoint() + cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex2)) } diff --git a/integration/getting_started_single_process_config_local_test.go b/integration/getting_started_single_process_config_local_test.go index 34eec0d661e..0a7434c5a3f 100644 --- a/integration/getting_started_single_process_config_local_test.go +++ b/integration/getting_started_single_process_config_local_test.go @@ -4,6 +4,7 @@ package integration import ( + "path/filepath" "testing" "time" @@ -24,7 +25,14 @@ func TestGettingStartedSingleProcessConfigWithFilesystem(t *testing.T) { // Start Cortex components. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) - flags := map[string]string{} + flags := map[string]string{ + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) diff --git a/integration/getting_started_single_process_config_test.go b/integration/getting_started_single_process_config_test.go index 1cdf689790c..624928abf0a 100644 --- a/integration/getting_started_single_process_config_test.go +++ b/integration/getting_started_single_process_config_test.go @@ -5,6 +5,7 @@ package integration import ( "fmt" + "path/filepath" "testing" "time" @@ -38,7 +39,13 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) { "-blocks-storage.s3.bucket-name": bucketName, "-blocks-storage.s3.endpoint": fmt.Sprintf("%s-minio-9000:9000", networkName), "-blocks-storage.s3.insecure": "true", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) diff --git a/integration/getting_started_with_gossiped_ring_test.go b/integration/getting_started_with_gossiped_ring_test.go index 3b9239310b0..4bcd8a090b4 100644 --- a/integration/getting_started_with_gossiped_ring_test.go +++ b/integration/getting_started_with_gossiped_ring_test.go @@ -5,6 +5,7 @@ package integration import ( "fmt" + "path/filepath" "testing" "time" @@ -45,8 +46,15 @@ func TestGettingStartedWithGossipedRing(t *testing.T) { "-blocks-storage.s3.insecure": "true", "-store-gateway.sharding-ring.wait-stability-min-duration": "0", // start quickly "-store-gateway.sharding-ring.wait-stability-max-duration": "0", // start quickly + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // This cortex will fail to join the cluster configured in yaml file. That's fine. cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", "config1.yaml", e2e.MergeFlags(flags, map[string]string{ "-ingester.lifecycler.addr": networkName + "-cortex-1", // Ingester's hostname in docker setup diff --git a/integration/integration_memberlist_single_binary_test.go b/integration/integration_memberlist_single_binary_test.go index ce62772dc70..68bd343ee06 100644 --- a/integration/integration_memberlist_single_binary_test.go +++ b/integration/integration_memberlist_single_binary_test.go @@ -43,6 +43,9 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string) require.NoError(t, err) defer s.Close() + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // Start dependencies minio := e2edb.NewMinio(9000, bucketName) // Look ma, no Consul! @@ -116,16 +119,21 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string) } func newSingleBinary(name string, servername string, join string, testFlags map[string]string) *e2ecortex.CortexService { - flags := map[string]string{ - "-ingester.final-sleep": "0s", - "-ingester.join-after": "0s", // join quickly - "-ingester.min-ready-duration": "0s", - "-ingester.num-tokens": "512", - "-ingester.observe-period": "5s", // to avoid conflicts in tokens - "-ring.store": "memberlist", - "-memberlist.bind-port": "8000", - "-memberlist.left-ingesters-timeout": "600s", // effectively disable - } + flags := mergeFlags( + AlertmanagerLocalFlags(), + map[string]string{ + "-ingester.final-sleep": "0s", + "-ingester.join-after": "0s", // join quickly + "-ingester.min-ready-duration": "0s", + "-ingester.num-tokens": "512", + "-ingester.observe-period": "5s", // to avoid conflicts in tokens + "-ring.store": "memberlist", + "-memberlist.bind-port": "8000", + "-memberlist.left-ingesters-timeout": "600s", // effectively disable + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + }, + ) if join != "" { flags["-memberlist.join"] = join @@ -158,6 +166,9 @@ func TestSingleBinaryWithMemberlistScaling(t *testing.T) { require.NoError(t, err) defer s.Close() + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + minio := e2edb.NewMinio(9000, bucketName) require.NoError(t, s.StartAndWaitReady(minio)) diff --git a/integration/otlp_test.go b/integration/otlp_test.go index 9a603e8cb03..da689e7b674 100644 --- a/integration/otlp_test.go +++ b/integration/otlp_test.go @@ -6,6 +6,7 @@ package integration import ( "fmt" "math/rand" + "path/filepath" "testing" "time" @@ -41,7 +42,13 @@ func TestOTLP(t *testing.T) { "-blocks-storage.s3.endpoint": fmt.Sprintf("%s-minio-9000:9000", networkName), "-blocks-storage.s3.insecure": "true", "-blocks-storage.tsdb.enable-native-histograms": "true", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) diff --git a/integration/querier_test.go b/integration/querier_test.go index a5e92e299e7..7c7bd167fc4 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -365,26 +365,33 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { // Configure the blocks storage to frequently compact TSDB head // and ship blocks to the storage. - flags := mergeFlags(BlocksStorageFlags(), map[string]string{ - "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), - "-blocks-storage.tsdb.ship-interval": "1s", - "-blocks-storage.bucket-store.sync-interval": "1s", - "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), - "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, - "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), - "-querier.query-store-for-labels-enabled": "true", - "-querier.thanos-engine": strconv.FormatBool(thanosEngine), - // Ingester. - "-ring.store": "consul", - "-consul.hostname": consul.NetworkHTTPEndpoint(), - // Distributor. - "-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10), - // Store-gateway. - "-store-gateway.sharding-enabled": strconv.FormatBool(testCfg.blocksShardingEnabled), - "-store-gateway.sharding-ring.store": "consul", - "-store-gateway.sharding-ring.consul.hostname": consul.NetworkHTTPEndpoint(), - "-store-gateway.sharding-ring.replication-factor": "1", - }) + flags := mergeFlags( + BlocksStorageFlags(), + AlertmanagerLocalFlags(), + map[string]string{ + "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), + "-blocks-storage.tsdb.ship-interval": "1s", + "-blocks-storage.bucket-store.sync-interval": "1s", + "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), + "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, + "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), + "-querier.query-store-for-labels-enabled": "true", + "-querier.thanos-engine": strconv.FormatBool(thanosEngine), + // Ingester. + "-ring.store": "consul", + "-consul.hostname": consul.NetworkHTTPEndpoint(), + // Distributor. + "-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10), + // Store-gateway. + "-store-gateway.sharding-enabled": strconv.FormatBool(testCfg.blocksShardingEnabled), + "-store-gateway.sharding-ring.store": "consul", + "-store-gateway.sharding-ring.consul.hostname": consul.NetworkHTTPEndpoint(), + "-store-gateway.sharding-ring.replication-factor": "1", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + }, + ) + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml))) // Add the cache address to the flags. if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go index 058a94dbd78..292f94421d8 100644 --- a/integration/query_fuzz_test.go +++ b/integration/query_fuzz_test.go @@ -45,25 +45,32 @@ func TestVerticalShardingFuzz(t *testing.T) { consul2 := e2edb.NewConsulWithName("consul2") require.NoError(t, s.StartAndWaitReady(consul1, consul2)) - flags := map[string]string{ - "-store.engine": blocksStorageEngine, - "-blocks-storage.backend": "filesystem", - "-blocks-storage.tsdb.head-compaction-interval": "4m", - "-blocks-storage.tsdb.block-ranges-period": "2h", - "-blocks-storage.tsdb.ship-interval": "1h", - "-blocks-storage.bucket-store.sync-interval": "15m", - "-blocks-storage.tsdb.retention-period": "2h", - "-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory, - "-blocks-storage.bucket-store.bucket-index.enabled": "true", - "-querier.query-store-for-labels-enabled": "true", - // Ingester. - "-ring.store": "consul", - "-consul.hostname": consul1.NetworkHTTPEndpoint(), - // Distributor. - "-distributor.replication-factor": "1", - // Store-gateway. - "-store-gateway.sharding-enabled": "false", - } + flags := mergeFlags( + AlertmanagerLocalFlags(), + map[string]string{ + "-store.engine": blocksStorageEngine, + "-blocks-storage.backend": "filesystem", + "-blocks-storage.tsdb.head-compaction-interval": "4m", + "-blocks-storage.tsdb.block-ranges-period": "2h", + "-blocks-storage.tsdb.ship-interval": "1h", + "-blocks-storage.bucket-store.sync-interval": "15m", + "-blocks-storage.tsdb.retention-period": "2h", + "-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory, + "-blocks-storage.bucket-store.bucket-index.enabled": "true", + "-querier.query-store-for-labels-enabled": "true", + // Ingester. + "-ring.store": "consul", + "-consul.hostname": consul1.NetworkHTTPEndpoint(), + // Distributor. + "-distributor.replication-factor": "1", + // Store-gateway. + "-store-gateway.sharding-enabled": "false", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + }, + ) + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) path1 := path.Join(s.SharedDir(), "cortex-1") path2 := path.Join(s.SharedDir(), "cortex-2") diff --git a/integration/ruler_test.go b/integration/ruler_test.go index e4c5ad54394..23815ea4ee6 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -154,7 +154,13 @@ func TestRulerAPISingleBinary(t *testing.T) { "-ruler-storage.local.directory": filepath.Join(e2e.ContainerSharedDir, "ruler_configs"), "-ruler.poll-interval": "2s", "-ruler.rule-path": filepath.Join(e2e.ContainerSharedDir, "rule_tmp/"), + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) // Start Cortex components. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) diff --git a/integration/runtime_config_test.go b/integration/runtime_config_test.go index 7f9f0ceb302..f8f722084bd 100644 --- a/integration/runtime_config_test.go +++ b/integration/runtime_config_test.go @@ -40,6 +40,10 @@ func TestLoadRuntimeConfigFromStorageBackend(t *testing.T) { name: "no storage backend provided", flags: map[string]string{ "-runtime-config.file": filePath, + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), }, }, { @@ -47,9 +51,15 @@ func TestLoadRuntimeConfigFromStorageBackend(t *testing.T) { flags: map[string]string{ "-runtime-config.file": filePath, "-runtime-config.backend": "filesystem", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), }, }, } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -79,7 +89,14 @@ func TestLoadRuntimeConfigFromCloudStorage(t *testing.T) { "-runtime-config.s3.insecure": "true", "-runtime-config.file": configFileName, "-runtime-config.reload-period": "2s", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // create s3 storage backend minio := e2edb.NewMinio(9000, bucketName) require.NoError(t, s.StartAndWaitReady(minio)) diff --git a/pkg/cortex/cortex_test.go b/pkg/cortex/cortex_test.go index 891f8608988..bac7c0021c6 100644 --- a/pkg/cortex/cortex_test.go +++ b/pkg/cortex/cortex_test.go @@ -12,10 +12,6 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/cortex/storage" - "github.com/cortexproject/cortex/pkg/ruler/rulestore" - "github.com/cortexproject/cortex/pkg/ruler/rulestore/local" - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,14 +20,20 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "github.com/cortexproject/cortex/pkg/alertmanager" + "github.com/cortexproject/cortex/pkg/alertmanager/alertstore" + "github.com/cortexproject/cortex/pkg/cortex/storage" "github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv" + "github.com/cortexproject/cortex/pkg/ruler/rulestore" + "github.com/cortexproject/cortex/pkg/ruler/rulestore/local" "github.com/cortexproject/cortex/pkg/scheduler/schedulerpb" "github.com/cortexproject/cortex/pkg/storage/bucket" "github.com/cortexproject/cortex/pkg/storage/bucket/s3" "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -92,10 +94,23 @@ func TestCortex(t *testing.T) { Directory: os.TempDir(), }, }, - - Target: []string{All, Compactor}, + AlertmanagerStorage: alertstore.Config{ + Config: bucket.Config{ + Backend: "local", + }, + }, + Target: []string{All}, } + externalURL := flagext.URLValue{} + err := externalURL.Set("http://localhost/alertmanager") + require.NoError(t, err) + + multiAlertmanagerCfg := &alertmanager.MultitenantAlertmanagerConfig{} + flagext.DefaultValues(multiAlertmanagerCfg) + multiAlertmanagerCfg.ExternalURL = externalURL + cfg.Alertmanager = *multiAlertmanagerCfg + c, err := New(cfg) require.NoError(t, err) @@ -114,8 +129,9 @@ func TestCortex(t *testing.T) { require.NotNil(t, serviceMap[Ring]) require.NotNil(t, serviceMap[DistributorService]) - // check that compactor is configured which is not part of Target=All + // check compactor and alertmanager are configured. require.NotNil(t, serviceMap[Compactor]) + require.NotNil(t, serviceMap[AlertManager]) } func TestConfigValidation(t *testing.T) { diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 9e72894f9e0..ed3c61d9488 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -788,7 +788,7 @@ func (t *Cortex) setupModuleManager() error { TenantDeletion: {API, Overrides}, Purger: {TenantDeletion}, TenantFederation: {Queryable}, - All: {QueryFrontend, Querier, Ingester, Distributor, Purger, StoreGateway, Ruler}, + All: {QueryFrontend, Querier, Ingester, Distributor, Purger, StoreGateway, Ruler, Compactor, AlertManager}, } if t.Cfg.ExternalPusher != nil && t.Cfg.ExternalQueryable != nil { deps[Ruler] = []string{Overrides, RulerStorage}