Update serverless e2e for Lambda

.drone/readme.md

@@ -9,7 +9,7 @@ branches (`r**`).
 # Pipelines
 
 The pipelines are `docker-amd64`, `docker-arm64`, and `manifest`. The two docker pipelines
-run concurrently and create images tagged like `tempo:<tag>-<arch>` or `tempo:<branch>-<sha>-<arch>.
+run concurrently and create images tagged like `tempo:<tag>-<arch>` or `tempo:<branch>-<sha>-<arch>`.
 E.g. `tempo:1.1.0-arm64` or `tempo:main-e2a314-amd64`. The manifest step then creates a manifest 
 that combines the mentioned images into one multiarch image named as you would expect: 
 `tempo:1.1.0` or `tempo:main-e2a314`.  

CHANGELOG.md

@@ -5,6 +5,7 @@
 * [CHANGE] Updated storage.trace.pool.queue_depth default from 200->10000. [#1345](https://github.com/grafana/tempo/pull/1345) (@joe-elliott)
 * [CHANGE] Update alpine images to 3.15 [#1330](https://github.com/grafana/tempo/pull/1330) (@zalegrala)
 * [CHANGE] Updated flags `-storage.trace.azure.storage-account-name` and `-storage.trace.s3.access_key` to no longer to be considered as secrets [#1356](https://github.com/grafana/tempo/pull/1356) (@simonswine)
+* [CHANGE] Include lambda in serverless e2e tests [#1357](https://github.com/grafana/tempo/pull/1357) (@zalegrala)
 * [FEATURE]: v2 object encoding added. This encoding adds a start/end timestamp to every record to reduce proto marshalling and increase search speed.  
   **BREAKING CHANGE** After this rollout the distributors will use a new API on the ingesters. As such you must rollout all ingesters before rolling the 
   distributors. Also, during this period, the ingesters will use considerably more resources and as such should be scaled up (or incoming traffic should be

cmd/tempo-serverless/Makefile

@@ -10,7 +10,10 @@ IN_LAMBDA=cd lambda &&
 # build docker images for local testing and code zip files for google cloud functions
 #
 .PHONY: build-docker
-build-docker: 
+build-docker: build-docker-gcf build-docker-lambda-test
+
+.PHONY: build-docker-gcf
+build-docker-gcf: 
 	$(IN_CLOUD_FUNCTIONS) go mod vendor
 	$(IN_CLOUD_FUNCTIONS) $(PACK) build tempo-serverless \
 	  								--builder gcr.io/buildpacks/builder:v1 \
@@ -28,14 +31,10 @@ build-gcf-zip:
 #
 # build docker images for local testing and code zip files for aws lambda
 #
-.PHONY: build-lambda-docker
-build-lambda-docker:
-#   todo: build exe in docker?
-#	$(IN_LAMBDA) go mod vendor
-	$(IN_LAMBDA) go build -o ./lambda
-	$(IN_LAMBDA) docker build -t tempo-serverless .
-	$(IN_LAMBDA) rm ./lambda
-#	$(IN_LAMBDA) rm -rf vendor
+.PHONY: build-docker-lambda-test
+build-docker-lambda-test:
+	$(IN_LAMBDA) CGO_ENABLED=0 go build -o ./lambda
+	$(IN_LAMBDA) docker build -f ./Dockerfile -t tempo-serverless-lambda .
 
 # lambda zips expect a compiled executable in root. the filename "main" is important
 # as that should the handler config option in aws
@@ -54,4 +53,3 @@ test:
 update-mod:
 	$(IN_LAMBDA) go mod tidy -e
 	$(IN_CLOUD_FUNCTIONS) go mod tidy -e
-

cmd/tempo-serverless/lambda/Dockerfile

@@ -21,7 +21,7 @@ COPY lambda /
 # copy in https://github.com/treasure-data/lambda-local-proxy to include in the docker image.
 #   this project acts as an "alb" and translates normal http requests into Lambda invokes
 RUN    curl -L https://github.com/treasure-data/lambda-local-proxy/releases/download/v0.0.5/lambda-local-proxy_0.0.5_Linux_x86_64.tar.gz | tar xvz -C / \
-    && chmod +x /lambda-local-proxy
+  && chmod +x /lambda-local-proxy
 
 #
 # https://docs.aws.amazon.com/lambda/latest/dg/images-create.html
@@ -34,12 +34,11 @@ COPY --from=build /lambda-local-proxy /lambda-local-proxy
 RUN chmod +x /lambda-local-proxy
 
 RUN echo -e '\
-#!/bin/bash \n\
-AWS_REGION=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /lambda-local-proxy -e http://localhost:8080 -p 9000 -f function & \n\
-exec /lambda-entrypoint.sh lambda \n\
-' > /run.sh
+  #!/bin/bash \n\
+  AWS_REGION=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /lambda-local-proxy -e http://localhost:8080 -p 9000 -f function & \n\
+  exec /lambda-entrypoint.sh lambda \n\
+  ' > /run.sh
 RUN chmod +x /run.sh
 
 # run lambda-local-proxy and the actual lambda runtime itself.
 ENTRYPOINT /run.sh
-

integration/e2e/serverless/config-serverless.yaml → integration/e2e/serverless/config-serverless-gcf.yaml

@@ -32,7 +32,7 @@ storage:
     backend: s3
     s3:
       bucket: tempo
-      endpoint: tempo_e2e-minio-9000:9000  # TODO: this is brittle, fix this eventually
+      endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
       access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
       secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
       insecure: true
@@ -44,15 +44,15 @@ memberlist:
   abort_if_cluster_join_fails: false
   bind_port: 7946
   join_members:
-  - tempo_e2e-distributor:7946
-  - tempo_e2e-ingester-1:7946
-  - tempo_e2e-ingester-2:7946
-  - tempo_e2e-ingester-3:7946
-  - tempo_e2e-querier:7946
+    - tempo_e2e-distributor:7946
+    - tempo_e2e-ingester-1:7946
+    - tempo_e2e-ingester-2:7946
+    - tempo_e2e-ingester-3:7946
+    - tempo_e2e-querier:7946
 
 querier:
   search:
-    external_endpoints: 
-    - http://tempo_e2e-serverless:8080/
+    external_endpoints:
+      - http://tempo_e2e-serverless:8080/
   frontend_worker:
     frontend_address: tempo_e2e-query-frontend:9095

integration/e2e/serverless/config-serverless-lambda.yaml

@@ -0,0 +1,58 @@
+search_enabled: true
+
+server:
+  http_listen_port: 3200
+
+query_frontend:
+  search:
+    query_backend_after: 0 # setting these both to 0 will force all range searches to hit the backend
+    query_ingesters_until: 0
+
+distributor:
+  receivers:
+    jaeger:
+      protocols:
+        grpc:
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: memberlist
+      replication_factor: 3
+    heartbeat_period: 100ms
+  trace_idle_period: 1s
+  max_block_duration: 2s
+  complete_block_timeout: 5s
+  flush_check_period: 1s
+
+storage:
+  trace:
+    blocklist_poll: 2s
+    backend: s3
+    s3:
+      bucket: tempo
+      endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
+      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
+      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
+      insecure: true
+    pool:
+      max_workers: 10
+      queue_depth: 100
+
+memberlist:
+  abort_if_cluster_join_fails: false
+  bind_port: 7946
+  join_members:
+    - tempo_e2e-distributor:7946
+    - tempo_e2e-ingester-1:7946
+    - tempo_e2e-ingester-2:7946
+    - tempo_e2e-ingester-3:7946
+    - tempo_e2e-querier:7946
+
+querier:
+  search:
+    external_endpoints:
+      - http://tempo_e2e-serverless:9000/
+  frontend_worker:
+    frontend_address: tempo_e2e-query-frontend:9095

integration/e2e/serverless/serverless_test.go

@@ -15,88 +15,136 @@ import (
 )
 
 const (
-	configServerless = "config-serverless.yaml"
+	configServerlessGCF    = "config-serverless-gcf.yaml"
+	configServerlessLambda = "config-serverless-lambda.yaml"
 )
 
 func TestServerless(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	minio := e2e_db.NewMinio(9000, "tempo")
-	require.NotNil(t, minio)
-	require.NoError(t, s.StartAndWaitReady(minio))
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configServerless, "config.yaml"))
-	tempoIngester1 := util.NewTempoIngester(1)
-	tempoIngester2 := util.NewTempoIngester(2)
-	tempoIngester3 := util.NewTempoIngester(3)
-	tempoDistributor := util.NewTempoDistributor()
-	tempoQueryFrontend := util.NewTempoQueryFrontend()
-	tempoQuerier := util.NewTempoQuerier()
-	tempoServerless := newTempoServerless()
-	require.NoError(t, s.StartAndWaitReady(tempoIngester1, tempoIngester2, tempoIngester3, tempoDistributor, tempoQueryFrontend, tempoQuerier, tempoServerless))
-
-	// wait for 2 active ingesters
-	time.Sleep(1 * time.Second)
-	matchers := []*labels.Matcher{
+
+	testClouds := []struct {
+		name       string
+		serverless *e2e.HTTPService
+		config     string
+	}{
 		{
-			Type:  labels.MatchEqual,
-			Name:  "name",
-			Value: "ingester",
+			name:       "gcf",
+			serverless: newTempoServerlessGCF(),
+			config:     configServerlessGCF,
 		},
 		{
-			Type:  labels.MatchEqual,
-			Name:  "state",
-			Value: "ACTIVE",
+			name:       "lambda",
+			serverless: newTempoServerlessLambda(),
+			config:     configServerlessLambda,
 		},
 	}
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`cortex_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerGRPCClient(tempoDistributor.Endpoint(14250))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
-
-	// wait trace_idle_time and ensure trace is created in ingester
-	time.Sleep(1 * time.Second)
-	require.NoError(t, tempoIngester1.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_traces_created_total"))
-	require.NoError(t, tempoIngester2.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_traces_created_total"))
-	require.NoError(t, tempoIngester3.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_traces_created_total"))
 
-	apiClient := tempoUtil.NewClient("http://"+tempoQueryFrontend.Endpoint(3200), "")
+	for _, tc := range testClouds {
+		t.Run(tc.name, func(t *testing.T) {
+			s, err := e2e.NewScenario("tempo_e2e")
+			require.NoError(t, err)
+			defer s.Close()
+
+			minio := e2e_db.NewMinio(9000, "tempo")
+			require.NotNil(t, minio)
+			require.NoError(t, s.StartAndWaitReady(minio))
+
+			require.NoError(t, util.CopyFileToSharedDir(s, tc.config, "config.yaml"))
+			tempoIngester1 := util.NewTempoIngester(1)
+			tempoIngester2 := util.NewTempoIngester(2)
+			tempoIngester3 := util.NewTempoIngester(3)
+			tempoDistributor := util.NewTempoDistributor()
+			tempoQueryFrontend := util.NewTempoQueryFrontend()
+			tempoQuerier := util.NewTempoQuerier()
+			tempoServerless := tc.serverless
+			require.NoError(t, s.StartAndWaitReady(tempoIngester1, tempoIngester2, tempoIngester3, tempoDistributor, tempoQueryFrontend, tempoQuerier, tempoServerless))
+
+			// wait for 2 active ingesters
+			time.Sleep(1 * time.Second)
+			matchers := []*labels.Matcher{
+				{
+					Type:  labels.MatchEqual,
+					Name:  "name",
+					Value: "ingester",
+				},
+				{
+					Type:  labels.MatchEqual,
+					Name:  "state",
+					Value: "ACTIVE",
+				},
+			}
+			require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`cortex_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
+
+			// Get port for the Jaeger gRPC receiver endpoint
+			c, err := util.NewJaegerGRPCClient(tempoDistributor.Endpoint(14250))
+			require.NoError(t, err)
+			require.NotNil(t, c)
+
+			info := tempoUtil.NewTraceInfo(time.Now(), "")
+			require.NoError(t, info.EmitAllBatches(c))
+
+			// wait trace_idle_time and ensure trace is created in ingester
+			time.Sleep(1 * time.Second)
+			require.NoError(t, tempoIngester1.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_traces_created_total"))
+			require.NoError(t, tempoIngester2.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_traces_created_total"))
+			require.NoError(t, tempoIngester3.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_traces_created_total"))
+
+			apiClient := tempoUtil.NewClient("http://"+tempoQueryFrontend.Endpoint(3200), "")
+
+			// flush trace to backend
+			res, err := e2e.DoGet("http://" + tempoIngester1.Endpoint(3200) + "/flush")
+			require.NoError(t, err)
+			require.Equal(t, 204, res.StatusCode)
+
+			res, err = e2e.DoGet("http://" + tempoIngester2.Endpoint(3200) + "/flush")
+			require.NoError(t, err)
+			require.Equal(t, 204, res.StatusCode)
+
+			res, err = e2e.DoGet("http://" + tempoIngester3.Endpoint(3200) + "/flush")
+			require.NoError(t, err)
+			require.Equal(t, 204, res.StatusCode)
+
+			// zzz
+			time.Sleep(10 * time.Second)
+
+			// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
+			now := time.Now()
+			util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
+
+		})
+	}
 
-	// flush trace to backend
-	res, err := e2e.DoGet("http://" + tempoIngester1.Endpoint(3200) + "/flush")
-	require.NoError(t, err)
-	require.Equal(t, 204, res.StatusCode)
+}
 
-	res, err = e2e.DoGet("http://" + tempoIngester2.Endpoint(3200) + "/flush")
-	require.NoError(t, err)
-	require.Equal(t, 204, res.StatusCode)
+func newTempoServerlessGCF() *e2e.HTTPService {
+	s := e2e.NewHTTPService(
+		"serverless",
+		"tempo-serverless", // created by buildpacks in ./cmd/tempo-serverless
+		nil,
+		nil,
+		8080,
+	)
 
-	res, err = e2e.DoGet("http://" + tempoIngester3.Endpoint(3200) + "/flush")
-	require.NoError(t, err)
-	require.Equal(t, 204, res.StatusCode)
+	s.SetEnvVars(map[string]string{
+		"TEMPO_S3_BUCKET":     "tempo",
+		"TEMPO_S3_ENDPOINT":   "tempo_e2e-minio-9000:9000",
+		"TEMPO_S3_ACCESS_KEY": e2e_db.MinioAccessKey,
+		"TEMPO_S3_SECRET_KEY": e2e_db.MinioSecretKey,
+		"TEMPO_S3_INSECURE":   "true",
+		"TEMPO_BACKEND":       "s3",
+	})
 
-	// zzz
-	time.Sleep(10 * time.Second)
+	s.SetBackoff(util.TempoBackoff())
 
-	// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
-	now := time.Now()
-	util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
+	return s
 }
 
-func newTempoServerless() *e2e.HTTPService {
+func newTempoServerlessLambda() *e2e.HTTPService {
 	s := e2e.NewHTTPService(
 		"serverless",
-		"tempo-serverless", // created by buildpacks in ./cmd/tempo-serverless
+		"tempo-serverless-lambda", // created by build-docker-lambda-test make target
 		nil,
 		nil,
-		8080,
+		9000,
 	)
 
 	s.SetEnvVars(map[string]string{