feat(contributing): add support for debugging live pods (#585)

Author: michael-todorovic

Dockerfile

@@ -17,12 +17,13 @@ ENV VITE_API_BASE_URL=/api
 RUN yarn build
 
 # Build the manager binary
-FROM docker.io/library/golang:1.23.7@sha256:1acb493b9f9dfdfe705042ce09e8ded908ce4fb342405ecf3ca61ce7f3b168c7 AS builder
+FROM docker.io/library/golang:1.23.7-alpine@sha256:e438c135c348bd7677fde18d1576c2f57f265d5dfa1a6b26fca975d4aa40b3bb AS builder
 ARG TARGETOS
 ARG TARGETARCH
 ARG PACKAGE=github.com/padok-team/burrito
 ARG COMMIT_HASH
 ARG BUILD_TIMESTAMP
+ARG BUILD_MODE=Release
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -47,12 +48,24 @@ COPY --from=builder-ui /workspace/dist internal/server/dist
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
 ARG VERSION
 ENV GOCACHE=/root/.cache/go-build
-RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \
-  -ldflags="-w -s \
-  -X ${PACKAGE}/internal/version.Version=${VERSION} \
-  -X ${PACKAGE}/internal/version.CommitHash=${COMMIT_HASH} \
-  -X ${PACKAGE}/internal/version.BuildTimestamp=${BUILD_TIMESTAMP}" \
-  -o bin/burrito main.go
+
+RUN if [ "${BUILD_MODE}" = "Debug" ]; then go install github.com/go-delve/delve/cmd/dlv@latest; fi
+
+# Build with different flags based on debug mode
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    if [ "${BUILD_MODE}" = "Debug" ]; then \
+        GCFLAGS="all=-N -l"; \
+        LDFLAGS=""; \
+    else \
+        GCFLAGS=""; \
+        LDFLAGS="-w -s"; \
+    fi && \
+    CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \
+        -gcflags "${GCFLAGS}" \
+        -ldflags="${LDFLAGS} -X ${PACKAGE}/internal/version.Version=${VERSION} \
+        -X ${PACKAGE}/internal/version.CommitHash=${COMMIT_HASH} \
+        -X ${PACKAGE}/internal/version.BuildTimestamp=${BUILD_TIMESTAMP}" \
+        -o bin/burrito main.go
 
 FROM docker.io/library/alpine:3.21.3@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c
 
@@ -79,11 +92,13 @@ RUN addgroup \
   $USER
 
 # Copy the binary to the production image from the builder stage
-COPY --from=builder /workspace/bin/burrito /usr/local/bin/burrito
+# Copy /go/bin/dlv*: the wildcard makes the copy to work, even if the binary is not present (in Release mode)
+COPY --from=builder /workspace/bin/burrito /go/bin/dlv* /usr/local/bin/
 
 RUN mkdir -p /runner/bin
-RUN chmod +x /usr/local/bin/burrito
-RUN chown -R burrito:burrito /runner
+RUN chmod +x /usr/local/bin/*
+# /home/burrito/.config is required for debug mode
+RUN mkdir -p /home/burrito/.config && chown -R burrito:burrito /runner /home/burrito/.config    
 
 # Use an unprivileged user
 USER 65532:65532

Makefile

@@ -109,17 +109,35 @@ test: manifests generate fmt vet envtest ## Run tests.
 
 NEW_VERSION := $(shell date +%s)
 
+# Common function for helm upgrades
+define upgrade-helm-common
+	helm upgrade --install -f deploy/charts/burrito/values.yaml -f deploy/charts/burrito/$(1) -n burrito-system --create-namespace burrito-system deploy/charts/burrito
+endef
+
+# Common function for kind upgrades
+define upgrade-kind-common
+	docker buildx build --tag burrito:$(NEW_VERSION) --build-arg VERSION=${NEW_VERSION} $(2) .
+	kind load docker-image burrito:$(NEW_VERSION)
+	yq e '.global.deployment.image.tag = "$(NEW_VERSION)"' -i deploy/charts/burrito/$(1)
+	yq e '.config.burrito.runner.image.tag = "$(NEW_VERSION)"' -i deploy/charts/burrito/$(1)
+	$(call upgrade-helm-common,$(1))
+endef
+
 .PHONY: upgrade-dev-kind
 upgrade-dev-kind:
-	docker buildx build --tag burrito:$(NEW_VERSION) --build-arg VERSION=${NEW_VERSION} .
-	kind load docker-image burrito:$(NEW_VERSION)
-	yq e '.global.deployment.image.tag = "$(NEW_VERSION)"' -i deploy/charts/burrito/values-dev.yaml
-	yq e '.config.burrito.runner.image.tag = "$(NEW_VERSION)"' -i deploy/charts/burrito/values-dev.yaml
-	helm upgrade --install -f deploy/charts/burrito/values.yaml -f deploy/charts/burrito/values-dev.yaml -n burrito-system --create-namespace burrito-system deploy/charts/burrito
+	$(call upgrade-kind-common,values-dev.yaml,--build-arg BUILD_MODE=Release)
+
+.PHONY: upgrade-debug-kind
+upgrade-debug-kind:
+	$(call upgrade-kind-common,values-debug.yaml,--build-arg BUILD_MODE=Debug)
 
 .PHONY: upgrade-dev-helm
 upgrade-dev-helm:
-	helm upgrade --install -f deploy/charts/burrito/values.yaml -f deploy/charts/burrito/values-dev.yaml -n burrito-system --create-namespace burrito-system deploy/charts/burrito
+	$(call upgrade-helm-common,values-dev.yaml)
+
+.PHONY: upgrade-debug-helm
+upgrade-debug-helm:
+	$(call upgrade-helm-common,values-debug.yaml)
 
 ##@ Build
 

deploy/charts/burrito/templates/controllers.yaml

@@ -46,10 +46,12 @@ spec:
           imagePullPolicy: {{ .deployment.image.pullPolicy }}
           ports:
             {{- toYaml .deployment.ports | nindent 12 }}
+          {{- if eq .deployment.mode "Release" }}
           livenessProbe:
             {{- toYaml .deployment.livenessProbe | nindent 12 }}
           readinessProbe:
             {{- toYaml .deployment.readinessProbe | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .deployment.resources | nindent 12 }}
           env:

deploy/charts/burrito/templates/datastore.yaml

@@ -50,10 +50,12 @@ spec:
           imagePullPolicy: {{ .deployment.image.pullPolicy }}
           ports:
             {{- toYaml .deployment.ports | nindent 12 }}
+          {{- if eq .deployment.mode "Release" }}
           livenessProbe:
             {{- toYaml .deployment.livenessProbe | nindent 12 }}
           readinessProbe:
             {{- toYaml .deployment.readinessProbe | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .deployment.resources | nindent 12 }}
           env:

deploy/charts/burrito/values-debug.yaml

@@ -0,0 +1,32 @@
+config:
+  burrito:
+    runner:
+      image:
+        repository: burrito
+        tag: DEV_TAG
+        pullPolicy: Never
+    datastore:
+      storage:
+        mock: true
+global:
+  deployment:
+    image:
+      repository: burrito
+      tag: DEV_TAG
+      pullPolicy: Never
+tenants:
+  - namespace:
+      create: true
+      name: "burrito-project"
+
+# controllers:
+#   deployment:
+#     mode: Debug
+#     command: ["/usr/local/bin/dlv"]
+#     args: ["--listen=0.0.0.0:2345", "--headless=true", "--accept-multiclient", "--api-version=2", "--log", "exec", "/usr/local/bin/burrito", "controllers", "start"]
+
+# datastore:
+#   deployment:
+#     mode: Debug
+#     command: ["/usr/local/bin/dlv"]
+#     args: ["--listen=0.0.0.0:2347", "--headless=true", "--accept-multiclient", "--api-version=2", "--log", "exec", "/usr/local/bin/burrito", "datastore", "start"]

deploy/charts/burrito/values.yaml

@@ -205,6 +205,7 @@ global:
       app.kubernetes.io/part-of: burrito
     annotations: {}
   deployment:
+    mode: Release
     autoscaling:
       # -- Enable/Disable autoscaling for Burrito pods
       enabled: false

docs/assets/demo/vscode-breakpoint.png

@@ -188,3 +188,161 @@ It is strongly recommended to create a GitHub token with no specific rights to b
 
 - [Controller-runtime documentation](https://pkg.go.dev/sigs.k8s.io/controller-runtime) (Burrito heavily relies on this package)
 - [Burrito documentation](https://docs.burrito.tf/)
+
+## Debugging Burrito
+
+To debug Burrito efficiently in Kubernetes, you can use [Delve](https://github.com/go-delve/delve), optionally with [Visual Studio Code](https://code.visualstudio.com/) (recommended). We have set a few things to help you getting started.
+
+You'll need to follow instructions in [Getting Started: Set Up a Local Development Environment (kind)](#getting-started-set-up-a-local-development-environment-kind) to get a local Kubernetes development instance.
+
+### Enable debugging
+
+First, being by installing dlv: `go install github.com/go-delve/delve/cmd/dlv@latest`
+
+We'll rely on `deploy/charts/burrito/values-debug.yaml` to deploy the configuration to start the debugging session.
+
+By default, the `controller` (that includes the runner) and `datastore` are commented in the Helm values. Indeed, starting the application with dlv server will hang until you connect with the dlv client so it has to be enabled only when you need it.
+
+```yaml
+# controllers:
+#   deployment:
+#     mode: Debug
+#     command: ["/usr/local/bin/dlv"]
+#     args: ["--listen=0.0.0.0:2345", "--headless=true", "--accept-multiclient", "--api-version=2", "--log", "exec", "/usr/local/bin/burrito", "controllers", "start"]
+
+# datastore:
+#   deployment:
+#     mode: Debug
+#     command: ["/usr/local/bin/dlv"]
+#     args: ["--listen=0.0.0.0:2347", "--headless=true", "--accept-multiclient", "--api-version=2", "--log", "exec", "/usr/local/bin/burrito", "datastore", "start"]
+
+# server:
+#   deployment:
+#     mode: Debug
+#     command: ["/usr/local/bin/dlv"]
+#     args: ["--listen=0.0.0.0:2348", "--headless=true", "--accept-multiclient", "--api-version=2", "--log", "exec", "/usr/local/bin/burrito", "server", "start"]
+```
+
+By default, we'll start the application with the usual command. If you want to debug the controller or the runner, uncomment the required block. This will open a port on the pod on which you'll connect from your computer.
+
+`mode: Debug` is removing liveness and readiness probes: they won't be able to start as dlv will await for you to start the debugging session.
+
+### Deploy/refresh commands
+
+You'll need to deploy the debug container images and config. This is the same command if you need to refresh your deployment.
+
+To build a new local debug image of Burrito, push it into your local Kind cluster, and update the Helm release with the new image tag, run the following:
+
+```bash
+make upgrade-debug-kind
+```
+
+To refresh the Helm chart with debug values, run:
+
+```bash
+make upgrade-debug-helm
+```
+
+Check the [Makefile](https://github.com/padok-team/burrito/blob/main/Makefile) for more details about these commands.
+
+### Connect from your computer to the debug session
+
+The debugging port won't be exposed by default so you'll need to port-forward it.
+
+- For the controller:
+
+```bash
+kubectl port-forward $(kubectl get pods -n burrito-system | awk '/burrito-controllers.*Running/{print $1}') -n burrito-system 2345:2345
+```
+
+- For the runner:
+
+```bash
+kubectl port-forward -n burrito-project <layerName> 2346:2345
+```
+
+It will listen on the same port than the controller so we're exposing it on port 2346 on your computer so you can debug the controller and the runner if needed.
+
+- For the datastore:
+
+```bash
+kubectl port-forward $(kubectl get pods -n burrito-system | awk '/burrito-datastore.*Running/{print $1}') -n burrito-system 2347:2347
+```
+
+- For the server:
+
+```bash
+kubectl port-forward $(kubectl get pods -n burrito-system | awk '/burrito-server.*Running/{print $1}') -n burrito-system 2348:2348
+```
+
+### Start debugging
+
+#### With vscode
+
+!!! note
+    You can get more information about Vscode+Go debugging [here](https://github.com/golang/vscode-go/blob/master/docs/debugging.md)
+
+If you want to use Vscode to debug the app, you'll need to get the [Go extension](https://marketplace.visualstudio.com/items?itemName=golang.go) and create a `.vscode/launch.json`:
+
+```json
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Attach to Controller",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "port": 2345,
+            "host": "127.0.0.1",
+            "apiVersion": 2
+        },
+        {
+            "name": "Attach to Runner",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "port": 2346,
+            "host": "127.0.0.1",
+            "apiVersion": 2
+        },
+        {
+            "name": "Attach to Datastore",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "port": 2347,
+            "host": "127.0.0.1",
+            "apiVersion": 2
+        },
+        {
+            "name": "Attach to Server",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "port": 2348,
+            "host": "127.0.0.1",
+            "apiVersion": 2
+        }
+    ]
+}
+```
+
+!!! question "New to debugging on Vscode?"
+    For a vscode debug introduction, you can check [Debug code with Visual Studio Code](https://code.visualstudio.com/docs/debugtest/debugging).
+
+Browse your code to set breakpoints by clicking on the left side of your line.
+
+![Vscode breakpoint](assets/demo/vscode-breakpoint.png)
+
+Open the `Run and Debug` pane, select your debugging configuration and hit `F5` to connect to the remove `dlv`.
+
+![Vscode debug configuration](assets/demo/vscode-debug.png)
+
+Once your line is reached, vscode will show you the variables, current stack, etc
+
+![Vscode debug variables](assets/demo/vscode-debug-variables.png)
+
+#### With `dlv`
+
+If you prefer to debug on cli, you can connect with `dlv connect 127.0.0.1:<debuggingPort>` where `<debuggingPort>` is 2345, 2346, 2347 or 2348, depending on what you're debugging.