gha: make MaxConcurrentReconciles for each reconciler configurable

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -82,6 +82,18 @@ spec:
         {{- range .Values.flags.excludeLabelPropagationPrefixes }}
         - "--exclude-label-propagation-prefix={{ . }}"
         {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForAutoscalingRunnerSet }}
+        - "--max-concurrent-reconciles-for-autoscaling-runner-set={{ . }}"
+        {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForEphemeralRunnerSet }}
+        - "--max-concurrent-reconciles-for-ephemeral-runner-set={{ . }}"
+        {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForEphemeralRunner }}
+        - "--max-concurrent-reconciles-for-ephemeral-runner={{ . }}"
+        {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForAutoscalingListener }}
+        - "--max-concurrent-reconciles-for-autoscaling-listener={{ . }}"
+        {{- end }}
         command:
         - "/manager"
         {{- with .Values.metrics }}

charts/gha-runner-scale-set-controller/values.yaml

@@ -130,3 +130,9 @@ flags:
   ## Labels that match prefix specified in the list are excluded from propagation.
   # excludeLabelPropagationPrefixes:
   #   - "argocd.argoproj.io/instance"
+
+  ## Defines the maximum number of concurrent reconciles for each reconciler.
+  # maxConcurrentReconcilesForAutoscalingRunnerSet: 1
+  # maxConcurrentReconcilesForEphemeralRunnerSet: 1
+  # maxConcurrentReconcilesForEphemeralRunner: 1
+  # maxConcurrentReconcilesForAutoscalingListener: 1

controllers/actions.github.com/autoscalinglistener_controller.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -55,6 +56,8 @@ type AutoscalingListenerReconciler struct {
 	ListenerMetricsAddr     string
 	ListenerMetricsEndpoint string
 
+	MaxConcurrentReconciles int
+
 	ResourceBuilder
 }
 
@@ -730,6 +733,7 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error
 		Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
 		Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/actions.github.com/autoscalinglistener_controller_test.go

@@ -44,9 +44,10 @@ var _ = Describe("Test AutoScalingListener controller", func() {
 		configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
 
 		controller := &AutoscalingListenerReconciler{
-			Client: mgr.GetClient(),
-			Scheme: mgr.GetScheme(),
-			Log:    logf.Log,
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			MaxConcurrentReconciles: 1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -508,9 +509,10 @@ var _ = Describe("Test AutoScalingListener customization", func() {
 		configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
 
 		controller := &AutoscalingListenerReconciler{
-			Client: mgr.GetClient(),
-			Scheme: mgr.GetScheme(),
-			Log:    logf.Log,
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			MaxConcurrentReconciles: 1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -782,9 +784,10 @@ var _ = Describe("Test AutoScalingListener controller with proxy", func() {
 		configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
 
 		controller := &AutoscalingListenerReconciler{
-			Client: mgr.GetClient(),
-			Scheme: mgr.GetScheme(),
-			Log:    logf.Log,
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			MaxConcurrentReconciles: 1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -978,9 +981,10 @@ var _ = Describe("Test AutoScalingListener controller with template modification
 		configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
 
 		controller := &AutoscalingListenerReconciler{
-			Client: mgr.GetClient(),
-			Scheme: mgr.GetScheme(),
-			Log:    logf.Log,
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			MaxConcurrentReconciles: 1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -1094,9 +1098,10 @@ var _ = Describe("Test GitHub Server TLS configuration", func() {
 		Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs")
 
 		controller := &AutoscalingListenerReconciler{
-			Client: mgr.GetClient(),
-			Scheme: mgr.GetScheme(),
-			Log:    logf.Log,
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			MaxConcurrentReconciles: 1,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")

controllers/actions.github.com/autoscalingrunnerset_controller.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -79,6 +80,7 @@ type AutoscalingRunnerSetReconciler struct {
 	DefaultRunnerScaleSetListenerImagePullSecrets []string
 	UpdateStrategy                                UpdateStrategy
 	ActionsClient                                 actions.MultiClient
+	MaxConcurrentReconciles                       int
 	ResourceBuilder
 }
 
@@ -763,6 +765,7 @@ func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) erro
 			},
 		)).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/actions.github.com/autoscalingrunnerset_controller_test.go

@@ -71,6 +71,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
 			ControllerNamespace:                autoscalingNS.Name,
 			DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
 			ActionsClient:                      fake.NewMultiClient(),
+			MaxConcurrentReconciles:            1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -704,6 +705,7 @@ var _ = Describe("Test AutoScalingController updates", Ordered, func() {
 						nil,
 					),
 				),
+				MaxConcurrentReconciles: 1,
 			}
 			err := controller.SetupWithManager(mgr)
 			Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -819,6 +821,7 @@ var _ = Describe("Test AutoscalingController creation failures", Ordered, func()
 				ControllerNamespace:                autoscalingNS.Name,
 				DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
 				ActionsClient:                      fake.NewMultiClient(),
+				MaxConcurrentReconciles:            1,
 			}
 			err := controller.SetupWithManager(mgr)
 			Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -945,6 +948,7 @@ var _ = Describe("Test client optional configuration", Ordered, func() {
 				ControllerNamespace:                autoscalingNS.Name,
 				DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
 				ActionsClient:                      actions.NewMultiClient(logr.Discard()),
+				MaxConcurrentReconciles:            1,
 			}
 
 			err := controller.SetupWithManager(mgr)
@@ -1128,6 +1132,7 @@ var _ = Describe("Test client optional configuration", Ordered, func() {
 				ControllerNamespace:                autoscalingNS.Name,
 				DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
 				ActionsClient:                      fake.NewMultiClient(),
+				MaxConcurrentReconciles:            1,
 			}
 			err = controller.SetupWithManager(mgr)
 			Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -1362,6 +1367,7 @@ var _ = Describe("Test external permissions cleanup", Ordered, func() {
 			ControllerNamespace:                autoscalingNS.Name,
 			DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
 			ActionsClient:                      fake.NewMultiClient(),
+			MaxConcurrentReconciles:            1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -1520,6 +1526,7 @@ var _ = Describe("Test external permissions cleanup", Ordered, func() {
 			ControllerNamespace:                autoscalingNS.Name,
 			DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
 			ActionsClient:                      fake.NewMultiClient(),
+			MaxConcurrentReconciles:            1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -1728,6 +1735,7 @@ var _ = Describe("Test resource version and build version mismatch", func() {
 			ControllerNamespace:                autoscalingNS.Name,
 			DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
 			ActionsClient:                      fake.NewMultiClient(),
+			MaxConcurrentReconciles:            1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")

controllers/actions.github.com/ephemeralrunner_controller.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
@@ -49,9 +50,10 @@ const (
 // EphemeralRunnerReconciler reconciles a EphemeralRunner object
 type EphemeralRunnerReconciler struct {
 	client.Client
-	Log           logr.Logger
-	Scheme        *runtime.Scheme
-	ActionsClient actions.MultiClient
+	Log                     logr.Logger
+	Scheme                  *runtime.Scheme
+	ActionsClient           actions.MultiClient
+	MaxConcurrentReconciles int
 	ResourceBuilder
 }
 
@@ -828,6 +830,7 @@ func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&v1alpha1.EphemeralRunner{}).
 		Owns(&corev1.Pod{}).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/actions.github.com/ephemeralrunner_controller_test.go

@@ -101,10 +101,11 @@ var _ = Describe("EphemeralRunner", func() {
 			configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
 
 			controller = &EphemeralRunnerReconciler{
-				Client:        mgr.GetClient(),
-				Scheme:        mgr.GetScheme(),
-				Log:           logf.Log,
-				ActionsClient: fake.NewMultiClient(),
+				Client:                  mgr.GetClient(),
+				Scheme:                  mgr.GetScheme(),
+				Log:                     logf.Log,
+				ActionsClient:           fake.NewMultiClient(),
+				MaxConcurrentReconciles: 1,
 			}
 
 			err := controller.SetupWithManager(mgr)
@@ -681,6 +682,7 @@ var _ = Describe("EphemeralRunner", func() {
 						nil,
 					),
 				),
+				MaxConcurrentReconciles: 1,
 			}
 			err := controller.SetupWithManager(mgr)
 			Expect(err).To(BeNil(), "failed to setup controller")
@@ -737,10 +739,11 @@ var _ = Describe("EphemeralRunner", func() {
 			configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoScalingNS.Name)
 
 			controller = &EphemeralRunnerReconciler{
-				Client:        mgr.GetClient(),
-				Scheme:        mgr.GetScheme(),
-				Log:           logf.Log,
-				ActionsClient: fake.NewMultiClient(),
+				Client:                  mgr.GetClient(),
+				Scheme:                  mgr.GetScheme(),
+				Log:                     logf.Log,
+				ActionsClient:           fake.NewMultiClient(),
+				MaxConcurrentReconciles: 1,
 			}
 			err := controller.SetupWithManager(mgr)
 			Expect(err).To(BeNil(), "failed to setup controller")
@@ -901,10 +904,11 @@ var _ = Describe("EphemeralRunner", func() {
 			Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs")
 
 			controller = &EphemeralRunnerReconciler{
-				Client:        mgr.GetClient(),
-				Scheme:        mgr.GetScheme(),
-				Log:           logf.Log,
-				ActionsClient: fake.NewMultiClient(),
+				Client:                  mgr.GetClient(),
+				Scheme:                  mgr.GetScheme(),
+				Log:                     logf.Log,
+				ActionsClient:           fake.NewMultiClient(),
+				MaxConcurrentReconciles: 1,
 			}
 
 			err = controller.SetupWithManager(mgr)

controllers/actions.github.com/ephemeralrunnerset_controller.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
@@ -53,6 +54,8 @@ type EphemeralRunnerSetReconciler struct {
 
 	PublishMetrics bool
 
+	MaxConcurrentReconciles int
+
 	ResourceBuilder
 }
 
@@ -575,6 +578,7 @@ func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error
 		For(&v1alpha1.EphemeralRunnerSet{}).
 		Owns(&v1alpha1.EphemeralRunner{}).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/actions.github.com/ephemeralrunnerset_controller_test.go

@@ -48,10 +48,11 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() {
 		configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
 
 		controller := &EphemeralRunnerSetReconciler{
-			Client:        mgr.GetClient(),
-			Scheme:        mgr.GetScheme(),
-			Log:           logf.Log,
-			ActionsClient: fake.NewMultiClient(),
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			ActionsClient:           fake.NewMultiClient(),
+			MaxConcurrentReconciles: 1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -1098,10 +1099,11 @@ var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func(
 		configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)
 
 		controller := &EphemeralRunnerSetReconciler{
-			Client:        mgr.GetClient(),
-			Scheme:        mgr.GetScheme(),
-			Log:           logf.Log,
-			ActionsClient: actions.NewMultiClient(logr.Discard()),
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			ActionsClient:           actions.NewMultiClient(logr.Discard()),
+			MaxConcurrentReconciles: 1,
 		}
 		err := controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -1397,10 +1399,11 @@ var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func(
 		Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs")
 
 		controller := &EphemeralRunnerSetReconciler{
-			Client:        mgr.GetClient(),
-			Scheme:        mgr.GetScheme(),
-			Log:           logf.Log,
-			ActionsClient: actions.NewMultiClient(logr.Discard()),
+			Client:                  mgr.GetClient(),
+			Scheme:                  mgr.GetScheme(),
+			Log:                     logf.Log,
+			ActionsClient:           actions.NewMultiClient(logr.Discard()),
+			MaxConcurrentReconciles: 1,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")