feat(warm-pool): add ghr:warm-pool-member tag for visual identification

Tag instances with ghr:warm-pool-member=true when stopped into the warm
pool (scale-down). Remove the tag when starting from the warm pool
(scale-up). This provides instant visibility in the EC2 console for
which stopped instances are currently warm pool members.

- scale-down: tag after stopRunner in both removeRunner and
  stopOrTerminateOrphan paths
- scale-up: untag in findAndStartWarmRunners alongside the
  ghr:started-from-warm-pool tag
- lambda-warm-pool.json: add ec2:DeleteTags permission for untag

Author: Brend-Smits

lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, beforeEach, vi } from 'vitest';
-import { startRunner, tag } from './../aws/runners';
+import { startRunner, tag, untag } from './../aws/runners';
 import {
   getWarmPoolConfig,
   getPoolStrategy,
@@ -46,6 +46,7 @@ vi.mock('./job-retry', () => ({
 
 const mockStartRunner = vi.mocked(startRunner);
 const mockTag = vi.mocked(tag);
+const mockUntag = vi.mocked(untag);
 const mockGetWarmPoolConfig = vi.mocked(getWarmPoolConfig);
 const mockGetPoolStrategy = vi.mocked(getPoolStrategy);
 const mockListWarmInstances = vi.mocked(listWarmInstancesByOwner);
@@ -144,6 +145,7 @@ describe('findAndStartWarmRunners', () => {
 
     expect(result).toEqual(['i-warm-1']);
     expect(mockTag).toHaveBeenCalledWith('i-warm-1', [{ Key: 'ghr:started-from-warm-pool', Value: 'true' }]);
+    expect(mockUntag).toHaveBeenCalledWith('i-warm-1', [{ Key: 'ghr:warm-pool-member' }]);
   });
 
   it('should succeed even if tag fails (best-effort)', async () => {

lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts

@@ -719,6 +719,9 @@ describe('Scale down runners', () => {
         await scaleDown();
 
         expect(mockStopRunner).toHaveBeenCalledWith(runners[0].instanceId);
+        expect(mockTagRunners).toHaveBeenCalledWith(runners[0].instanceId, [
+          { Key: 'ghr:warm-pool-member', Value: 'true' },
+        ]);
         expect(mockAddToWarmPool).toHaveBeenCalledWith({
           instanceId: runners[0].instanceId,
           runnerOwner: runners[0].owner,

lambdas/functions/control-plane/src/scale-runners/scale-down.ts

@@ -174,6 +174,7 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promi
           if (warmCount < warmPoolConfig.maxWarmInstances) {
             try {
               await stopRunner(ec2runner.instanceId);
+              await tag(ec2runner.instanceId, [{ Key: 'ghr:warm-pool-member', Value: 'true' }]);
               await addToWarmPool({
                 instanceId: ec2runner.instanceId,
                 runnerOwner: ec2runner.owner,
@@ -350,6 +351,7 @@ async function stopOrTerminateOrphan(
     if (warmCount < warmPoolConfig.maxWarmInstances) {
       try {
         await stopRunner(runner.instanceId);
+        await tag(runner.instanceId, [{ Key: 'ghr:warm-pool-member', Value: 'true' }]);
         await addToWarmPool({
           instanceId: runner.instanceId,
           runnerOwner: owner,

lambdas/functions/control-plane/src/scale-runners/scale-up.ts

@@ -4,7 +4,7 @@ import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util';
 import yn from 'yn';
 
 import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth';
-import { createRunner, listEC2Runners, startRunner, tag, terminateRunner } from './../aws/runners';
+import { createRunner, listEC2Runners, startRunner, tag, untag, terminateRunner } from './../aws/runners';
 import { RunnerInputParameters } from './../aws/runners.d';
 import { metricGitHubAppRateLimit } from '../github/rate-limit';
 import { publishRetryMessage } from './job-retry';
@@ -282,9 +282,12 @@ export async function findAndStartWarmRunners(
       emitWarmPoolMetric('WarmPoolInstanceStarted', 1, { Owner: runnerOwner });
       logger.info(`Started warm instance '${entry.instanceId}' for owner '${runnerOwner}'`);
 
-      // Observability tag per ADR: marks this instance was started from warm pool (permanent, best-effort)
-      await tag(entry.instanceId, [{ Key: 'ghr:started-from-warm-pool', Value: 'true' }]).catch((e) => {
-        logger.warn(`Failed to tag '${entry.instanceId}' as started-from-warm-pool, continuing`, { error: e });
+      // Observability tags (best-effort): mark started from warm pool + remove membership tag
+      await Promise.all([
+        tag(entry.instanceId, [{ Key: 'ghr:started-from-warm-pool', Value: 'true' }]),
+        untag(entry.instanceId, [{ Key: 'ghr:warm-pool-member' }]),
+      ]).catch((e) => {
+        logger.warn(`Failed to update tags on '${entry.instanceId}', continuing`, { error: e });
       });
     } catch (e) {
       logger.warn(`Failed to start warm instance '${entry.instanceId}', skipping`, { error: e as Error });

modules/runners/policies/lambda-warm-pool.json

@@ -19,7 +19,8 @@
       "Action": [
         "ec2:StopInstances",
         "ec2:StartInstances",
-        "ec2:CreateTags"
+        "ec2:CreateTags",
+        "ec2:DeleteTags"
       ],
       "Resource": "*",
       "Condition": {
@@ -33,7 +34,8 @@
       "Action": [
         "ec2:StopInstances",
         "ec2:StartInstances",
-        "ec2:CreateTags"
+        "ec2:CreateTags",
+        "ec2:DeleteTags"
       ],
       "Resource": "*",
       "Condition": {