Merge remote-tracking branch 'origin/main' into v4/self-hosting

Author: nicktrn

.changeset/flat-pianos-live.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Runtime agnostic SDK config via env vars

.changeset/ninety-games-grow.md

@@ -0,0 +1,11 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Resolve issue where CLI could get stuck during deploy finalization
+- Unify local and remote build logic, with multi-platform build support
+- Improve switch command; now accepts profile name as an argument
+- Registry configuration is now fully managed by the webapp
+- The deploy `--self-hosted` flag is no longer required
+- Enhance deployment error reporting and image digest retrieval

.env.example

@@ -80,14 +80,9 @@ CLOUD_SLACK_CLIENT_SECRET=
 PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32`
 COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32`
 
-# Uncomment the following line to enable the registry proxy
-# ENABLE_REGISTRY_PROXY=true
+# DEPOT_ORG_ID=<Depot org id>
 # DEPOT_TOKEN=<Depot org token>
-# DEPOT_PROJECT_ID=<Depot project id>
-# DEPLOY_REGISTRY_HOST=${APP_ORIGIN} # This is the host that the deploy CLI will use to push images to the registry
-# CONTAINER_REGISTRY_ORIGIN=<Container registry origin e.g. https://registry.digitalocean.com>
-# CONTAINER_REGISTRY_USERNAME=<Container registry username e.g. Digital ocean email address>
-# CONTAINER_REGISTRY_PASSWORD=<Container registry password e.g. Digital ocean PAT>
+DEPLOY_REGISTRY_HOST=${APP_ORIGIN} # This is the host that the deploy CLI will use to push images to the registry
 # DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
 # These are needed for the object store (for handling large payloads/outputs)
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"

.github/workflows/publish-webapp.yml

@@ -56,6 +56,17 @@ jobs:
 
           echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
 
+      - name: 📝 Set the build info
+        id: set_build_info
+        run: |
+          tag=${{ steps.get_tag.outputs.tag }}
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            echo "BUILD_APP_VERSION=${tag}" >> "$GITHUB_OUTPUT"
+          fi
+          echo "BUILD_GIT_SHA=${{ github.sha }}" >> "$GITHUB_OUTPUT"
+          echo "BUILD_GIT_REF_NAME=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
+          echo "BUILD_TIMESTAMP_SECONDS=$(date +%s)" >> "$GITHUB_OUTPUT"
+
       - name: 🐙 Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -70,3 +81,8 @@ jobs:
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.set_tags.outputs.image_tags }}
           push: true
+          build-args: |
+            BUILD_APP_VERSION=${{ steps.set_build_info.outputs.BUILD_APP_VERSION }}
+            BUILD_GIT_SHA=${{ steps.set_build_info.outputs.BUILD_GIT_SHA }}
+            BUILD_GIT_REF_NAME=${{ steps.set_build_info.outputs.BUILD_GIT_REF_NAME }}
+            BUILD_TIMESTAMP_SECONDS=${{ steps.set_build_info.outputs.BUILD_TIMESTAMP_SECONDS }}

.github/workflows/unit-tests-internal.yml

@@ -127,4 +127,4 @@ jobs:
           merge-multiple: true
 
       - name: Merge reports
-        run: pnpm dlx [email protected] run --merge-reports
+        run: pnpm dlx [email protected] run --merge-reports --pass-with-no-tests

.github/workflows/unit-tests-packages.yml

@@ -127,4 +127,4 @@ jobs:
           merge-multiple: true
 
       - name: Merge reports
-        run: pnpm dlx [email protected] run --merge-reports
+        run: pnpm dlx [email protected] run --merge-reports --pass-with-no-tests

.github/workflows/unit-tests-webapp.yml

@@ -86,6 +86,7 @@ jobs:
           SESSION_SECRET: "secret"
           MAGIC_LINK_SECRET: "secret"
           ENCRYPTION_KEY: "secret"
+          DEPLOY_REGISTRY_HOST: "docker.io"
 
       - name: Gather all reports
         if: ${{ !cancelled() }}
@@ -133,4 +134,4 @@ jobs:
           merge-multiple: true
 
       - name: Merge reports
-        run: pnpm dlx [email protected] run --merge-reports
+        run: pnpm dlx [email protected] run --merge-reports --pass-with-no-tests

apps/supervisor/src/env.ts

@@ -29,8 +29,9 @@ const Env = z.object({
   RUNNER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().optional(),
   RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().optional(),
   RUNNER_ADDITIONAL_ENV_VARS: AdditionalEnvVars, // optional (csv)
+  RUNNER_PRETTY_LOGS: BoolEnv.default(false),
 
-  // Dequeue settings
+  // Dequeue settings (provider mode)
   TRIGGER_DEQUEUE_ENABLED: BoolEnv.default("true"),
   TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250),
   TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000),
@@ -48,6 +49,12 @@ const Env = z.object({
   RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
 
   // Docker settings
+  DOCKER_API_VERSION: z.string().default("v1.41"),
+  DOCKER_PLATFORM: z.string().optional(), // e.g. linux/amd64, linux/arm64
+  DOCKER_STRIP_IMAGE_DIGEST: BoolEnv.default(true),
+  DOCKER_REGISTRY_USERNAME: z.string().optional(),
+  DOCKER_REGISTRY_PASSWORD: z.string().optional(),
+  DOCKER_REGISTRY_URL: z.string().optional(), // e.g. https://index.docker.io/v1
   DOCKER_ENFORCE_MACHINE_PRESETS: BoolEnv.default(true),
   DOCKER_AUTOREMOVE_EXITED_CONTAINERS: BoolEnv.default(true),
   /**

apps/supervisor/src/util.ts

@@ -1,8 +1,7 @@
-export function getDockerHostDomain() {
-  const isMacOs = process.platform === "darwin";
-  const isWindows = process.platform === "win32";
+import { isMacOS, isWindows } from "std-env";
 
-  return isMacOs || isWindows ? "host.docker.internal" : "localhost";
+export function getDockerHostDomain() {
+  return isMacOS || isWindows ? "host.docker.internal" : "localhost";
 }
 
 export function getRunnerId(runId: string, attemptNumber?: number) {

apps/supervisor/src/workloadManager/docker.ts

@@ -14,9 +14,13 @@ export class DockerWorkloadManager implements WorkloadManager {
   private readonly docker: Docker;
 
   private readonly runnerNetworks: string[];
+  private readonly auth?: Docker.AuthConfig;
+  private readonly platformOverride?: string;
 
   constructor(private opts: WorkloadManagerOptions) {
-    this.docker = new Docker();
+    this.docker = new Docker({
+      version: env.DOCKER_API_VERSION,
+    });
 
     if (opts.workloadApiDomain) {
       this.logger.warn("⚠️ Custom workload API domain", {
@@ -25,6 +29,29 @@ export class DockerWorkloadManager implements WorkloadManager {
     }
 
     this.runnerNetworks = env.DOCKER_RUNNER_NETWORKS.split(",");
+
+    this.platformOverride = env.DOCKER_PLATFORM;
+    if (this.platformOverride) {
+      this.logger.info("🖥️  Platform override", {
+        targetPlatform: this.platformOverride,
+        hostPlatform: process.arch,
+      });
+    }
+
+    if (env.DOCKER_REGISTRY_USERNAME && env.DOCKER_REGISTRY_PASSWORD && env.DOCKER_REGISTRY_URL) {
+      this.logger.info("🐋 Using Docker registry credentials", {
+        username: env.DOCKER_REGISTRY_USERNAME,
+        url: env.DOCKER_REGISTRY_URL,
+      });
+
+      this.auth = {
+        username: env.DOCKER_REGISTRY_USERNAME,
+        password: env.DOCKER_REGISTRY_PASSWORD,
+        serveraddress: env.DOCKER_REGISTRY_URL,
+      };
+    } else {
+      this.logger.warn("🐋 No Docker registry credentials provided, skipping auth");
+    }
   }
 
   async create(opts: WorkloadManagerCreateOptions) {
@@ -47,6 +74,7 @@ export class DockerWorkloadManager implements WorkloadManager {
       `TRIGGER_RUNNER_ID=${runnerId}`,
       `TRIGGER_MACHINE_CPU=${opts.machine.cpu}`,
       `TRIGGER_MACHINE_MEMORY=${opts.machine.memory}`,
+      `PRETTY_LOGS=${env.RUNNER_PRETTY_LOGS}`,
     ];
 
     if (this.opts.warmStartUrl) {
@@ -89,41 +117,103 @@ export class DockerWorkloadManager implements WorkloadManager {
       hostConfig.Memory = opts.machine.memory * 1024 * 1024 * 1024;
     }
 
+    let imageRef = opts.image;
+
+    if (env.DOCKER_STRIP_IMAGE_DIGEST) {
+      imageRef = opts.image.split("@")[0]!;
+    }
+
     const containerCreateOpts: Docker.ContainerCreateOptions = {
-      Env: envVars,
       name: runnerId,
       Hostname: runnerId,
       HostConfig: hostConfig,
-      Image: opts.image,
+      Image: imageRef,
       AttachStdout: false,
       AttachStderr: false,
       AttachStdin: false,
     };
 
-    try {
-      // Create container
-      const container = await this.docker.createContainer(containerCreateOpts);
+    if (this.platformOverride) {
+      containerCreateOpts.platform = this.platformOverride;
+    }
+
+    const logger = this.logger.child({ opts, containerCreateOpts });
+
+    const [inspectError, inspectResult] = await tryCatch(this.docker.getImage(imageRef).inspect());
+
+    let shouldPull = !!inspectError;
+    if (this.platformOverride) {
+      const imageArchitecture = inspectResult?.Architecture;
+
+      // When the image architecture doesn't match the platform, we need to pull the image
+      if (imageArchitecture && !this.platformOverride.includes(imageArchitecture)) {
+        shouldPull = true;
+      }
+    }
+
+    // If the image is not present, try to pull it
+    if (shouldPull) {
+      logger.info("Pulling image", {
+        error: inspectError,
+        image: opts.image,
+        targetPlatform: this.platformOverride,
+        imageArchitecture: inspectResult?.Architecture,
+      });
 
-      // If there are multiple networks to attach to we need to attach the remaining ones after creation
-      if (remainingNetworks.length > 0) {
-        await this.attachContainerToNetworks({
-          containerId: container.id,
-          networkNames: remainingNetworks,
-        });
+      // Ensure the image is present
+      const [createImageError, imageResponseReader] = await tryCatch(
+        this.docker.createImage(this.auth, {
+          fromImage: imageRef,
+          ...(this.platformOverride ? { platform: this.platformOverride } : {}),
+        })
+      );
+      if (createImageError) {
+        logger.error("Failed to pull image", { error: createImageError });
+        return;
       }
 
-      // Start container
-      const startResult = await container.start();
+      const [imageReadError, imageResponse] = await tryCatch(readAllChunks(imageResponseReader));
+      if (imageReadError) {
+        logger.error("failed to read image response", { error: imageReadError });
+        return;
+      }
+
+      logger.debug("pulled image", { image: opts.image, imageResponse });
+    } else {
+      // Image is present, so we can use it to create the container
+    }
+
+    // Create container
+    const [createContainerError, container] = await tryCatch(
+      this.docker.createContainer({
+        ...containerCreateOpts,
+        // Add env vars here so they're not logged
+        Env: envVars,
+      })
+    );
 
-      this.logger.debug("create succeeded", {
-        opts,
-        startResult,
+    if (createContainerError) {
+      logger.error("Failed to create container", { error: createContainerError });
+      return;
+    }
+
+    // If there are multiple networks to attach to we need to attach the remaining ones after creation
+    if (remainingNetworks.length > 0) {
+      await this.attachContainerToNetworks({
         containerId: container.id,
-        containerCreateOpts,
+        networkNames: remainingNetworks,
       });
-    } catch (error) {
-      this.logger.error("create failed:", { opts, error, containerCreateOpts });
     }
+
+    // Start container
+    const [startError, startResult] = await tryCatch(container.start());
+
+    if (startError) {
+      logger.error("Failed to start container", { error: startError, containerId: container.id });
+      return;
+    }
+
+    logger.debug("create succeeded", { startResult, containerId: container.id });
   }
 
   private async attachContainerToNetworks({
@@ -172,3 +262,11 @@ export class DockerWorkloadManager implements WorkloadManager {
     });
   }
 }
+
+async function readAllChunks(reader: NodeJS.ReadableStream) {
+  const chunks = [];
+  for await (const chunk of reader) {
+    chunks.push(chunk.toString());
+  }
+  return chunks;
+}

apps/webapp/app/components/GitMetadata.tsx

@@ -8,8 +8,8 @@ export function GitMetadata({ git }: { git?: GitMetaLinks | null }) {
   return (
     <>
       {git.pullRequestUrl && git.pullRequestNumber && <GitMetadataPullRequest git={git} />}
-      {git.shortSha && <GitMetadataCommit git={git} />}
       {git.branchUrl && <GitMetadataBranch git={git} />}
+      {git.shortSha && <GitMetadataCommit git={git} />}
     </>
   );
 }

apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx

@@ -22,16 +22,27 @@ import { SideMenuItem } from "./SideMenuItem";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { Paragraph } from "../primitives/Paragraph";
 import { Badge } from "../primitives/Badge";
+import { useHasAdminAccess } from "~/hooks/useUser";
+
+export type BuildInfo = {
+  appVersion: string | undefined;
+  packageVersion: string;
+  buildTimestampSeconds: string | undefined;
+  gitSha: string | undefined;
+  gitRefName: string | undefined;
+};
 
 export function OrganizationSettingsSideMenu({
   organization,
-  version,
+  buildInfo,
 }: {
   organization: MatchedOrganization;
-  version: string;
+  buildInfo: BuildInfo;
 }) {
   const { isManagedCloud } = useFeatures();
   const currentPlan = useCurrentPlan();
+  const isAdmin = useHasAdminAccess();
+  const showBuildInfo = isAdmin || !isManagedCloud;
 
   return (
     <div
@@ -94,9 +105,33 @@ export function OrganizationSettingsSideMenu({
         <div className="flex flex-col gap-1">
           <SideMenuHeader title="App version" />
           <Paragraph variant="extra-small" className="px-2 text-text-dimmed">
-            v{version}
+            {buildInfo.appVersion || `v${buildInfo.packageVersion}`}
           </Paragraph>
         </div>
+        {showBuildInfo && buildInfo.buildTimestampSeconds && (
+          <div className="flex flex-col gap-1">
+            <SideMenuHeader title="Build timestamp" />
+            <Paragraph variant="extra-small" className="px-2 text-text-dimmed">
+              {new Date(Number(buildInfo.buildTimestampSeconds) * 1000).toISOString()}
+            </Paragraph>
+          </div>
+        )}
+        {showBuildInfo && buildInfo.gitRefName && (
+          <div className="flex flex-col gap-1">
+            <SideMenuHeader title="Git ref" />
+            <Paragraph variant="extra-small" className="px-2 text-text-dimmed">
+              {buildInfo.gitRefName}
+            </Paragraph>
+          </div>
+        )}
+        {showBuildInfo && buildInfo.gitSha && (
+          <div className="flex flex-col gap-1">
+            <SideMenuHeader title="Git sha" />
+            <Paragraph variant="extra-small" className="px-2 text-text-dimmed">
+              {buildInfo.gitSha.slice(0, 9)}
+            </Paragraph>
+          </div>
+        )}
       </div>
       <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
         <HelpAndFeedback />