[CLI] Explore spawning new OS processes in CLI spawnProcess (#3001)

## Motivation

This PR changes the Playground CLI so that every PHP subprocess spawned
by another PHP instance (e.g. `proc_open('php -v')`) is started in a
separate OS process.

When the sandboxed spawn handler detects a PHP subprocess request, it
now creates an entirely new Node.js worker thread running. Any file
locks acquired by the subprocess will properly conflict with the
parent's locks.

Before this PR two PHP instances running in the same OS process could
acquire overlapping OS-level locks (via `fcntl()` and `LockFileEx()`).

## Implementation

The way I'm augmenting `sandboxedSpawnHandlerFactory` calls in this PR
is somewhat convoluted. I'd like to revisit it once we finish the
OS-locks-related refactoring of the CLI code.

The `sandboxedSpawnHandlerFactory` now accepts a callback that creates a
new PHP worker. When a PHP subprocess is requested via `proc_open()`,
the CLI:

1. Spawns a new Node.js worker thread via `spawnWorkerThread('v1')`
2. Boots a fresh PHP runtime in that worker using `bootWorker()`
3. Connects the subprocess I/O streams to the parent
4. Terminates the worker when the subprocess exits

The key change is in `worker-thread-v1.ts` where `createPHPWorker()` now
spawns an entirely new OS process instead of reusing the existing PHP
process manager's pool.

## Follow-up work

* Refactor PlaygroundCliBlueprintV2Worker
* Refactor `@php-wasm/cli`
* Limit PHPProcessManager to one PHP instance per worker even for
request handling
* Get rid of `PHPProcessManager` usage in CLI workers

Author: adamziel

packages/php-wasm/node/src/test/php-part-1.spec.ts

@@ -1850,7 +1850,11 @@ describe('sandboxedSpawnHandlerFactory', () => {
 					'Hello, world!'
 				);
 				await php.setSpawnHandler(
-					sandboxedSpawnHandlerFactory(processManager)
+					sandboxedSpawnHandlerFactory(() =>
+						processManager.acquirePHPInstance({
+							considerPrimary: false,
+						})
+					)
 				);
 				return php;
 			},

packages/php-wasm/node/src/test/php-part-2.spec.ts

@@ -1180,7 +1180,11 @@ describe('sandboxedSpawnHandlerFactory', () => {
 					'Hello, world!'
 				);
 				await php.setSpawnHandler(
-					sandboxedSpawnHandlerFactory(processManager)
+					sandboxedSpawnHandlerFactory(() =>
+						processManager.acquirePHPInstance({
+							considerPrimary: false,
+						})
+					)
 				);
 				return php;
 			},

packages/php-wasm/universal/src/lib/index.ts

@@ -88,3 +88,4 @@ export { sandboxedSpawnHandlerFactory } from './sandboxed-spawn-handler-factory'
 
 export * from './api';
 export type { WithAPIState as WithIsReady } from './api';
+export type { Remote } from './comlink-sync';

packages/php-wasm/universal/src/lib/sandboxed-spawn-handler-factory.ts

@@ -1,5 +1,7 @@
 import { createSpawnHandler } from '@php-wasm/util';
-import type { PHPProcessManager } from './php-process-manager';
+import type { PHP } from './php';
+import type { PHPWorker } from './php-worker';
+import type { Remote } from './comlink-sync';
 
 /**
  * An isomorphic proc_open() handler that implements typical shell in TypeScript
@@ -12,7 +14,10 @@ import type { PHPProcessManager } from './php-process-manager';
  * parser.
  */
 export function sandboxedSpawnHandlerFactory(
-	processManager: PHPProcessManager
+	getPHPInstance: () => Promise<{
+		php: PHP | Remote<PHPWorker>;
+		reap: () => void;
+	}>
 ) {
 	return createSpawnHandler(async function (args, processApi, options) {
 		processApi.notifySpawn();
@@ -63,16 +68,14 @@ export function sandboxedSpawnHandlerFactory(
 			return;
 		}
 
-		const { php, reap } = await processManager.acquirePHPInstance({
-			considerPrimary: false,
-		});
+		const { php, reap } = await getPHPInstance();
 
 		try {
 			if (options.cwd) {
-				php.chdir(options.cwd as string);
+				await php.chdir(options.cwd as string);
 			}
 
-			const cwd = php.cwd();
+			const cwd = await php.cwd();
 			switch (binaryName) {
 				case 'php': {
 					// Figure out more about setting env, putenv(), etc.
@@ -105,7 +108,7 @@ export function sandboxedSpawnHandlerFactory(
 					break;
 				}
 				case 'ls': {
-					const files = php.listFiles(args[1] ?? cwd);
+					const files = await php.listFiles(args[1] ?? cwd);
 					for (const file of files) {
 						processApi.stdout(file + '\n');
 					}

packages/playground/cli/src/blueprints-v1/worker-thread-v1.ts

@@ -21,6 +21,8 @@ import { jspi } from 'wasm-feature-detect';
 import { MessageChannel, type MessagePort, parentPort } from 'worker_threads';
 import { mountResources } from '../mounts';
 import { logger } from '@php-wasm/logger';
+import { spawnWorkerThread } from '../run-cli';
+
 import type { Mount } from '@php-wasm/cli-util';
 
 export type WorkerBootOptions = {
@@ -122,31 +124,22 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 		}
 	}
 
-	async bootAndSetUpInitialWorker({
-		siteUrl,
-		mountsBeforeWpInstall,
-		mountsAfterWpInstall,
-		phpVersion: php = RecommendedPHPVersion,
-		wordpressInstallMode,
-		wordPressZip,
-		sqliteIntegrationPluginZip,
-		firstProcessId,
-		processIdSpaceLength,
-		dataSqlPath,
-		followSymlinks,
-		trace,
-		internalCookieStore,
-		withXdebug,
-		nativeInternalDirPath,
-	}: PrimaryWorkerBootOptions) {
+	async bootAndSetUpInitialWorker(options: PrimaryWorkerBootOptions) {
+		const {
+			siteUrl,
+			mountsBeforeWpInstall,
+			mountsAfterWpInstall,
+			wordpressInstallMode,
+			wordPressZip,
+			sqliteIntegrationPluginZip,
+			dataSqlPath,
+			internalCookieStore,
+		} = options;
 		if (this.booted) {
 			throw new Error('Playground already booted');
 		}
 		this.booted = true;
 
-		let nextProcessId = firstProcessId;
-		const lastProcessId = firstProcessId + processIdSpaceLength - 1;
-
 		try {
 			const constants: Record<string, string | number | boolean | null> =
 				{
@@ -157,27 +150,10 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 			let wordpressBooted = false;
 			const requestHandler = await bootWordPressAndRequestHandler({
 				siteUrl,
-				createPhpRuntime: async () => {
-					const processId = nextProcessId;
-
-					if (nextProcessId < lastProcessId) {
-						nextProcessId++;
-					} else {
-						// We've reached the end of the process ID space. Start over.
-						nextProcessId = firstProcessId;
-					}
-
-					return await loadNodeRuntime(php, {
-						emscriptenOptions: {
-							fileLockManager: this.fileLockManager!,
-							processId,
-							trace: trace ? tracePhpWasm : undefined,
-							phpWasmInitOptions: { nativeInternalDirPath },
-						},
-						followSymlinks,
-						withXdebug,
-					});
-				},
+				createPhpRuntime: createPhpRuntimeFactory(
+					options,
+					this.fileLockManager!
+				),
 				wordpressInstallMode,
 				wordPressZip:
 					wordPressZip !== undefined
@@ -203,7 +179,10 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 				},
 				cookieStore: internalCookieStore ? undefined : false,
 				dataSqlPath,
-				spawnHandler: sandboxedSpawnHandlerFactory,
+				spawnHandler: () =>
+					sandboxedSpawnHandlerFactory(() =>
+						createPHPWorker(options, this.fileLockManager!)
+					),
 				async onPHPInstanceCreated(php) {
 					await mountResources(php, mountsBeforeWpInstall);
 					if (wordpressBooted) {
@@ -230,64 +209,37 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 		}
 	}
 
+	async hello() {
+		return 'hello';
+	}
+
 	async bootWorker(args: WorkerBootOptions) {
 		await this.bootRequestHandler(args);
 	}
 
-	async bootRequestHandler({
-		siteUrl,
-		followSymlinks,
-		phpVersion,
-		firstProcessId,
-		processIdSpaceLength,
-		trace,
-		nativeInternalDirPath,
-		mountsBeforeWpInstall,
-		mountsAfterWpInstall,
-		withXdebug,
-	}: WorkerBootRequestHandlerOptions) {
+	async bootRequestHandler(options: WorkerBootRequestHandlerOptions) {
 		if (this.booted) {
 			throw new Error('Playground already booted');
 		}
 		this.booted = true;
 
-		let nextProcessId = firstProcessId;
-		const lastProcessId = firstProcessId + processIdSpaceLength - 1;
-
 		try {
 			const requestHandler = await bootRequestHandler({
-				siteUrl,
-				createPhpRuntime: async () => {
-					const processId = nextProcessId;
-
-					if (nextProcessId < lastProcessId) {
-						nextProcessId++;
-					} else {
-						// We've reached the end of the process ID space. Start over.
-						nextProcessId = firstProcessId;
-					}
-
-					return await loadNodeRuntime(phpVersion, {
-						emscriptenOptions: {
-							fileLockManager: this.fileLockManager!,
-							processId,
-							trace: trace ? tracePhpWasm : undefined,
-							ENV: {
-								DOCROOT: '/wordpress',
-							},
-							phpWasmInitOptions: { nativeInternalDirPath },
-						},
-						followSymlinks,
-						withXdebug,
-					});
-				},
+				siteUrl: options.siteUrl,
+				createPhpRuntime: createPhpRuntimeFactory(
+					options,
+					this.fileLockManager!
+				),
 				onPHPInstanceCreated: async (php) => {
-					await mountResources(php, mountsBeforeWpInstall);
-					await mountResources(php, mountsAfterWpInstall);
+					await mountResources(php, options.mountsBeforeWpInstall);
+					await mountResources(php, options.mountsAfterWpInstall);
 				},
 				sapiName: 'cli',
 				cookieStore: false,
-				spawnHandler: sandboxedSpawnHandlerFactory,
+				spawnHandler: () =>
+					sandboxedSpawnHandlerFactory(() =>
+						createPHPWorker(options, this.fileLockManager!)
+					),
 			});
 			this.__internal_setRequestHandler(requestHandler);
 
@@ -307,6 +259,91 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 	}
 }
 
+/**
+ * Returns a factory function that starts a new PHP runtime in the currently
+ * running process. This is used for rotating the PHP runtime periodically.
+ */
+function createPhpRuntimeFactory(
+	options: WorkerBootRequestHandlerOptions,
+	fileLockManager: FileLockManager | RemoteAPI<FileLockManager>
+) {
+	let nextProcessId = options.firstProcessId;
+	const lastProcessId =
+		options.firstProcessId + options.processIdSpaceLength - 1;
+	return async () => {
+		const processId = nextProcessId;
+
+		if (nextProcessId < lastProcessId) {
+			nextProcessId++;
+		} else {
+			// We've reached the end of the process ID space. Start over.
+			nextProcessId = options.firstProcessId;
+		}
+
+		return await loadNodeRuntime(
+			options.phpVersion || RecommendedPHPVersion,
+			{
+				emscriptenOptions: {
+					fileLockManager,
+					processId,
+					trace: options.trace ? tracePhpWasm : undefined,
+					phpWasmInitOptions: {
+						nativeInternalDirPath: options.nativeInternalDirPath,
+					},
+				},
+				followSymlinks: options.followSymlinks,
+				withXdebug: options.withXdebug,
+			}
+		);
+	};
+}
+
+/**
+ * Spawns a new PHP process to be used in the PHP spawn handler (in proc_open() etc. calls).
+ * It boots from this worker-thread-v1.ts file, but is a separate process.
+ *
+ * We explicitly avoid using PHPProcessManager.acquirePHPInstance() here.
+ *
+ * Why?
+ *
+ * Because each PHP instance acquires actual OS-level file locks via fcntl() and LockFileEx()
+ * syscalls. Running multiple PHP instances from the same OS process would allow them to
+ * acquire overlapping locks. Running every PHP instance in a separate OS process ensures
+ * any locks that overlap between PHP instances conflict with each other as expected.
+ *
+ * @param options - The options for the worker.
+ * @param fileLockManager - The file lock manager to use.
+ * @returns A promise that resolves to the PHP worker.
+ */
+async function createPHPWorker(
+	options: WorkerBootRequestHandlerOptions,
+	fileLockManager: FileLockManager | RemoteAPI<FileLockManager>
+) {
+	const spawnedWorker = await spawnWorkerThread('v1');
+
+	const handler = consumeAPI<PlaygroundCliBlueprintV1Worker>(
+		spawnedWorker.phpPort
+	);
+	handler.useFileLockManager(fileLockManager as any);
+	await handler.bootWorker(options);
+
+	return {
+		php: handler,
+		reap: () => {
+			try {
+				handler.dispose();
+			} catch {
+				/** */
+			}
+			try {
+				spawnedWorker.worker.terminate();
+			} catch {
+				/** */
+			}
+		},
+	};
+}
+
 process.on('unhandledRejection', (e: any) => {
 	logger.error('Unhandled rejection:', e);
 });