[vropkg] (#839) Fix 'Too many files open' Error on Windows for Package with many Elements

docs/versions/latest/Release.md

@@ -63,6 +63,16 @@ Executing a PowerShell 7 polyglot action with an imported module results in an e
 
 When building a PowerShell polyglot action with imported modules the build process downloads modules to out/Modules/ via Save-Module and the ZIP bundle now correctly includes all files from Modules/ with proper folder structure.
 
+#### Previous Behavior
+
+Running vropkg on Windows for packages with many elements fails with:
+
+Error: EMFILE: too many open files
+
+#### New Behavior
+
+The Promise now resolves only when the `output` stream emits the `'close'` event, ensuring all data is written and file handles are released, hence avoiding the error for a package with many elements.
+
 ## Upgrade procedure
 
 [//]: # (Explain in details if something needs to be done)

typescript/vropkg/src/packaging.ts

@@ -13,7 +13,7 @@
  * #L%
  */
 import * as fs from "fs-extra";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as unzipper from 'unzipper';
 import * as winston from 'winston';
 import { WINSTON_CONFIGURATION } from "./constants";
@@ -23,29 +23,94 @@ import { WINSTON_CONFIGURATION } from "./constants";
 
 /**
  * Create an archive for a given path. The archive will be finalized once
- * the archive object finalize method gets called
+ * the archive object finalize method gets called.
+ * 
+ * IMPORTANT: Callers must await the Promise returned by finalize() to ensure
+ * file handles are properly closed and prevent EMFILE errors.
  *
  * @param outputPath archive given path
- * @returns {archiver.Archiver} archiver object
+ * @returns {archiver.Archiver} archiver object with properly configured output stream
  */
 export const archive = (outputPath: string): archiver.Archiver => {
-	let instance =
-		archiver.create('zip', { zlib: { level: 6 } })
-			.on('warning', (err) => {
-				if (err.code === 'ENOENT') {
-					console.warn(err)
-				} else {
-					throw err;
-				}
-			})
-			.on('error', function (err) {
-				throw err;
-			})
-	instance.pipe(fs.createWriteStream(outputPath));
-
+	const output = fs.createWriteStream(outputPath);
+	const instance = archiver.create('zip', { zlib: { level: 6 } });
+
+	// Store the output stream reference for proper cleanup
+	(instance as any).__outputStream = output;
+	(instance as any).__outputPath = outputPath;
+
+	// Attach error handlers to output stream
+	output.on('error', (err) => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).error(
+			`Error writing archive to ${outputPath}: ${err.message}`
+		);
+		throw err;
+	});
+
+	// Attach event handlers to archiver
+	instance.on('warning', (err) => {
+		if (err.code === 'ENOENT') {
+			winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).warn(
+				`Archive warning: ${err.message}`
+			);
+		} else {
+			throw err;
+		}
+	});
+
+	instance.on('error', (err) => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).error(
+			`Error creating archive: ${err.message}`
+		);
+		throw err;
+	});
+
+	// Log when archive is finalized
+	output.on('close', () => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).debug(
+			`Archived ${instance.pointer()} bytes to ${outputPath}`
+		);
+	});
+
+	// Pipe archive to output stream
+	instance.pipe(output);
+
 	return instance;
 }
 
+/**
+ * Properly finalize an archive by awaiting both the archive finalization
+ * and the output stream closure. This prevents EMFILE errors by ensuring
+ * file handles are released before continuing.
+ * 
+ * @param archive The archiver instance to finalize
+ * @returns Promise that resolves when both archive and stream are complete
+ */
+export const finalizeArchive = async (archive: archiver.Archiver): Promise<void> => {
+	return new Promise<void>((resolve, reject) => {
+		const output = (archive as any).__outputStream;
+		const outputPath = (archive as any).__outputPath;
+
+		if (!output) {
+			// Fallback: just call finalize if no output stream is stored
+			archive.finalize().then(() => resolve()).catch(reject);
+			return;
+		}
+
+		// Wait for the output stream to close
+		output.on('close', () => {
+			resolve();
+		});
+
+		output.on('error', (err: Error) => {
+			reject(new Error(`Error writing archive to ${outputPath}: ${err.message}`));
+		});
+
+		// Start the finalization process
+		archive.finalize().catch(reject);
+	});
+}
+
 /**
  * Extract the content of a ZIP file into a selected folder
  *

typescript/vropkg/src/serialize/flat.ts

@@ -23,7 +23,7 @@ import * as s from "../security";
 import * as p from "../packaging"
 import { exist, isDirectory } from "../util";
 import { getPackageName, serialize, zipbundle, getActionXml, saveOptions, xmlOptions, infoOptions } from "./util"
-import * as archiver from "archiver";
+import archiver = require("archiver");
 import { decode } from "../encoding";
 import * as xmlDoc from "xmldoc";
 import { DEFAULT_ENCODING, FORM_ITEM_TEMPLATE, VSO_RESOURCE_INF, WINSTON_CONFIGURATION } from "../constants";
@@ -55,7 +55,8 @@ const initializeContext = (target: string) => {
     const bundle = (name: string): Promise<void> => {
         const arch = p.archive(path.join(target, name));
         [CERTIFICATES, ELEMENTS, SIGNATURES].forEach(folder => arch.directory(path.join(target, folder), folder));
-        return arch.append(fs.createReadStream(path.join(target, DUNES_META_INF)), { name: DUNES_META_INF }).finalize();
+        arch.append(fs.createReadStream(path.join(target, DUNES_META_INF)), { name: DUNES_META_INF });
+        return p.finalizeArchive(arch);
     }
     const store = serialize(target);
 
@@ -111,7 +112,7 @@ const serializeFlatElementData = (target: string) => {
         name: append(DATA_NAME),
         version: append(DATA_VERSION),
         data: append(DATA),
-        save: () => bundle.finalize()
+        save: () => p.finalizeArchive(bundle)
     }
 }
 

typescript/vropkg/src/serialize/tree.ts

@@ -14,7 +14,7 @@
  */
 import * as fs from "fs-extra";
 import * as path from "path";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as t from "../types";
 import * as xmlbuilder from "xmlbuilder";
 import { serialize, xmlOptions, complexActionComment, getActionXml } from "./util"
@@ -57,7 +57,7 @@ const serializeTreeElementContext = (target: string, elementName: string) => {
         bundle: (element: t.VroNativeElement, bundle: t.VroScriptBundle) => {
             if (bundle == null) {
                 // Empty promise that does nothing. Nothing needs to be done since bundle file does not exist.
-                return new Promise<void>((resolve, reject) => { });
+                return Promise.resolve();
             }
             let bundleFilePathSrc = bundle.contentPath;
             if (!exist(bundleFilePathSrc)) {
@@ -66,11 +66,33 @@ const serializeTreeElementContext = (target: string, elementName: string) => {
             }
             let bundleFilePathDest = path.join(target, `${elementName}.bundle.zip`);
             if (isDirectory(bundleFilePathSrc)) {
-                let output = fs.createWriteStream(bundleFilePathDest);
-                let archive = archiver('zip', { zlib: { level: ZLIB_COMPRESS_LEVEL } });
-                archive.directory(bundleFilePathSrc, false);
-                archive.pipe(output);
-                archive.finalize();
+                return new Promise<void>((resolve, reject) => {
+                    const output = fs.createWriteStream(bundleFilePathDest);
+                    const archive = archiver('zip', { zlib: { level: ZLIB_COMPRESS_LEVEL } });
+
+                    // Handle stream events
+                    output.on('close', () => {
+                        resolve();
+                    });
+
+                    output.on('error', (err) => {
+                        reject(new Error(`Error writing bundle archive to ${bundleFilePathDest}: ${err.message}`));
+                    });
+
+                    archive.on('error', (err) => {
+                        reject(new Error(`Error creating bundle archive for ${bundleFilePathSrc}: ${err.message}`));
+                    });
+
+                    archive.on('warning', (err) => {
+                        if (err.code !== 'ENOENT') {
+                            reject(err);
+                        }
+                    });
+
+                    archive.pipe(output);
+                    archive.directory(bundleFilePathSrc, false);
+                    archive.finalize();
+                });
             } else {
                 return fs.copyFile(bundleFilePathSrc, bundleFilePathDest);
             }

typescript/vropkg/src/serialize/util.ts

@@ -14,7 +14,7 @@
  */
 import * as fs from "fs-extra";
 import * as path from "path";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as t from "../types";
 import * as winston from 'winston';
 import * as xmlbuilder from "xmlbuilder";
@@ -58,11 +58,36 @@ export const zipbundle = (target: string) => {
     return (file: string) => async (sourcePath: string, isDir: boolean): Promise<void> => {
         const absoluteZipPath = path.join(target, file);
         if (isDir) {
-            let output = fs.createWriteStream(absoluteZipPath);
-            let archive = archiver('zip', { zlib: { level: 9 } });
-            archive.directory(sourcePath, false);
-            archive.pipe(output);
-            archive.finalize();
+            return new Promise<void>((resolve, reject) => {
+                const output = fs.createWriteStream(absoluteZipPath);
+                const archive = archiver('zip', { zlib: { level: 9 } });
+
+                // Handle stream events
+                output.on('close', () => {
+                    winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).debug(`Archived ${archive.pointer()} bytes to ${absoluteZipPath}`);
+                    resolve();
+                });
+
+                output.on('error', (err) => {
+                    reject(new Error(`Error writing archive to ${absoluteZipPath}: ${err.message}`));
+                });
+
+                archive.on('error', (err) => {
+                    reject(new Error(`Error creating archive for ${sourcePath}: ${err.message}`));
+                });
+
+                archive.on('warning', (err) => {
+                    if (err.code === 'ENOENT') {
+                        winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).warn(`Archive warning: ${err.message}`);
+                    } else {
+                        reject(err);
+                    }
+                });
+
+                archive.pipe(output);
+                archive.directory(sourcePath, false);
+                archive.finalize();
+            });
         } else {
             fs.copySync(sourcePath, absoluteZipPath);
         }