[vropkg] (#839) Fix 'Too many files open' Error on Windows for Package with many Elements

docs/versions/latest/Release.md

@@ -39,6 +39,38 @@
 [//]: # (Optional But higlhy recommended Specify *NONE* if missing)
 [//]: # (#### Relevant Documentation:)
 
+### *Improve TypeScript workflow decorator documentation*
+
+#### Previous Behavior
+
+Workflow decorator typings in `vrotsc-annotations` provided limited IDE documentation, and the written workflow documentation could drift from the actual decorator/type surface (for example, older decorator naming and stale parameter lists).
+
+#### New Behavior
+
+Workflow-related decorators and configuration types now include richer, more consistent JSDoc (including readable links), and the latest workflow documentation is aligned with the typings (updated decorator naming and supported parameters).
+
+### *Bugfix for failure to find imported module of Powershell 7 polyglot action*
+
+#### Previous Behavior
+
+Executing a PowerShell 7 polyglot action with an imported module results in an error of the type "The specified module '...' was not loaded because no valid module file was found in any module directory."
+
+#### New Behavior
+
+When building a PowerShell polyglot action with imported modules the build process downloads modules to out/Modules/ via Save-Module and the ZIP bundle now correctly includes all files from Modules/ with proper folder structure.
+
+#### Previous Behavior
+
+Running vropkg on Windows for packages with many elements fails with:
+
+Error: EMFILE: too many open files
+
+#### New Behavior
+
+The Promise now resolves only when the `output` stream emits the `'close'` event, ensuring all data is written and file handles are released, hence avoiding the error for a package with many elements.
+
+### Improve healthcheck script, add healthcheck script and instructions for Windows
+
 ## Upgrade procedure
 
 [//]: # (Explain in details if something needs to be done)

typescript/vropkg/src/packaging.ts

@@ -13,39 +13,107 @@
  * #L%
  */
 import * as fs from "fs-extra";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as unzipper from 'unzipper';
 import * as winston from 'winston';
 import { WINSTON_CONFIGURATION } from "./constants";
+
+/**
+ * Extended archiver interface with custom properties for tracking
+ * output stream and path for proper cleanup
+ */
+interface ExtendedArchiver extends archiver.Archiver {
+	__outputStream?: fs.WriteStream;
+	__outputPath?: string;
+}
+
 /*
  * Utility class for variety of packaging operations.
  */
 
 /**
  * Create an archive for a given path. The archive will be finalized once
- * the archive object finalize method gets called
+ * the archive object finalize method gets called.
+ * 
+ * IMPORTANT: Callers must await the Promise returned by finalize() to ensure
+ * file handles are properly closed and prevent EMFILE errors.
  *
  * @param outputPath archive given path
- * @returns {archiver.Archiver} archiver object
+ * @returns {archiver.Archiver} archiver object with properly configured output stream
  */
 export const archive = (outputPath: string): archiver.Archiver => {
-	let instance =
-		archiver.create('zip', { zlib: { level: 6 } })
-			.on('warning', (err) => {
-				if (err.code === 'ENOENT') {
-					console.warn(err)
-				} else {
-					throw err;
-				}
-			})
-			.on('error', function (err) {
-				throw err;
-			})
-	instance.pipe(fs.createWriteStream(outputPath));
-
+	const output = fs.createWriteStream(outputPath);
+	const instance: ExtendedArchiver = archiver.create('zip', { zlib: { level: 6 } });
+
+	// Store the output stream reference for proper cleanup
+	instance.__outputStream = output;
+	instance.__outputPath = outputPath;
+
+	// Attach error handlers to output stream
+	output.on('error', (err) => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).error(`Error writing archive to ${outputPath}: ${err.message}`);
+		throw err;
+	});
+
+	// Attach event handlers to archiver
+	instance.on('warning', (err) => {
+		if (err.code !== 'ENOENT') {
+			throw err;
+		} else {
+			winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).warn(`Archive warning: ${err.message}`);
+		}
+	});
+
+	instance.on('error', (err) => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).error(`Error creating archive: ${err.message}`);
+		throw err;
+	});
+
+	// Log when archive is finalized
+	output.on('close', () => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).debug(`Archived ${instance.pointer()} bytes to ${outputPath}`);
+	});
+
+	// Pipe archive to output stream
+	instance.pipe(output);
+
 	return instance;
 }
 
+/**
+ * Properly finalize an archive by awaiting both the archive finalization
+ * and the output stream closure. This prevents EMFILE errors by ensuring
+ * file handles are released before continuing.
+ * 
+ * @param archive The archiver instance to finalize
+ * @returns Promise that resolves when both archive and stream are complete
+ */
+export const finalizeArchive = async (archive: archiver.Archiver): Promise<void> => {
+	return new Promise<void>((resolve, reject) => {
+		const extendedArchiver = archive as ExtendedArchiver;
+		const output = extendedArchiver.__outputStream;
+		const outputPath = extendedArchiver.__outputPath;
+
+		if (!output) {
+			// Fallback: just call finalize if no output stream is stored
+			archive.finalize().then(() => resolve()).catch(reject);
+			return;
+		}
+
+		// Wait for the output stream to close
+		output.on('close', () => {
+			resolve();
+		});
+
+		output.on('error', (err: Error) => {
+			reject(new Error(`Error writing archive to ${outputPath}: ${err.message}`));
+		});
+
+		// Start the finalization process
+		archive.finalize().catch(reject);
+	});
+}
+
 /**
  * Extract the content of a ZIP file into a selected folder
  *

typescript/vropkg/src/serialize/flat.ts

@@ -23,7 +23,7 @@ import * as s from "../security";
 import * as p from "../packaging"
 import { exist, isDirectory } from "../util";
 import { getPackageName, serialize, zipbundle, getActionXml, saveOptions, xmlOptions, infoOptions } from "./util"
-import * as archiver from "archiver";
+import archiver = require("archiver");
 import { decode } from "../encoding";
 import * as xmlDoc from "xmldoc";
 import { DEFAULT_ENCODING, FORM_ITEM_TEMPLATE, VSO_RESOURCE_INF, WINSTON_CONFIGURATION } from "../constants";
@@ -55,7 +55,8 @@ const initializeContext = (target: string) => {
     const bundle = (name: string): Promise<void> => {
         const arch = p.archive(path.join(target, name));
         [CERTIFICATES, ELEMENTS, SIGNATURES].forEach(folder => arch.directory(path.join(target, folder), folder));
-        return arch.append(fs.createReadStream(path.join(target, DUNES_META_INF)), { name: DUNES_META_INF }).finalize();
+        arch.append(fs.createReadStream(path.join(target, DUNES_META_INF)), { name: DUNES_META_INF });
+        return p.finalizeArchive(arch);
     }
     const store = serialize(target);
 
@@ -111,7 +112,7 @@ const serializeFlatElementData = (target: string) => {
         name: append(DATA_NAME),
         version: append(DATA_VERSION),
         data: append(DATA),
-        save: () => bundle.finalize()
+        save: () => p.finalizeArchive(bundle)
     }
 }
 

typescript/vropkg/src/serialize/tree.ts

@@ -14,7 +14,7 @@
  */
 import * as fs from "fs-extra";
 import * as path from "path";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as t from "../types";
 import * as xmlbuilder from "xmlbuilder";
 import { serialize, xmlOptions, complexActionComment, getActionXml } from "./util"
@@ -36,62 +36,84 @@ const serializeTreeElementContext = (target: string, elementName: string) => {
         data: (element: t.VroNativeElement, sourceFile: string, type: t.VroElementType) => {
             switch (type) {
                 case t.VroElementType.ResourceElement: {
-                    return fs.copyFileSync(sourceFile, path.join(target, `${elementName}`));
+                    return fs.copyFile(sourceFile, path.join(target, `${elementName}`));
                 }
                 case t.VroElementType.ScriptModule: {
                     let elementXmlPath = path.join(target, `${elementName}.xml`)
 					let actionXml = getActionXml(element.id, element.name, element.description, element.action);
-					fs.mkdirsSync(path.dirname(elementXmlPath));
-                    return fs.writeFileSync(elementXmlPath, actionXml);
+					return fs.mkdirs(path.dirname(elementXmlPath))
+                        .then(() => fs.writeFile(elementXmlPath, actionXml));
                 }
 				case t.VroElementType.ActionEnvironment: {
-                    return fs.copyFileSync(sourceFile, path.join(target, `${elementName}`));
+                    return fs.copyFile(sourceFile, path.join(target, `${elementName}`));
 				}
                 default: {
                     // Re-encode the content to UTF-8
-                    let buffer = fs.readFileSync(sourceFile);
-                    return fs.writeFileSync(path.join(target, `${elementName}.xml`), decode(buffer));
+                    return fs.readFile(sourceFile)
+                        .then(buffer => fs.writeFile(path.join(target, `${elementName}.xml`), decode(buffer)));
                 }
             }
         },
         bundle: (element: t.VroNativeElement, bundle: t.VroScriptBundle) => {
             if (bundle == null) {
                 // Empty promise that does nothing. Nothing needs to be done since bundle file does not exist.
-                return new Promise<void>((resolve, reject) => { });
+                return Promise.resolve();
             }
             let bundleFilePathSrc = bundle.contentPath;
             if (!exist(bundleFilePathSrc)) {
                 throw new Error(`Bundle path "${bundleFilePathSrc}" does not exist. Cannot get script bundle for element `
                     + `"${element.name}" of type "${element.type}"; category "${element.categoryPath}"; id: "${element.id}"`);
             }
             let bundleFilePathDest = path.join(target, `${elementName}.bundle.zip`);
-            if (isDirectory(bundleFilePathSrc)) {
-                let output = fs.createWriteStream(bundleFilePathDest);
-                let archive = archiver('zip', { zlib: { level: ZLIB_COMPRESS_LEVEL } });
-                archive.directory(bundleFilePathSrc, false);
+			if (!isDirectory(bundleFilePathSrc) ) {
+				return Promise.resolve().then(() => {
+				    fs.copySync(bundleFilePathSrc, bundleFilePathDest);
+				});	
+			}
+
+            return new Promise<void>((resolve, reject) => {
+                const output = fs.createWriteStream(bundleFilePathDest);
+                const archive = archiver('zip', { zlib: { level: ZLIB_COMPRESS_LEVEL } });
+                // Handle stream events
+                output.on('close', () => {
+                    resolve();
+                });
+                output.on('error', (err) => {
+                    reject(new Error(`Error writing bundle archive to ${bundleFilePathDest}: ${err.message}`));
+                });
+                archive.on('error', (err) => {
+                    reject(new Error(`Error creating bundle archive for ${bundleFilePathSrc}: ${err.message}`));
+                });
+                archive.on('warning', (err) => {
+                    if (err.code !== 'ENOENT') {
+                        reject(err);
+                    }
+                });
                 archive.pipe(output);
+                archive.directory(bundleFilePathSrc, false);
                 archive.finalize();
-            } else {
-                return fs.copyFile(bundleFilePathSrc, bundleFilePathDest);
-            }
+            });           
         },
         info: store(`${elementName}.element_info.xml`),
         tags: store(`${elementName}.tags.xml`),
         // if the object contains custom form then store it on the file system
         form: (element: t.VroNativeElement) => {
             if (element.form?.data) {
                 const formFileName = VRO_FORM_TEMPLATE.replace("{{elementName}}", elementName);
-                fs.writeFile(path.join(target, formFileName), JSON.stringify(element.form?.data, null, JSON_DEFAULT_IDENT));
+                return fs.writeFile(path.join(target, formFileName), JSON.stringify(element.form?.data, null, JSON_DEFAULT_IDENT));
             }
+            return Promise.resolve();
         },
         // if the object contains more forms (i.e. custom interaction enabled workflow) then store them on the file system
         formItems: (element: t.VroNativeElement) => {
             if (element.formItems && Array.isArray(element.formItems)) {
-                element.formItems.forEach((formItem: t.VroNativeFormElement) => {
+                const writePromises = element.formItems.map((formItem: t.VroNativeFormElement) => {
                     const customFormFileName = VRO_CUSTOM_FORMS_FILENAME_TEMPLATE.replace("{{elementName}}", elementName).replace("{{formName}}", formItem.name);
-                    fs.writeFile(path.join(target, customFormFileName), JSON.stringify(formItem.data, null, JSON_DEFAULT_IDENT));
+                    return fs.writeFile(path.join(target, customFormFileName), JSON.stringify(formItem.data, null, JSON_DEFAULT_IDENT));
                 });
+                return Promise.all(writePromises);
             }
+            return Promise.resolve();
         }
     }
 }

typescript/vropkg/src/serialize/util.ts

@@ -14,7 +14,7 @@
  */
 import * as fs from "fs-extra";
 import * as path from "path";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as t from "../types";
 import * as winston from 'winston';
 import * as xmlbuilder from "xmlbuilder";
@@ -57,15 +57,38 @@ export const zipbundle = (target: string) => {
     fs.mkdirsSync(target);
     return (file: string) => async (sourcePath: string, isDir: boolean): Promise<void> => {
         const absoluteZipPath = path.join(target, file);
-        if (isDir) {
-            let output = fs.createWriteStream(absoluteZipPath);
-            let archive = archiver('zip', { zlib: { level: 9 } });
-            archive.directory(sourcePath, false);
+		if (!isDir) {
+			return Promise.resolve().then(() => {
+			    fs.copySync(sourcePath, absoluteZipPath);
+			});			
+		}
+
+        return new Promise<void>((resolve, reject) => {
+            const output = fs.createWriteStream(absoluteZipPath);
+            const archive = archiver('zip', { zlib: { level: 9 } });
+
+            // Handle stream events
+            output.on('close', () => {
+                winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).debug(`Archived ${archive.pointer()} bytes to ${absoluteZipPath}`);
+                resolve();
+            });                
+            output.on('error', (err) => {
+                reject(new Error(`Error writing archive to ${absoluteZipPath}: ${err.message}`));
+            });                
+            archive.on('error', (err) => {
+                reject(new Error(`Error creating archive for ${sourcePath}: ${err.message}`));
+            }); 
+            archive.on('warning', (err) => {
+                if (err.code === 'ENOENT') {
+                    winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).warn(`Archive warning: ${err.message}`);
+                } else {
+                    reject(err);
+                }
+            });
             archive.pipe(output);
+            archive.directory(sourcePath, false);
             archive.finalize();
-        } else {
-            fs.copySync(sourcePath, absoluteZipPath);
-        }
+        });
     }
 }