[vropkg] (#839) Fix 'Too many files open' Error on Windows for Package with many Elements

.github/workflows/release-drafter.yml

@@ -3,34 +3,39 @@ name: Release Drafter
 on:
   workflow_dispatch:
   push:
-    # branches to consider in the event; optional, defaults to all
-    branches:
-      - main
-  # pull_request event is required only for autolabeler
+    branches: [main]
   pull_request:
-    # Only following types are handled by the action, but one can default to all as well
-    types: [opened, reopened, synchronize]
-  # pull_request_target event is required for autolabeler to support PRs from forks
-  pull_request_target:
     types: [opened, reopened, synchronize]
 
 permissions:
   contents: read
 
 jobs:
   update_release_draft:
+    # Only run release drafting when it is safe/allowed to write releases
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     permissions:
-      # write permission is required to create a github release
       contents: write
-      # write permission is required for autolabeler
-      # otherwise, read permission is required at least
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      # Drafts your next Release notes as Pull Requests are merged into "main"
       - uses: release-drafter/release-drafter@139054aeaa9adc52ab36ddf67437541f039b88e2 # v7.1.1
         with:
           config-name: release-drafter.yml
           commitish: main
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  autolabel_pr:
+    if: github.event_name == 'pull_request'
+    permissions:
+      pull-requests: write
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@139054aeaa9adc52ab36ddf67437541f039b88e2
+        with:
+          config-name: release-drafter.yml
+          disable-releaser: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

docs/versions/latest/Release.md

@@ -47,9 +47,32 @@ Added a shim for `Object.entries`, which is not natively supported by the curren
 [//]: # (Optional But higlhy recommended Specify *NONE* if missing)
 [//]: # (#### Relevant Documentation:)
 
+### *Improve TypeScript workflow decorator documentation*
+
+#### Previous Behavior
+
+Workflow decorator typings in `vrotsc-annotations` provided limited IDE documentation, and the written workflow documentation could drift from the actual decorator/type surface (for example, older decorator naming and stale parameter lists).
+
+#### New Behavior
+
+Workflow-related decorators and configuration types now include richer, more consistent JSDoc (including readable links), and the latest workflow documentation is aligned with the typings (updated decorator naming and supported parameters).
+
+### *Bugfix for failure to find imported module of Powershell 7 polyglot action*
+
+#### Previous Behavior
+
+Executing a PowerShell 7 polyglot action with an imported module results in an error of the type "The specified module '...' was not loaded because no valid module file was found in any module directory."
+
+#### New Behavior
+
+When building a PowerShell polyglot action with imported modules the build process downloads modules to out/Modules/ via Save-Module and the ZIP bundle now correctly includes all files from Modules/ with proper folder structure.
+
+### Improve healthcheck script, add healthcheck script and instructions for Windows
+
 ### *Update type definitions for `RESTOperation` from `o11n-plugin-rest` plugin*
 
 ### *Extend type definitions for `o11n-plugin-vc` plugin*
+
 Update `VcToolsConfigInfo` type definition.
 Add the following new definitions:
 * `VcVirtualPCIPassthroughAllowedDevice`
@@ -59,6 +82,7 @@ Add the following new definitions:
 * `VcVirtualPCIPassthroughDynamicBackingOption`
 
 ### *Update type definitions for `FileReader` from `o11n-core` plugin*
+
 Add `close()` method.
 
 ### *Bugfix for Shimming `Object.values`*
@@ -71,6 +95,18 @@ Using `Object.values` would not be replaced by a Shim call and would result in a
 
 Using `Object.values` now correctly gets replaced by a Shim call (`VROES.Shims.objectValues`), preventing runtime errors when executed inside of vRO's Rhino JavaScript engine.
 
+### *Bugfix for too many opened files on Windows*
+
+#### Previous Behavior
+
+Running vropkg on Windows for packages with many elements fails with:
+
+Error: EMFILE: too many open files
+
+#### New Behavior
+
+The Promise now resolves only when the `output` stream emits the `'close'` event, ensuring all data is written and file handles are released, hence avoiding the error for a package with many elements.
+
 ## Upgrade procedure
 
 [//]: # (Explain in details if something needs to be done)

typescript/vropkg/src/packaging.ts

@@ -13,39 +13,125 @@
  * #L%
  */
 import * as fs from "fs-extra";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as unzipper from 'unzipper';
 import * as winston from 'winston';
 import { WINSTON_CONFIGURATION } from "./constants";
+
+/**
+ * Extended archiver interface with custom properties for tracking
+ * output stream and path for proper cleanup
+ */
+interface ExtendedArchiver extends archiver.Archiver {
+	__outputStream?: fs.WriteStream;
+	__outputPath?: string;
+}
+
 /*
  * Utility class for variety of packaging operations.
  */
 
 /**
  * Create an archive for a given path. The archive will be finalized once
- * the archive object finalize method gets called
+ * the archive object finalize method gets called.
+ * 
+ * IMPORTANT: Callers must await the Promise returned by finalize() to ensure
+ * file handles are properly closed and prevent EMFILE errors.
  *
  * @param outputPath archive given path
- * @returns {archiver.Archiver} archiver object
+ * @returns {archiver.Archiver} archiver object with properly configured output stream
  */
 export const archive = (outputPath: string): archiver.Archiver => {
-	let instance =
-		archiver.create('zip', { zlib: { level: 6 } })
-			.on('warning', (err) => {
-				if (err.code === 'ENOENT') {
-					console.warn(err)
-				} else {
-					throw err;
-				}
-			})
-			.on('error', function (err) {
-				throw err;
-			})
-	instance.pipe(fs.createWriteStream(outputPath));
-
+	const output = fs.createWriteStream(outputPath);
+	const instance: ExtendedArchiver = archiver.create('zip', { zlib: { level: 6 } });
+
+	// Store the output stream reference for proper cleanup
+	instance.__outputStream = output;
+	instance.__outputPath = outputPath;
+
+	// Attach error handlers to output stream
+	// Don't throw in event handlers - let finalizeArchive() handle errors via Promise rejection
+	output.on('error', (err) => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).error(`Error writing archive to ${outputPath}: ${err.message}`);
+		// Error will be caught by finalizeArchive() error handler
+	});
+
+	// Attach event handlers to archiver
+	instance.on('warning', (err) => {
+		if (err.code !== 'ENOENT') {
+			winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).warn(`Archive warning (non-ENOENT): ${err.message}`);
+			// Warning will be handled by finalizeArchive() if it becomes an error
+		} else {
+			winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).warn(`Archive warning: ${err.message}`);
+		}
+	});
+
+	instance.on('error', (err) => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).error(`Error creating archive: ${err.message}`);
+		// Error will be caught by finalizeArchive() error handler
+	});
+
+	// Log when archive is finalized
+	output.on('close', () => {
+		winston.loggers.get(WINSTON_CONFIGURATION.logPrefix).debug(`Archived ${instance.pointer()} bytes to ${outputPath}`);
+	});
+
+	// Pipe archive to output stream
+	instance.pipe(output);
+
 	return instance;
 }
 
+/**
+ * Properly finalize an archive by awaiting both the archive finalization
+ * and the output stream closure. This prevents EMFILE errors by ensuring
+ * file handles are released before continuing.
+ * 
+ * @param archive The archiver instance to finalize
+ * @returns Promise that resolves when both archive and stream are complete
+ */
+export const finalizeArchive = async (archive: archiver.Archiver): Promise<void> => {
+	return new Promise<void>((resolve, reject) => {
+		const extendedArchiver = archive as ExtendedArchiver;
+		const output = extendedArchiver.__outputStream;
+		const outputPath = extendedArchiver.__outputPath;
+
+		if (!output) {
+			// Fallback: just call finalize if no output stream is stored
+			archive.finalize().then(() => resolve()).catch(reject);
+			return;
+		}
+
+		// Guard against multiple Promise settlements
+		let settled = false;
+		const settleOnce = (settler: () => void) => {
+			if (!settled) {
+				settled = true;
+				settler();
+			}
+		};
+
+		// Use once() to avoid accumulating listeners
+		output.once('close', () => {
+			settleOnce(() => resolve());
+		});
+
+		output.once('error', (err: Error) => {
+			settleOnce(() => reject(new Error(`Error writing archive to ${outputPath}: ${err.message}`)));
+		});
+
+		// Also capture errors from the archiver itself
+		archive.once('error', (err: Error) => {
+			settleOnce(() => reject(new Error(`Archive error for ${outputPath}: ${err.message}`)));
+		});
+
+		// Start the finalization process and capture any immediate rejection
+		archive.finalize().catch((err: Error) => {
+			settleOnce(() => reject(new Error(`Failed to finalize archive ${outputPath}: ${err.message}`)));
+		});
+	});
+}
+
 /**
  * Extract the content of a ZIP file into a selected folder
  *

typescript/vropkg/src/serialize/flat.ts

@@ -23,7 +23,7 @@ import * as s from "../security";
 import * as p from "../packaging"
 import { exist, isDirectory } from "../util";
 import { getPackageName, serialize, zipbundle, getActionXml, saveOptions, xmlOptions, infoOptions } from "./util"
-import * as archiver from "archiver";
+import archiver = require("archiver");
 import { decode } from "../encoding";
 import * as xmlDoc from "xmldoc";
 import { DEFAULT_ENCODING, FORM_ITEM_TEMPLATE, VSO_RESOURCE_INF, WINSTON_CONFIGURATION } from "../constants";
@@ -55,7 +55,8 @@ const initializeContext = (target: string) => {
     const bundle = (name: string): Promise<void> => {
         const arch = p.archive(path.join(target, name));
         [CERTIFICATES, ELEMENTS, SIGNATURES].forEach(folder => arch.directory(path.join(target, folder), folder));
-        return arch.append(fs.createReadStream(path.join(target, DUNES_META_INF)), { name: DUNES_META_INF }).finalize();
+        arch.append(fs.createReadStream(path.join(target, DUNES_META_INF)), { name: DUNES_META_INF });
+        return p.finalizeArchive(arch);
     }
     const store = serialize(target);
 
@@ -111,7 +112,7 @@ const serializeFlatElementData = (target: string) => {
         name: append(DATA_NAME),
         version: append(DATA_VERSION),
         data: append(DATA),
-        save: () => bundle.finalize()
+        save: () => p.finalizeArchive(bundle)
     }
 }
 

typescript/vropkg/src/serialize/tree.ts

@@ -14,9 +14,10 @@
  */
 import * as fs from "fs-extra";
 import * as path from "path";
-import * as archiver from 'archiver';
+import archiver = require('archiver');
 import * as t from "../types";
 import * as xmlbuilder from "xmlbuilder";
+import * as p from "../packaging";
 import { serialize, xmlOptions, complexActionComment, getActionXml } from "./util"
 import { exist, isDirectory } from "../util";
 import { decode } from "../encoding";
@@ -36,62 +37,65 @@ const serializeTreeElementContext = (target: string, elementName: string) => {
         data: (element: t.VroNativeElement, sourceFile: string, type: t.VroElementType) => {
             switch (type) {
                 case t.VroElementType.ResourceElement: {
-                    return fs.copyFileSync(sourceFile, path.join(target, `${elementName}`));
+                    return fs.copyFile(sourceFile, path.join(target, `${elementName}`));
                 }
                 case t.VroElementType.ScriptModule: {
                     let elementXmlPath = path.join(target, `${elementName}.xml`)
 					let actionXml = getActionXml(element.id, element.name, element.description, element.action);
-					fs.mkdirsSync(path.dirname(elementXmlPath));
-                    return fs.writeFileSync(elementXmlPath, actionXml);
+					return fs.mkdirs(path.dirname(elementXmlPath))
+                        .then(() => fs.writeFile(elementXmlPath, actionXml));
                 }
 				case t.VroElementType.ActionEnvironment: {
-                    return fs.copyFileSync(sourceFile, path.join(target, `${elementName}`));
+                    return fs.copyFile(sourceFile, path.join(target, `${elementName}`));
 				}
                 default: {
                     // Re-encode the content to UTF-8
-                    let buffer = fs.readFileSync(sourceFile);
-                    return fs.writeFileSync(path.join(target, `${elementName}.xml`), decode(buffer));
+                    return fs.readFile(sourceFile)
+                        .then(buffer => fs.writeFile(path.join(target, `${elementName}.xml`), decode(buffer)));
                 }
             }
         },
         bundle: (element: t.VroNativeElement, bundle: t.VroScriptBundle) => {
             if (bundle == null) {
                 // Empty promise that does nothing. Nothing needs to be done since bundle file does not exist.
-                return new Promise<void>((resolve, reject) => { });
+                return Promise.resolve();
             }
             let bundleFilePathSrc = bundle.contentPath;
             if (!exist(bundleFilePathSrc)) {
                 throw new Error(`Bundle path "${bundleFilePathSrc}" does not exist. Cannot get script bundle for element `
                     + `"${element.name}" of type "${element.type}"; category "${element.categoryPath}"; id: "${element.id}"`);
             }
             let bundleFilePathDest = path.join(target, `${elementName}.bundle.zip`);
-            if (isDirectory(bundleFilePathSrc)) {
-                let output = fs.createWriteStream(bundleFilePathDest);
-                let archive = archiver('zip', { zlib: { level: ZLIB_COMPRESS_LEVEL } });
-                archive.directory(bundleFilePathSrc, false);
-                archive.pipe(output);
-                archive.finalize();
-            } else {
-                return fs.copyFile(bundleFilePathSrc, bundleFilePathDest);
-            }
+			if (!isDirectory(bundleFilePathSrc) ) {
+				return fs.copy(bundleFilePathSrc, bundleFilePathDest);
+			}
+			
+            const archive = p.archive(bundleFilePathDest);
+            archive.directory(bundleFilePathSrc, false);
+
+            // Use finalizeArchive instead of direct finalize() to avoid EMFILE errors
+            return p.finalizeArchive(archive);           
         },
         info: store(`${elementName}.element_info.xml`),
         tags: store(`${elementName}.tags.xml`),
         // if the object contains custom form then store it on the file system
         form: (element: t.VroNativeElement) => {
             if (element.form?.data) {
                 const formFileName = VRO_FORM_TEMPLATE.replace("{{elementName}}", elementName);
-                fs.writeFile(path.join(target, formFileName), JSON.stringify(element.form?.data, null, JSON_DEFAULT_IDENT));
+                return fs.writeFile(path.join(target, formFileName), JSON.stringify(element.form?.data, null, JSON_DEFAULT_IDENT));
             }
+            return Promise.resolve();
         },
         // if the object contains more forms (i.e. custom interaction enabled workflow) then store them on the file system
         formItems: (element: t.VroNativeElement) => {
             if (element.formItems && Array.isArray(element.formItems)) {
-                element.formItems.forEach((formItem: t.VroNativeFormElement) => {
+                const writePromises = element.formItems.map((formItem: t.VroNativeFormElement) => {
                     const customFormFileName = VRO_CUSTOM_FORMS_FILENAME_TEMPLATE.replace("{{elementName}}", elementName).replace("{{formName}}", formItem.name);
-                    fs.writeFile(path.join(target, customFormFileName), JSON.stringify(formItem.data, null, JSON_DEFAULT_IDENT));
+                    return fs.writeFile(path.join(target, customFormFileName), JSON.stringify(formItem.data, null, JSON_DEFAULT_IDENT));
                 });
+                return Promise.all(writePromises);
             }
+            return Promise.resolve();
         }
     }
 }