Spike: Pre-bundle certain plugins for Insomnia and Inso

.gitignore

@@ -15,6 +15,7 @@ coverage
 build/Release
 node_modules/
 **/node_modules/
+plugins/
 .npm
 .eslintcache
 .node_repl_history

packages/insomnia-inso/esbuild.ts

@@ -3,6 +3,7 @@ import { build, type BuildOptions, context } from 'esbuild';
 const isProd = Boolean(process.env.NODE_ENV === 'production');
 const watch = Boolean(process.env.ESBUILD_WATCH);
 const version = process.env.VERSION || 'dev';
+
 const config: BuildOptions = {
   outfile: './dist/index.js',
   bundle: true,

packages/insomnia-inso/package.json

@@ -19,16 +19,21 @@
   "bin": {
     "inso": "bin/inso"
   },
+  "pkg": {
+    "assets": "plugins/**/*",
+    "outputPath": "plugins"
+  },
   "scripts": {
     "lint": "eslint . --ext .js,.ts,.tsx --cache",
+    "bundle-plugins": "esr src/scripts/bundle-plugins.ts",
     "test:unit": "cross-env NO_COLOR=1 vitest run --exclude '**/cli.test.ts'",
     "test:bundle": "vitest cli.test.ts -t \"inso dev bundle\"",
     "test:binary": "vitest cli.test.ts -t \"inso packaged binary\"",
     "type-check": "tsc --noEmit --project tsconfig.json",
     "build": "esr esbuild.ts",
     "build:production": "cross-env NODE_ENV=production esr esbuild.ts",
     "start": "ESBUILD_WATCH=true esr esbuild.ts",
-    "prepackage": "npm run build:production && rm -rf binaries/inso",
+    "prepackage": "npm run build:production && rm -rf binaries/inso && npm run bundle-plugins",
     "package": "npx -y @yao-pkg/[email protected] . --output binaries/inso --targets host",
     "postpackage": "node src/scripts/verify-pkg.js",
     "artifacts": "esr src/scripts/artifacts.ts"

packages/insomnia-inso/src/db/index.ts

@@ -2,6 +2,7 @@ import { stat } from 'node:fs/promises';
 
 import type { CaCertificate } from 'insomnia/src/models/ca-certificate';
 import type { ClientCertificate } from 'insomnia/src/models/client-certificate';
+import type { CloudProviderCredential } from 'insomnia/src/models/cloud-credential';
 import type { CookieJar } from 'insomnia/src/models/cookie-jar';
 
 import { logger } from '../cli';
@@ -30,6 +31,7 @@ export interface Database {
   ClientCertificate: ClientCertificate[];
   CaCertificate: CaCertificate[];
   CookieJar: CookieJar[];
+  CloudCredential: CloudProviderCredential[];
 }
 
 export const emptyDb = (): Database => ({
@@ -44,6 +46,7 @@ export const emptyDb = (): Database => ({
   ClientCertificate: [],
   CaCertificate: [],
   CookieJar: [],
+  CloudCredential: [],
 });
 
 export type DbAdapter = (dir: string, filterTypes?: (keyof Database)[]) => Promise<Database | null>;

packages/insomnia-inso/src/scripts/bundle-plugins.ts

@@ -0,0 +1,36 @@
+// scripts/bundle-plugins.ts
+import { execSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { cp, mkdir, rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
+
+const appPluginDir = resolve(__dirname, '../../../insomnia/plugins');
+const insoPluginDir = resolve(__dirname, '../../plugins');
+
+if (require.main === module) {
+  process.nextTick(async () => {
+    try {
+      // remove existing bundle plugins directory
+      await rm(insoPluginDir, { recursive: true, force: true });
+      // Recreate empty
+      await mkdir(insoPluginDir, { recursive: true });
+      // Check if app plugin directory exists
+      if (!existsSync(appPluginDir)) {
+        console.log(`Source plugins directory not found at ${appPluginDir}. Creating and bundle plugins`);
+
+        execSync('npm run bundle-plugins', {
+          stdio: 'inherit',
+          cwd: resolve(appPluginDir, '../'),
+        });
+      }
+      // Copy plugins from app to inso directory
+      console.log(`Copying plugins from ${appPluginDir} to ${insoPluginDir}`);
+      await cp(appPluginDir, insoPluginDir, {
+        recursive: true,
+      });
+    } catch (err) {
+      console.log('[bundle-plugin] ERROR:', err);
+      process.exit(1);
+    }
+  });
+}

packages/insomnia/electron-builder.config.js

@@ -1,10 +1,11 @@
+const { execSync } = require('child_process');
 const BINARY_PREFIX = 'Insomnia.Core';
 // NOTE: USE_HARD_LINKS
 // https://github.com/electron-userland/electron-builder/issues/4594#issuecomment-574653870
 
 /**
  * @type {import('electron-builder').Configuration}
- * @see https://www.electron.build/configuration/configuration
+ * @see https://www.electron.build/configuration
  */
 const config = {
   npmRebuild: false,
@@ -31,6 +32,11 @@ const config = {
       to: './bin',
       filter: 'yarn-standalone.js',
     },
+    {
+      from: './plugins',
+      to: './plugins',
+      filter: ['**/*'],
+    }
   ],
   extraMetadata: {
     main: 'main.min.js', // Override the main path in package.json

packages/insomnia/package.json

@@ -17,12 +17,13 @@
   "author": "Kong <[email protected]>",
   "main": "src/main.min.js",
   "scripts": {
-    "build": "esr --cache ./scripts/build.ts --noErrorTruncation",
+    "build": "NODE_OPTIONS='--max-old-space-size=8192' esr --cache ./scripts/build.ts --noErrorTruncation",
+    "bundle-plugins": "esr scripts/bundle-plugins.ts",
     "generate:schema": "esr ./src/schema.ts",
     "build:main.min.js": "cross-env NODE_ENV=development esr esbuild.main.ts",
     "lint": "eslint . --ext .js,.ts,.tsx --cache",
-    "package": "npm run build && cross-env USE_HARD_LINKS=false electron-builder build --config electron-builder.config.js",
-    "package:windows:unpacked": "npm run build && cross-env USE_HARD_LINKS=false electron-builder build --config electron-builder.config.js --dir",
+    "package": "npm run build && npm run bundle-plugins && cross-env USE_HARD_LINKS=false electron-builder build --config electron-builder.config.js",
+    "package:windows:unpacked": "npm run build && npm run bundle-plugins && cross-env USE_HARD_LINKS=false electron-builder build --config electron-builder.config.js --dir",
     "package:windows:dist": "cross-env USE_HARD_LINKS=false electron-builder build --config electron-builder.config.js --win squirrel --prepackaged ./dist/win-unpacked",
     "start": "npx -y concurrently -n browser,main --kill-others \"npm run start:dev-server\" \"npm run start:electron\"",
     "start:dev-server": "vite dev",

packages/insomnia/scripts/bundle-plugins.ts

@@ -0,0 +1,180 @@
+import { execFile } from 'node:child_process';
+import { cp, mkdir, mkdtemp, readdir, rm, stat, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { promisify } from 'node:util';
+
+const bundlePlugins = ['@kenttest/plugin-external-vault'];
+const bundlePluginsDir = path.resolve(__dirname, '..', 'plugins');
+const yarnPath = path.resolve(__dirname, '..', 'bin', 'yarn-standalone.js');
+const execFilePromise = promisify(execFile);
+
+if (require.main === module) {
+  process.nextTick(async () => {
+    try {
+      // remove existing bundle plugins directory
+      await rm(bundlePluginsDir, { recursive: true, force: true });
+      // Recreate empty
+      await mkdir(bundlePluginsDir, { recursive: true });
+      for (const bundlePlugin of bundlePlugins) {
+        await installPlugin(bundlePlugin);
+      }
+    } catch (err) {
+      console.log('[bundle-plugin] ERROR:', err);
+      process.exit(1);
+    }
+  });
+}
+
+export async function runYarnCommand(args: string[], cwd?: string) {
+  const { stdout, stderr } = await execFilePromise(process.execPath, ['--no-deprecation', yarnPath, ...args], {
+    cwd,
+    env: {
+      NODE_ENV: 'production',
+    },
+    timeout: 5 * 60 * 1000, // 5 minutes
+    maxBuffer: 1024 * 1024, // 1MB buffer
+  });
+
+  if (stderr) {
+    throw new Error(`Yarn error: ${stderr}`);
+  }
+
+  return stdout.toString();
+}
+
+export default async function installPlugin(name: string) {
+  const { scope, pkg: pluginName } = parsePackageName(name);
+  const pluginDir = path.resolve(bundlePluginsDir, pluginName);
+  // Ensure the plugin directory exists
+  await mkdir(pluginDir, { recursive: true });
+  let tmpDir = '';
+
+  try {
+    // Install the plugin into a temporary directory
+    tmpDir = await installPluginToTmpDir(name);
+    console.log(`[plugins] Moving plugin from temp directory ${tmpDir} to final plugin directory ${pluginDir}`);
+
+    // Handle the plugin's dependencies
+    // Create a node_modules directory inside the plugin directory
+    const pluginModulesDir = path.resolve(pluginDir, 'node_modules');
+    await mkdir(pluginModulesDir, { recursive: true });
+
+    // Read all folders/files in the temp directory
+    const tmpFiles = await readdir(tmpDir);
+    console.log(`[plugins] Moving plugin from temp directory ${tmpDir} to final plugin directory ${pluginDir}`);
+
+    // Move the main plugin folder into the plugin directory
+    await cp(path.resolve(tmpDir, scope || '', pluginName), pluginDir, {
+      recursive: true,
+      verbatimSymlinks: true,
+    });
+
+    // Filter out the main plugin directory and non-directories
+    // and copy each directory to the plugin's node_modules directory
+    // Use Promise.all to copy all directories in parallel
+    const filtered = await Promise.all(
+      tmpFiles.map(async filename => {
+        const fullPath = path.resolve(tmpDir, filename);
+        const fileStat = await stat(fullPath);
+        return {
+          filename,
+          include: filename !== (scope || pluginName) && filename !== 'node_modules' && fileStat.isDirectory(),
+        };
+      }),
+    );
+
+    await Promise.all(
+      filtered
+        .filter(f => f.include)
+        .map(async ({ filename }) => {
+          const src = path.resolve(tmpDir, filename);
+          const dest = path.resolve(pluginModulesDir, filename);
+          await cp(src, dest, { recursive: true, verbatimSymlinks: true });
+        }),
+    );
+  } catch (error) {
+    console.error(`Failed to install plugin ${pluginName}: ${error.toString()}`);
+    throw error;
+  } finally {
+    // Ensure the temporary directory is cleaned up
+    if (tmpDir) {
+      try {
+        console.log(`[plugins] Cleaning up temporary directory: ${tmpDir}`);
+        await rm(tmpDir, { recursive: true, force: true });
+      } catch (error) {
+        console.warn(`[plugins] Failed to clean tmp dir ${tmpDir}: ${error.toString()}`);
+      }
+    }
+  }
+}
+
+/**
+ * Installs a plugin into a temporary directory using Yarn.
+ * Creates a minimal package.json and downloads the dependency.
+ */
+export async function installPluginToTmpDir(name: string) {
+  const { scope, pkg: pluginName } = parsePackageName(name);
+  try {
+    const tmpDir = await mkdtemp(path.resolve(tmpdir(), `${pluginName}-${Date.now()}`));
+
+    await writeFile(
+      path.resolve(tmpDir, 'package.json'),
+      JSON.stringify({ license: 'ISC', workspaces: [] }, null, 2),
+      'utf-8',
+    );
+
+    console.log(`[plugins] Installing plugin into temp dir: ${tmpDir}`);
+
+    await runYarnCommand(
+      [
+        'add',
+        name,
+        '--modules-folder',
+        tmpDir,
+        '--cwd',
+        tmpDir,
+        '--no-lockfile',
+        '--production',
+        '--no-progress',
+        '--ignore-workspace-root-check',
+        '--registry',
+        'https://registry.npmjs.org/',
+      ],
+      tmpDir,
+    );
+
+    // Check if the plugin was installed successfully
+    const pluginDir = path.resolve(tmpDir, scope || pluginName);
+    const pluginExists = await stat(pluginDir)
+      .then(() => true)
+      .catch(() => false);
+    if (!pluginExists) {
+      throw new Error(`Plugin "${name}" not found in temporary directory`);
+    }
+
+    console.log(`[plugins] Plugin installed successfully in temp dir: ${tmpDir}`);
+
+    // Check if the plugin has a package.json file
+    const packageJsonPath = path.resolve(tmpDir, scope || '', pluginName, 'package.json');
+    const packageJsonExists = await stat(packageJsonPath)
+      .then(() => true)
+      .catch(() => false);
+
+    if (!packageJsonExists) {
+      throw new Error(`Plugin "${name}" does not have a package.json file`);
+    }
+
+    return tmpDir;
+  } catch (err) {
+    throw new Error(`Failed to install plugin: ${(err as Error).message}`);
+  }
+}
+
+function parsePackageName(name: string) {
+  if (name.startsWith('@')) {
+    const [scope, pkg] = name.split('/');
+    return { scope, pkg };
+  }
+  return { scope: null, pkg: name };
+}

packages/insomnia/send-request/electron/index.js

@@ -3,6 +3,9 @@ module.exports = {
   app: {
     getPath: (/** @type {string} */ name) =>
       name === 'temp' ? require('os').tmpdir() : require('path').join(require('os').tmpdir(), 'insomnia-send-request'),
+    getAppPath: () => {
+      return require('node:path').resolve(__dirname, '../plugins');
+    },
   },
   ipcMain: {
     on: () => {

packages/insomnia/src/common/constants.ts

@@ -14,7 +14,13 @@ export const INSOMNIA_GITLAB_API_URL = env.INSOMNIA_GITLAB_API_URL;
 export const PLAYWRIGHT = env.PLAYWRIGHT;
 export const ENTERPRISE_PLUGINS: Record<string, any> = {
   'external-vault': {
-    pluginName: 'insomnia-plugin-external-vault',
+    pluginName: '@kenttest/plugin-external-vault',
+    pluginVersion: '0.0.1-beta1',
+    checksum: {
+      'dist/index.js':
+        'sha512-leeE7/gFrc899RXLgVbNLioproBC6NXjMJZQpnzYAsJ0DiehIhy6/9ghIwkQahqs5lkQxK8uJgAfabwUYJLHVQ==',
+      'package.json': 'sha512-liCGPClbqhiHMYhRLIKFpE02o8JTEMPzpWuoGC4YiflVzMBu7x0JzHz1eE2d/4D8ndCG4fXGUH6cRNJfXExEiQ==',
+    },
   },
 };
 // App Stuff

packages/insomnia/src/common/render.ts

@@ -15,7 +15,7 @@ import { isRequestGroup } from '../models/request-group';
 import type { WebSocketRequest } from '../models/websocket-request';
 import { isWorkspace, type Workspace } from '../models/workspace';
 import { getOrInheritAuthentication, getOrInheritHeaders } from '../network/network';
-import { isEnterprisePluginTemplateTag } from '../plugins';
+import { isPreBundlePluginTemplateTag } from '../plugins';
 import * as templating from '../templating';
 import { RenderError } from '../templating/render-error';
 import type {
@@ -296,7 +296,7 @@ export async function render<T>(
         const currentProcessIsRendererAndPluginsAreRestricted =
           process.type === 'renderer' && pluginsAreRestrictedToRunInWorker;
         const renderFork = async (renderInput: RenderInputType) => {
-          const inputIsEnterprisePluginTag = await isEnterprisePluginTemplateTag(input as string);
+          const inputIsEnterprisePluginTag = await isPreBundlePluginTemplateTag(input as string);
           return currentProcessIsRendererAndPluginsAreRestricted && !inputIsEnterprisePluginTag
             ? (await import('../ui/worker/templating-handler')).renderInWorker(renderInput)
             : renderInThisProcess(renderInput);

packages/insomnia/src/plugins/context/network.ts

@@ -73,7 +73,12 @@ export function init() {
         const settings = await models.settings.get();
         const settingFollowRedirects = settings?.followRedirects ? 'on' : 'off';
         const { request: originRequest, caCertficatePath = null } = options;
-        const response = await window.main.curlRequest({
+        const curlRequest =
+          process.type === 'renderer' || process.type === 'worker'
+            ? window.main.curlRequest
+            : // when exeucted in Inso;
+              (await import('../../main/network/libcurl-promise')).curlRequest;
+        const response = await curlRequest({
           requestId: `cloud-service-integration-${requestId}`,
           req: {
             authentication: {},