feat(exporter-otlp-*)!: support custom HTTP agents

.mocharc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  require: `${__dirname}/ts-node.js`,
+};

api/package.json

@@ -34,7 +34,7 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "lint": "eslint . --ext .ts",
     "test:browser": "karma start --single-run",
-    "test": "nyc mocha 'test/**/*.test.ts'",
+    "test": "cross-env MOCHA_DONT_PATCH_ESM=true nyc mocha 'test/**/*.test.ts'",
     "test:webworker": "karma start karma.worker.js --single-run",
     "cycle-check": "dpdm --exit-code circular:1 src/index.ts",
     "version": "node ../scripts/version-update.js",
@@ -80,6 +80,7 @@
     "@types/webpack-env": "1.16.3",
     "babel-plugin-istanbul": "7.0.0",
     "cross-var": "1.1.0",
+    "cross-env": "6.0.3",
     "dpdm": "3.13.1",
     "karma": "6.4.4",
     "karma-chrome-launcher": "3.1.0",

experimental/CHANGELOG.md

@@ -10,6 +10,8 @@ For notes on migrating to 2.x / 0.200.x see [the upgrade guide](doc/upgrade-to-2
 
 ### :rocket: Features
 
+* feat(exporter-otlp-\*): support custom HTTP agents [#5719](https://github.com/open-telemetry/opentelemetry-js/pull/5719) @raphael-theriault-swi
+
 ### :bug: Bug Fixes
 
 ### :books: Documentation

experimental/packages/opentelemetry-instrumentation/package.json

@@ -48,7 +48,7 @@
     "tdd:node": "npm run test -- --watch-extensions ts --watch",
     "tdd:browser": "karma start",
     "test:cjs": "nyc mocha 'test/**/*.test.ts' --exclude 'test/browser/**/*.ts'",
-    "test:esm": "nyc node --experimental-loader=./hook.mjs ../../../node_modules/mocha/bin/mocha 'test/node/*.test.mjs'",
+    "test:esm": "cross-env MOCHA_DONT_PATCH_ESM=true nyc node --experimental-loader=./hook.mjs ../../../node_modules/mocha/bin/mocha 'test/node/*.test.mjs'",
     "test": "npm run test:cjs && npm run test:esm",
     "test:browser": "karma start --single-run",
     "version": "node ../../../scripts/version-update.js",
@@ -91,6 +91,7 @@
     "babel-plugin-istanbul": "7.0.0",
     "codecov": "3.8.3",
     "cross-var": "1.1.0",
+    "cross-env": "6.0.3",
     "karma": "6.4.4",
     "karma-chrome-launcher": "3.1.0",
     "karma-coverage": "2.2.1",

experimental/packages/otlp-exporter-base/src/configuration/convert-legacy-node-http-options.ts

@@ -16,18 +16,22 @@
 import { OTLPExporterNodeConfigBase } from './legacy-node-configuration';
 import {
   getHttpConfigurationDefaults,
+  HttpAgentFactory,
+  httpAgentFactoryFromOptions,
   mergeOtlpHttpConfigurationWithDefaults,
   OtlpHttpConfiguration,
 } from './otlp-http-configuration';
 import { getHttpConfigurationFromEnvironment } from './otlp-http-env-configuration';
-import type * as http from 'http';
-import type * as https from 'https';
 import { diag } from '@opentelemetry/api';
 import { wrapStaticHeadersInFunction } from './shared-configuration';
 
 function convertLegacyAgentOptions(
   config: OTLPExporterNodeConfigBase
-): http.AgentOptions | https.AgentOptions | undefined {
+): HttpAgentFactory | undefined {
+  if (typeof config.httpAgentOptions === 'function') {
+    return config.httpAgentOptions;
+  }
+
   // populate keepAlive for use with new settings
   if (config?.keepAlive != null) {
     if (config.httpAgentOptions != null) {
@@ -44,7 +48,11 @@ function convertLegacyAgentOptions(
     }
   }
 
-  return config.httpAgentOptions;
+  if (config.httpAgentOptions != null) {
+    return httpAgentFactoryFromOptions(config.httpAgentOptions);
+  } else {
+    return undefined;
+  }
 }
 
 /**
@@ -72,7 +80,7 @@ export function convertLegacyHttpOptions(
       concurrencyLimit: config.concurrencyLimit,
       timeoutMillis: config.timeoutMillis,
       compression: config.compression,
-      agentOptions: convertLegacyAgentOptions(config),
+      agent: convertLegacyAgentOptions(config),
     },
     getHttpConfigurationFromEnvironment(signalIdentifier, signalResourcePath),
     getHttpConfigurationDefaults(requiredHeaders, signalResourcePath)

experimental/packages/otlp-exporter-base/src/configuration/legacy-node-configuration.ts

@@ -18,15 +18,16 @@
 import type * as http from 'http';
 import type * as https from 'https';
 
-import { OTLPExporterConfigBase } from './legacy-base-configuration';
+import type { OTLPExporterConfigBase } from './legacy-base-configuration';
+import type { HttpAgentFactory } from './otlp-http-configuration';
 
 /**
  * Collector Exporter node base config
  */
 export interface OTLPExporterNodeConfigBase extends OTLPExporterConfigBase {
   keepAlive?: boolean;
   compression?: CompressionAlgorithm;
-  httpAgentOptions?: http.AgentOptions | https.AgentOptions;
+  httpAgentOptions?: http.AgentOptions | https.AgentOptions | HttpAgentFactory;
 }
 
 export enum CompressionAlgorithm {

experimental/packages/otlp-exporter-base/src/configuration/otlp-http-configuration.ts

@@ -25,10 +25,16 @@ import { validateAndNormalizeHeaders } from '../util';
 import type * as http from 'http';
 import type * as https from 'https';
 
+export type HttpAgentFactory =
+  | ((protocol: string) => http.Agent)
+  | ((protocol: string) => https.Agent)
+  | ((protocol: string) => Promise<http.Agent>)
+  | ((protocol: string) => Promise<https.Agent>);
+
 export interface OtlpHttpConfiguration extends OtlpSharedConfiguration {
   url: string;
   headers: () => Record<string, string>;
-  agentOptions: http.AgentOptions | https.AgentOptions;
+  agent: HttpAgentFactory;
 }
 
 function mergeHeaders(
@@ -71,6 +77,16 @@ function validateUserProvidedUrl(url: string | undefined): string | undefined {
   }
 }
 
+export function httpAgentFactoryFromOptions(
+  options: http.AgentOptions | https.AgentOptions
+): HttpAgentFactory {
+  return async protocol => {
+    const { Agent } =
+      protocol === 'http:' ? await import('http') : await import('https');
+    return new Agent(options);
+  };
+}
+
 /**
  * @param userProvidedConfiguration  Configuration options provided by the user in code.
  * @param fallbackConfiguration Fallback to use when the {@link userProvidedConfiguration} does not specify an option.
@@ -96,10 +112,10 @@ export function mergeOtlpHttpConfigurationWithDefaults(
       validateUserProvidedUrl(userProvidedConfiguration.url) ??
       fallbackConfiguration.url ??
       defaultConfiguration.url,
-    agentOptions:
-      userProvidedConfiguration.agentOptions ??
-      fallbackConfiguration.agentOptions ??
-      defaultConfiguration.agentOptions,
+    agent:
+      userProvidedConfiguration.agent ??
+      fallbackConfiguration.agent ??
+      defaultConfiguration.agent,
   };
 }
 
@@ -111,6 +127,6 @@ export function getHttpConfigurationDefaults(
     ...getSharedConfigurationDefaults(),
     headers: () => requiredHeaders,
     url: 'http://localhost:4318/' + signalResourcePath,
-    agentOptions: { keepAlive: true },
+    agent: httpAgentFactoryFromOptions({ keepAlive: true }),
   };
 }

experimental/packages/otlp-exporter-base/src/index-node-http.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+export { httpAgentFactoryFromOptions } from './configuration/otlp-http-configuration';
 export { createOtlpHttpExportDelegate } from './otlp-http-export-delegate';
 export { getSharedConfigurationFromEnvironment } from './configuration/shared-env-configuration';
 export { convertLegacyHttpOptions } from './configuration/convert-legacy-node-http-options';

experimental/packages/otlp-exporter-base/src/transport/http-exporter-transport.ts

@@ -37,7 +37,7 @@ class HttpExporterTransport implements IExporterTransport {
   constructor(private _parameters: HttpRequestParameters) {}
 
   async send(data: Uint8Array, timeoutMillis: number): Promise<ExportResponse> {
-    const { agent, send } = this._loadUtils();
+    const { agent, send } = await this._loadUtils();
 
     return new Promise<ExportResponse>(resolve => {
       send(
@@ -56,23 +56,20 @@ class HttpExporterTransport implements IExporterTransport {
     // intentionally left empty, nothing to do.
   }
 
-  private _loadUtils(): Utils {
+  private async _loadUtils(): Promise<Utils> {
     let utils = this._utils;
 
     if (utils === null) {
-      // Lazy require to ensure that http/https is not required before instrumentations can wrap it.
-      const {
-        sendWithHttp,
-        createHttpAgent,
-        // eslint-disable-next-line @typescript-eslint/no-require-imports
-      } = require('./http-transport-utils');
+      // Lazy import to ensure that http/https is not required before instrumentations can wrap it.
+      const imported = await import('./http-transport-utils.js');
 
       utils = this._utils = {
-        agent: createHttpAgent(
-          this._parameters.url,
-          this._parameters.agentOptions
+        agent: await this._parameters.agent(
+          new URL(this._parameters.url).protocol
         ),
-        send: sendWithHttp,
+        // @ts-expect-error dynamic imports are never transpiled, but named exports only work when
+        // dynamically importing an esm file, and the utils file might be transpiled to cjs
+        send: imported.sendWithHttp ?? imported.default.sendWithHttp,
       };
     }
 

experimental/packages/otlp-exporter-base/src/transport/http-transport-types.ts

@@ -17,6 +17,7 @@
 import type * as http from 'http';
 import type * as https from 'https';
 import { ExportResponse } from '../export-response';
+import { HttpAgentFactory } from '../configuration/otlp-http-configuration';
 
 export type sendWithHttp = (
   params: HttpRequestParameters,
@@ -30,5 +31,5 @@ export interface HttpRequestParameters {
   url: string;
   headers: () => Record<string, string>;
   compression: 'gzip' | 'none';
-  agentOptions: http.AgentOptions | https.AgentOptions;
+  agent: HttpAgentFactory;
 }

experimental/packages/otlp-exporter-base/src/transport/http-transport-utils.ts

@@ -141,12 +141,3 @@ function readableFromUint8Array(buff: string | Uint8Array): Readable {
 
   return readable;
 }
-
-export function createHttpAgent(
-  rawUrl: string,
-  agentOptions: http.AgentOptions | https.AgentOptions
-) {
-  const parsedUrl = new URL(rawUrl);
-  const Agent = parsedUrl.protocol === 'http:' ? http.Agent : https.Agent;
-  return new Agent(agentOptions);
-}

experimental/packages/otlp-exporter-base/test/common/configuration/otlp-http-configuration.test.ts

@@ -28,7 +28,7 @@ describe('mergeOtlpHttpConfigurationWithDefaults', function () {
     compression: 'none',
     concurrencyLimit: 2,
     headers: () => ({ 'User-Agent': 'default-user-agent' }),
-    agentOptions: { keepAlive: true },
+    agent: () => null!,
   };
 
   describe('headers', function () {

experimental/packages/otlp-exporter-base/test/node/configuration/convert-legacy-node-otlp-http-options.test.ts

@@ -16,6 +16,7 @@
 
 import * as sinon from 'sinon';
 import * as assert from 'assert';
+import type * as https from 'https';
 import { convertLegacyHttpOptions } from '../../../src/configuration/convert-legacy-node-http-options';
 import { registerMockDiagLogger } from '../../common/test-utils';
 
@@ -40,7 +41,21 @@ describe('convertLegacyHttpOptions', function () {
     );
   });
 
-  it('should keep specific keepAlive', () => {
+  it('should keep agent factory as-is', function () {
+    // act
+    const factory = () => null!;
+    const options = convertLegacyHttpOptions(
+      { httpAgentOptions: factory },
+      'SIGNAL',
+      'v1/signal',
+      {}
+    );
+
+    // assert
+    assert.strictEqual(options.agent, factory);
+  });
+
+  it('should keep specific keepAlive', async () => {
     // act
     const options = convertLegacyHttpOptions(
       {
@@ -50,12 +65,13 @@ describe('convertLegacyHttpOptions', function () {
       'v1/signal',
       {}
     );
+    const agent = (await options.agent('https:')) as https.Agent;
 
     // assert
-    assert.ok(options.agentOptions.keepAlive);
+    assert.ok(agent.options.keepAlive);
   });
 
-  it('should set keepAlive on AgentOptions when not explicitly set in AgentOptions but set in config', () => {
+  it('should set keepAlive on AgentOptions when not explicitly set in AgentOptions but set in config', async () => {
     // act
     const options = convertLegacyHttpOptions(
       {
@@ -69,9 +85,10 @@ describe('convertLegacyHttpOptions', function () {
       'v1/signal',
       {}
     );
+    const agent = (await options.agent('https:')) as https.Agent;
 
     // assert
-    assert.ok(options.agentOptions.keepAlive);
-    assert.strictEqual(options.agentOptions.port, 1234);
+    assert.ok(agent.options.keepAlive);
+    assert.strictEqual(agent.options.port, 1234);
   });
 });