Added logic for Url redaction

CHANGELOG.md

@@ -10,6 +10,8 @@ For notes on migrating to 2.x / 0.200.x see [the upgrade guide](doc/upgrade-to-2
 
 ## Unreleased
 
+* feat(instrumentation-http): Added support for redacting specific url query string values and url credentials in instrumentations [#5743](https://github.com/open-telemetry/opentelemetry-js/pull/5743) @rads-1996
+
 ### :boom: Breaking Changes
 
 ### :rocket: Features

experimental/packages/opentelemetry-instrumentation-http/src/http.ts

@@ -781,6 +781,7 @@ export class HttpInstrumentation extends InstrumentationBase<HttpInstrumentation
             optionsParsed,
             instrumentation.getConfig().startOutgoingSpanHook
           ),
+          redactedQueryParams: instrumentation.getConfig().redactedQueryParams, // Added config for adding custom query strings
         },
         instrumentation._semconvStability,
         instrumentation.getConfig().enableSyntheticSourceDetection || false

experimental/packages/opentelemetry-instrumentation-http/src/internal-types.ts

@@ -57,3 +57,18 @@ export const SYNTHETIC_TEST_NAMES = ['alwayson'];
  * Names of possible synthetic bot sources.
  */
 export const SYNTHETIC_BOT_NAMES = ['googlebot', 'bingbot'];
+
+/**
+ * REDACTED string used to replace sensitive information in URLs.
+ */
+export const STR_REDACTED = 'REDACTED';
+
+/**
+ * List of URL query keys that are considered sensitive and whose value should be redacted.
+ */
+export const DEFAULT_QUERY_STRINGS_TO_REDACT = [
+  'sig',
+  'Signature',
+  'AWSAccessKeyId',
+  'X-Goog-Signature',
+] as const;

experimental/packages/opentelemetry-instrumentation-http/src/types.ts

@@ -92,4 +92,12 @@ export interface HttpInstrumentationConfig extends InstrumentationConfig {
    * @experimental
    **/
   enableSyntheticSourceDetection?: boolean;
+  /**
+   * [Optional] Additional query parameters to redact.
+   * Use this to specify custom query strings that contain sensitive information.
+   * These will replace/overwrite the default query strings that are to be redacted.
+   * @example default strings ['sig','Signature','AWSAccessKeyId','X-Goog-Signature']
+   * @experimental
+   */
+  redactedQueryParams?: string[];
 }

experimental/packages/opentelemetry-instrumentation-http/src/utils.ts

@@ -82,6 +82,10 @@ import * as url from 'url';
 import { AttributeNames } from './enums/AttributeNames';
 import { Err, IgnoreMatcher, ParsedRequestOptions } from './internal-types';
 import { SYNTHETIC_BOT_NAMES, SYNTHETIC_TEST_NAMES } from './internal-types';
+import {
+  DEFAULT_QUERY_STRINGS_TO_REDACT,
+  STR_REDACTED,
+} from './internal-types';
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 import forwardedParse = require('forwarded-parse');
 
@@ -91,15 +95,15 @@ import forwardedParse = require('forwarded-parse');
 export const getAbsoluteUrl = (
   requestUrl: ParsedRequestOptions | null,
   headers: IncomingHttpHeaders | OutgoingHttpHeaders,
-  fallbackProtocol = 'http:'
+  fallbackProtocol = 'http:',
+  redactedQueryParams: string[] = Array.from(DEFAULT_QUERY_STRINGS_TO_REDACT)
 ): string => {
   const reqUrlObject = requestUrl || {};
   const protocol = reqUrlObject.protocol || fallbackProtocol;
   const port = (reqUrlObject.port || '').toString();
-  const path = reqUrlObject.path || '/';
+  let path = reqUrlObject.path || '/';
   let host =
     reqUrlObject.host || reqUrlObject.hostname || headers.host || 'localhost';
-
   // if there is no port in host and there is a port
   // it should be displayed if it's not 80 and 443 (default ports)
   if (
@@ -110,8 +114,29 @@ export const getAbsoluteUrl = (
   ) {
     host += `:${port}`;
   }
+  // Redact sensitive query parameters
+  if (path.includes('?')) {
+    //const [pathname, query] = path.split('?', 2);
+    const parsedUrl = url.parse(path);
+    const pathname = parsedUrl.pathname || '';
+    const query = parsedUrl.query || '';
+    const searchParams = new URLSearchParams(query);
+    const sensitiveParamsToRedact: string[] = redactedQueryParams || [];
+
+    for (const sensitiveParam of sensitiveParamsToRedact) {
+      if (
+        searchParams.has(sensitiveParam) &&
+        searchParams.get(sensitiveParam) !== ''
+      ) {
+        searchParams.set(sensitiveParam, STR_REDACTED);
+      }
+    }
 
-  return `${protocol}//${host}${path}`;
+    const redactedQuery = searchParams.toString();
+    path = `${pathname}?${redactedQuery}`;
+  }
+  const authPart = reqUrlObject.auth ? `${STR_REDACTED}:${STR_REDACTED}@` : '';
+  return `${protocol}//${authPart}${host}${path}`;
 };
 
 /**
@@ -442,6 +467,7 @@ export const getOutgoingRequestAttributes = (
     hostname: string;
     port: string | number;
     hookAttributes?: Attributes;
+    redactedQueryParams?: string[];
   },
   semconvStability: SemconvStability,
   enableSyntheticSourceDetection: boolean
@@ -455,7 +481,8 @@ export const getOutgoingRequestAttributes = (
   const urlFull = getAbsoluteUrl(
     requestOptions,
     headers,
-    `${options.component}:`
+    `${options.component}:`,
+    options.redactedQueryParams
   );
 
   const oldAttributes: Attributes = {

experimental/packages/opentelemetry-instrumentation-http/test/functionals/http-enable.test.ts

@@ -243,7 +243,7 @@ describe('HttpInstrumentation', () => {
         );
       });
 
-      it('should remove auth from the `http.url` attribute (client side and server side)', async () => {
+      it('should redact auth from the `http.url` attribute (client side and server side)', async () => {
         await httpRequest.get(
           `${protocol}://user:pass@${hostname}:${serverPort}${pathname}`
         );
@@ -258,7 +258,7 @@ describe('HttpInstrumentation', () => {
         );
         assert.strictEqual(
           outgoingSpan.attributes[ATTR_HTTP_URL],
-          `${protocol}://${hostname}:${serverPort}${pathname}`
+          `${protocol}://REDACTED:REDACTED@${hostname}:${serverPort}${pathname}`
         );
       });
     });
@@ -1162,7 +1162,7 @@ describe('HttpInstrumentation', () => {
         });
       });
 
-      it('should remove auth from the `url.full` attribute (client side and server side)', async () => {
+      it('should redact auth from the `url.full` attribute (client side and server side)', async () => {
         await httpRequest.get(
           `${protocol}://user:pass@${hostname}:${serverPort}${pathname}`
         );
@@ -1172,7 +1172,7 @@ describe('HttpInstrumentation', () => {
         assert.strictEqual(outgoingSpan.kind, SpanKind.CLIENT);
         assert.strictEqual(
           outgoingSpan.attributes[ATTR_URL_FULL],
-          `${protocol}://${hostname}:${serverPort}${pathname}`
+          `${protocol}://REDACTED:REDACTED@${hostname}:${serverPort}${pathname}`
         );
       });
 
@@ -1598,4 +1598,220 @@ describe('HttpInstrumentation', () => {
       );
     });
   });
+  describe('URL Redaction', () => {
+    beforeEach(() => {
+      memoryExporter.reset();
+    });
+
+    before(async () => {
+      instrumentation.setConfig({});
+      instrumentation.enable();
+      server = http.createServer((request, response) => {
+        response.end('Test Server Response');
+      });
+      await new Promise<void>(resolve => server.listen(serverPort, resolve));
+    });
+
+    after(() => {
+      server.close();
+      instrumentation.disable();
+    });
+
+    it('should redact authentication credentials from URLs', async () => {
+      await httpRequest.get(
+        `${protocol}://user:password@${hostname}:${serverPort}${pathname}`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [incomingSpan, outgoingSpan] = spans;
+
+      assert.strictEqual(spans.length, 2);
+      assert.strictEqual(incomingSpan.kind, SpanKind.SERVER);
+      assert.strictEqual(outgoingSpan.kind, SpanKind.CLIENT);
+
+      // Server shouldn't see auth in URL
+      assert.strictEqual(
+        incomingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}`
+      );
+
+      // Client should have redacted auth
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://REDACTED:REDACTED@${hostname}:${serverPort}${pathname}`
+      );
+    });
+    it('should redact default query strings', async () => {
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?X-Goog-Signature=xyz789&normal=value`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?X-Goog-Signature=REDACTED&normal=value`
+      );
+    });
+
+    it('should handle both auth credentials and sensitive default query parameters', async () => {
+      await httpRequest.get(
+        `${protocol}://username:password@${hostname}:${serverPort}${pathname}?AWSAccessKeyId=secret`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://REDACTED:REDACTED@${hostname}:${serverPort}${pathname}?AWSAccessKeyId=REDACTED`
+      );
+    });
+    it('should handle URLs with special characters in auth and query', async () => {
+      await httpRequest.get(
+        `${protocol}://user%40domain:p%40ssword@${hostname}:${serverPort}${pathname}?sig=abc%3Ddef`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://REDACTED:REDACTED@${hostname}:${serverPort}${pathname}?sig=REDACTED`
+      );
+    });
+
+    it('should handle malformed query strings', async () => {
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?X-Goog-Signature=value&=nokey&malformed=`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?X-Goog-Signature=REDACTED&=nokey&malformed=`
+      );
+    });
+    it('should not modify URLs without auth or sensitive query parameters', async () => {
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?param=value&another=123`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?param=value&another=123`
+      );
+    });
+
+    it('should not modify URLs with no query string', async () => {
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}`
+      );
+    });
+
+    it('should not modify URLs with empty query parameters', async () => {
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?sig=&empty=`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?sig=&empty=`
+      );
+    });
+
+    it('should preserve non-sensitive query parameters when sensitive ones are redacted', async () => {
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?normal=value&Signature=secret&other=data`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?normal=value&Signature=REDACTED&other=data`
+      );
+    });
+    it('should redact only custom query parameters when user provides a populated config', async () => {
+      // Set additional parameters while keeping the default ones
+      instrumentation.setConfig({
+        redactedQueryParams: ['authorize', 'session_id'],
+      });
+
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?sig=abc123&authorize=xyz789&normal=value`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?sig=abc123&authorize=REDACTED&normal=value`
+      );
+    });
+    it('should not redact query strings when redactedQueryParams is empty', async () => {
+      instrumentation.setConfig({
+        redactedQueryParams: [],
+      });
+
+      // URL with both default sensitive params and custom ones
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?X-Goog-Signature=secret&api_key=12345&normal=value`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?X-Goog-Signature=secret&api_key=12345&normal=value`
+      );
+    });
+    it('should handle case-sensitive query parameter names correctly', async () => {
+      instrumentation.setConfig({
+        redactedQueryParams: ['TOKEN'],
+      });
+
+      await httpRequest.get(
+        `${protocol}://${hostname}:${serverPort}${pathname}?token=lowercase&TOKEN=uppercase&sig=secret`
+      );
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      // This tests whether parameter name matching is case-sensitive or case-insensitive
+      assert.strictEqual(
+        outgoingSpan.attributes[ATTR_HTTP_URL],
+        `${protocol}://${hostname}:${serverPort}${pathname}?token=lowercase&TOKEN=REDACTED&sig=secret`
+      );
+    });
+    it('should handle very complex URLs with multiple redaction points and if custom query strings are provided only redact those', async () => {
+      instrumentation.setConfig({
+        redactedQueryParams: ['api_key', 'token'],
+      });
+
+      const complexUrl =
+        `${protocol}://user:pass@${hostname}:${serverPort}${pathname}?` +
+        'sig=abc123&api_key=secret&normal=value&Signature=xyz&' +
+        'token=sensitive&X-Goog-Signature=gcp&AWSAccessKeyId=aws';
+
+      await httpRequest.get(complexUrl);
+      const spans = memoryExporter.getFinishedSpans();
+      const [_, outgoingSpan] = spans;
+
+      const expectedUrl =
+        `${protocol}://REDACTED:REDACTED@${hostname}:${serverPort}${pathname}?` +
+        'sig=abc123&api_key=REDACTED&normal=value&Signature=xyz&' +
+        'token=REDACTED&X-Goog-Signature=gcp&AWSAccessKeyId=aws';
+
+      assert.strictEqual(outgoingSpan.attributes[ATTR_HTTP_URL], expectedUrl);
+    });
+  });
 });