feat(p2p): optional P2P_BROADCAST_PORT (#13525)

fixes: #13165

Author: Maddiaa0

aztec-up/bin/aztec

@@ -17,6 +17,10 @@ while [ "$#" -gt 0 ]; do
       P2P_PORT="$2"
       shift 2
       ;;
+    --p2p.p2pBroadcastPort)
+      P2P_BROADCAST_PORT="$2"
+      shift 2
+      ;;
     -a | --anvil-port)
       # Override default port exposed on container.
       ANVIL_PORT="$2"
@@ -93,11 +97,21 @@ case ${1:-} in
         node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --sandbox
       "
     else
+
       export P2P_PORT="${P2P_PORT:-40400}"
-      P2P_TCP_LISTEN_MAPPING="$P2P_PORT:$P2P_PORT"
-      P2P_UDP_LISTEN_MAPPING="${P2P_TCP_LISTEN_MAPPING}/udp"
+      # If the p2p broadcast port if provided, then map it to the p2p port on the container.
+      if [ -n "${P2P_BROADCAST_PORT:-}" ]; then
+        export P2P_BROADCAST_PORT
+        P2P_TCP_BROADCAST_MAPPING="$P2P_BROADCAST_PORT:$P2P_PORT"
+        P2P_UDP_BROADCAST_MAPPING="${P2P_TCP_BROADCAST_MAPPING}/udp"
+
+        PORTS_TO_EXPOSE="${PORTS_TO_EXPOSE:-} $P2P_TCP_BROADCAST_MAPPING $P2P_UDP_BROADCAST_MAPPING"
+      else
+        P2P_TCP_LISTEN_MAPPING="$P2P_PORT:$P2P_PORT"
+        P2P_UDP_LISTEN_MAPPING="${P2P_TCP_LISTEN_MAPPING}/udp"
 
-      PORTS_TO_EXPOSE="${PORTS_TO_EXPOSE:-} $P2P_TCP_LISTEN_MAPPING $P2P_UDP_LISTEN_MAPPING"
+        PORTS_TO_EXPOSE="${PORTS_TO_EXPOSE:-} $P2P_TCP_LISTEN_MAPPING $P2P_UDP_LISTEN_MAPPING"
+      fi
 
       exec $(dirname $0)/.aztec-run aztec-start \
         node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js "$@"

yarn-project/foundation/src/config/env_var.ts

@@ -112,6 +112,7 @@ export type EnvVar =
   | 'P2P_DOUBLE_SPEND_SEVERE_PEER_PENALTY_WINDOW'
   | 'P2P_LISTEN_ADDR'
   | 'P2P_PORT'
+  | 'P2P_BROADCAST_PORT'
   | 'P2P_IP'
   | 'P2P_TX_POOL_KEEP_PROVEN_FOR'
   | 'P2P_ATTESTATION_POOL_KEEP_FOR'

yarn-project/p2p/src/bootstrap/bootstrap.ts

@@ -31,19 +31,23 @@ export class BootstrapNode implements P2PBootstrapApi {
    * @returns An empty promise.
    */
   public async start(config: BootnodeConfig) {
-    const { p2pIp, p2pPort, listenAddress } = config;
-    const listenAddrUdp = multiaddr(convertToMultiaddr(listenAddress, p2pPort, 'udp'));
-
+    const { p2pIp, p2pPort, listenAddress, p2pBroadcastPort } = config;
     if (!p2pIp) {
       throw new Error('You need to provide a P2P IP address.');
     }
 
+    if (!p2pBroadcastPort) {
+      config.p2pBroadcastPort = p2pPort;
+    }
+
+    const listenAddrUdp = multiaddr(convertToMultiaddr(listenAddress, config.p2pBroadcastPort!, 'udp'));
+
     const peerIdPrivateKey = await getPeerIdPrivateKey(config, this.store, this.logger);
 
     const { enr: ourEnr, peerId } = await createBootnodeENRandPeerId(
       peerIdPrivateKey,
       p2pIp,
-      p2pPort,
+      config.p2pBroadcastPort!,
       config.l1ChainId,
     );
     this.peerId = peerId;

yarn-project/p2p/src/config.ts

@@ -53,6 +53,11 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
    */
   p2pPort: number;
 
+  /**
+   * The port to broadcast the P2P service on (included in the node's ENR).
+   */
+  p2pBroadcastPort?: number;
+
   /**
    * The IP address for the P2P service.
    */
@@ -194,6 +199,8 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
   maxTxPoolSize: number;
 }
 
+export const DEFAULT_P2P_PORT = 40400;
+
 export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
   p2pEnabled: {
     env: 'P2P_ENABLED',
@@ -227,8 +234,12 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
   },
   p2pPort: {
     env: 'P2P_PORT',
-    description: 'The port for the P2P service.',
-    ...numberConfigHelper(40400),
+    description: `The port for the P2P service. Defaults to ${DEFAULT_P2P_PORT}`,
+    ...numberConfigHelper(DEFAULT_P2P_PORT),
+  },
+  p2pBroadcastPort: {
+    env: 'P2P_BROADCAST_PORT',
+    description: `The port to broadcast the P2P service on (included in the node's ENR). Defaults to P2P_PORT.`,
   },
   p2pIp: {
     env: 'P2P_IP',
@@ -410,7 +421,13 @@ export function getP2PDefaultConfig(): P2PConfig {
  */
 export type BootnodeConfig = Pick<
   P2PConfig,
-  'p2pIp' | 'p2pPort' | 'peerIdPrivateKey' | 'peerIdPrivateKeyPath' | 'bootstrapNodes' | 'listenAddress'
+  | 'p2pIp'
+  | 'p2pPort'
+  | 'p2pBroadcastPort'
+  | 'peerIdPrivateKey'
+  | 'peerIdPrivateKeyPath'
+  | 'bootstrapNodes'
+  | 'listenAddress'
 > &
   Required<Pick<P2PConfig, 'p2pIp' | 'p2pPort'>> &
   Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKB'> &
@@ -419,6 +436,7 @@ export type BootnodeConfig = Pick<
 const bootnodeConfigKeys: (keyof BootnodeConfig)[] = [
   'p2pIp',
   'p2pPort',
+  'p2pBroadcastPort',
   'listenAddress',
   'peerIdPrivateKey',
   'peerIdPrivateKeyPath',

yarn-project/p2p/src/enr/generate-enr.ts

@@ -12,12 +12,12 @@ import { setAztecEnrKey } from '../versioning.js';
 export async function createBootnodeENRandPeerId(
   privateKey: string,
   p2pIp: string,
-  p2pPort: number,
+  p2pBroadcastPort: number,
   l1ChainId: number,
 ): Promise<{ enr: SignableENR; peerId: PeerId }> {
   const peerId = await createLibP2PPeerIdFromPrivateKey(privateKey);
   const enr = SignableENR.createFromPeerId(peerId);
-  const publicAddr = multiaddr(convertToMultiaddr(p2pIp, p2pPort, 'udp'));
+  const publicAddr = multiaddr(convertToMultiaddr(p2pIp, p2pBroadcastPort, 'udp'));
   enr.setLocationMultiaddr(publicAddr);
 
   const config: ChainConfig = {

yarn-project/p2p/src/services/discv5/discV5_service.ts

@@ -52,7 +52,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
     configOverrides: Partial<IDiscv5CreateOptions> = {},
   ) {
     super();
-    const { p2pIp, p2pPort, bootstrapNodes, trustedPeers, privatePeers } = config;
+    const { p2pIp, p2pPort, p2pBroadcastPort, bootstrapNodes, trustedPeers, privatePeers } = config;
     this.bootstrapNodeEnrs = bootstrapNodes.map(x => ENR.decodeTxt(x));
     const privatePeerEnrs = new Set(privatePeers);
     this.trustedPeerEnrs = trustedPeers.filter(x => !privatePeerEnrs.has(x)).map(x => ENR.decodeTxt(x));
@@ -61,14 +61,22 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
     // Add aztec identification to ENR
     this.versions = setAztecEnrKey(this.enr, config);
 
+    // If no overridden broadcast port is provided, use the p2p port as the broadcast port
+    if (!p2pBroadcastPort) {
+      config.p2pBroadcastPort = p2pPort;
+    }
+
     const bindAddrs: any = {
       ip4: multiaddr(convertToMultiaddr(config.listenAddress, p2pPort, 'udp')),
     };
 
     if (p2pIp) {
-      const multiAddrTcp = multiaddr(`${convertToMultiaddr(p2pIp!, p2pPort, 'tcp')}/p2p/${peerId.toString()}`);
-      // if no udp announce address is provided, use the tcp announce address
-      const multiAddrUdp = multiaddr(`${convertToMultiaddr(p2pIp!, p2pPort, 'udp')}/p2p/${peerId.toString()}`);
+      const multiAddrTcp = multiaddr(
+        `${convertToMultiaddr(p2pIp!, config.p2pBroadcastPort!, 'tcp')}/p2p/${peerId.toString()}`,
+      );
+      const multiAddrUdp = multiaddr(
+        `${convertToMultiaddr(p2pIp!, config.p2pBroadcastPort!, 'udp')}/p2p/${peerId.toString()}`,
+      );
 
       // set location multiaddr in ENR record
       this.enr.setLocationMultiaddr(multiAddrUdp);
@@ -116,7 +124,8 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
 
   private onMultiaddrUpdated(m: Multiaddr) {
     // We want to update our tcp port to match the udp port
-    const multiAddrTcp = multiaddr(convertToMultiaddr(m.nodeAddress().address, this.config.p2pPort, 'tcp'));
+    // p2pBroadcastPort is optional on config, however it is set to default within the p2p client factory
+    const multiAddrTcp = multiaddr(convertToMultiaddr(m.nodeAddress().address, this.config.p2pBroadcastPort!, 'tcp'));
     this.enr.setLocationMultiaddr(multiAddrTcp);
     this.logger.info('Multiaddr updated', { multiaddr: multiAddrTcp.toString() });
   }

yarn-project/p2p/src/services/discv5/discv5_service.test.ts

@@ -81,6 +81,15 @@ describe('Discv5Service', () => {
     await node.stop();
   });
 
+  it('should allow broadcast port to be set', async () => {
+    const broadcastPort = 7891;
+    const node = await createNode({ p2pBroadcastPort: broadcastPort });
+    const enr = node.getEnr();
+    expect(enr.ip).toEqual('127.0.0.1');
+    expect(enr.udp).toEqual(broadcastPort);
+    expect(enr.tcp).toEqual(broadcastPort);
+  });
+
   it('should discover & add a peer', async () => {
     const node1 = await createNode();
     const node2 = await createNode();

yarn-project/p2p/src/util.ts

@@ -80,7 +80,12 @@ export async function configureP2PClientAddresses(
   _config: P2PConfig & DataStoreConfig,
 ): Promise<P2PConfig & DataStoreConfig> {
   const config = { ..._config };
-  const { p2pIp, queryForIp } = config;
+  const { p2pIp, queryForIp, p2pBroadcastPort, p2pPort } = config;
+
+  // If no broadcast port is provided, use the given p2p port as the broadcast port
+  if (!p2pBroadcastPort) {
+    config.p2pBroadcastPort = p2pPort;
+  }
 
   // check if no announce IP was provided
   if (!p2pIp) {