Allow integrated relay chain light client (#2270)

* Add embedded light client to cli

* Prepare for light-client-worker

* First working version

* Clean up

* Remove unwanted logs

* Simplify subscription code

* Let jsonrpsee handle rpc management

* Simplify implementation

* Reorganize crate structure

* Use relay chain arg chainspec for light-client

* Clean up command line

* Add light client worker file

* Use smoldot master to avoid wasmtime conflict

* Remove sleep

* Improve naming of cli option

* Remove conflict with `validator`

* Improve docs

* Update smoldot, remove unwanted change

* Apply suggestions from code review

Co-authored-by: Dmitry Markin <dmitry@markin.tech>

* Disable collation

* Reviewer comments

* Update smoldot and tokio-platform

* Update smoldot

* Update smoldot

* Adjust to new version

* Patch substrate

* Use constants

* Add readme entry, improve zombienet tests

* Apply suggestions from code review

Co-authored-by: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com>

* Make execution mode an enum

* Update smoldot, remove substrate patch

* Update client/relay-chain-rpc-interface/src/rpc_client.rs

Co-authored-by: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com>

* Reduce duplicate code

* Update smoldot

* Update smoldot

* Fix build

* Update smoldot

* Make platform compile

* Clean up dependencies

* Use crates.io instead of github for smoldot

* Apply suggestions from code review

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Docs

* Improve docs

* Remove `RpcFrontend`

---------

Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com>
Co-authored-by: Davide Galassi <davxy@datawok.net>

Author: 4 people

Cargo.lock

@@ -43,9 +43,10 @@ You may run `polkadot-parachain` locally after building it or using one of the c
 
 ### Relay Chain Interaction
 To operate a parachain node, a connection to the corresponding relay chain is necessary. This can be
-achieved in one of two ways:
+achieved in one of three ways:
 1. Run a full relay chain node within the parachain node (default)
 2. Connect to an external relay chain node via WebSocket RPC
+3. Run a light client for the relay chain
 
 #### In-process Relay Chain Node
 If an external relay chain node is not specified (default behavior), then a full relay chain node is
@@ -55,7 +56,7 @@ This node has all of the typical components of a regular Polkadot node and will
 with the relay chain to work.
 
 ##### Example command
-```shell=
+```bash
 polkadot-parachain \
 	--chain parachain-chainspec.json \
 	--tmp \
@@ -76,7 +77,7 @@ they will use fewer system resources.
 relay chain node in-process. Even though they lack the majority of normal Polkadot subsystems, they
 will still need to connect directly to the relay chain network.
 ##### Example command
-```shell=
+```bash
 polkadot-parachain \
 	--chain parachain-chainspec.json \
 	--tmp \
@@ -87,6 +88,26 @@ polkadot-parachain \
 	--chain relaychain-chainspec.json
 ```
 
+#### Relay Chain Light Client
+An internal relay chain light client provides a fast and lightweight approach for connecting to the relay chain network.
+It provides relay chain notifications and facilitates runtime calls.
+
+To specify which chain the light client should connect to, users need to supply a relay chain chain-spec as part of the relay chain arguments.
+
+**Note:** At this time, any parachain nodes using this feature will still spawn a significantly cut-down
+relay chain node in-process. Even though they lack the majority of normal Polkadot subsystems, they
+will still need to connect directly to the relay chain network.
+
+##### Example command
+```bash
+polkadot-parachain \
+	--chain parachain-chainspec.json \
+	--tmp \
+	--relay-chain-light-client \
+	-- \
+	--chain relaychain-chainspec.json
+```
+
 ## Installation and Setup
 Before building Cumulus SDK based nodes / runtimes prepare your environment by following Substrate
 [installation instructions](https://docs.substrate.io/main-docs/install/).

README.md

@@ -304,6 +304,11 @@ pub struct RunCmd {
 		alias = "relay-chain-rpc-url"
 	)]
 	pub relay_chain_rpc_urls: Vec<Url>,
+
+	/// EXPERIMENTAL: Embed a light client for the relay chain. Only supported for full-nodes.
+	/// Will use the specified relay chain chainspec.
+	#[arg(long, conflicts_with_all = ["relay_chain_rpc_urls", "collator"])]
+	pub relay_chain_light_client: bool,
 }
 
 impl RunCmd {
@@ -319,15 +324,33 @@ impl RunCmd {
 
 	/// Create [`CollatorOptions`] representing options only relevant to parachain collator nodes
 	pub fn collator_options(&self) -> CollatorOptions {
-		CollatorOptions { relay_chain_rpc_urls: self.relay_chain_rpc_urls.clone() }
+		let relay_chain_mode =
+			match (self.relay_chain_light_client, !self.relay_chain_rpc_urls.is_empty()) {
+				(true, _) => RelayChainMode::LightClient,
+				(_, true) => RelayChainMode::ExternalRpc(self.relay_chain_rpc_urls.clone()),
+				_ => RelayChainMode::Embedded,
+			};
+
+		CollatorOptions { relay_chain_mode }
 	}
 }
 
+/// Possible modes for the relay chain to operate in.
+#[derive(Clone, Debug)]
+pub enum RelayChainMode {
+	/// Spawn embedded relay chain node
+	Embedded,
+	/// Connect to remote relay chain node via websocket RPC
+	ExternalRpc(Vec<Url>),
+	/// Spawn embedded relay chain light client
+	LightClient,
+}
+
 /// Options only relevant for collator nodes
 #[derive(Clone, Debug)]
 pub struct CollatorOptions {
-	/// Location of relay chain full node
-	pub relay_chain_rpc_urls: Vec<Url>,
+	/// How this collator retrieves relay chain information
+	pub relay_chain_mode: RelayChainMode,
 }
 
 /// A non-redundant version of the `RunCmd` that sets the `validator` field when the

client/cli/src/lib.rs

@@ -20,14 +20,12 @@ polkadot-node-core-runtime-api = { git = "https://github.com/paritytech/polkadot
 
 # substrate deps
 sc-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-network-common = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-utils = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -42,7 +40,6 @@ lru = "0.11.0"
 tracing = "0.1.37"
 async-trait = "0.1.73"
 futures = "0.3.28"
-tokio = { version = "1.32.0", features = ["macros"] }
 
 [features]
 network-protocol-staging = ["polkadot-node-network-protocol/network-protocol-staging"]

client/relay-chain-minimal-node/Cargo.toml

@@ -17,7 +17,7 @@
 use collator_overseer::{CollatorOverseerGenArgs, NewMinimalNode};
 
 use cumulus_relay_chain_interface::{RelayChainError, RelayChainInterface, RelayChainResult};
-use cumulus_relay_chain_rpc_interface::{RelayChainRpcInterface, Url};
+use cumulus_relay_chain_rpc_interface::{RelayChainRpcClient, RelayChainRpcInterface, Url};
 use network::build_collator_network;
 use polkadot_network_bridge::{peer_sets_info, IsAuthority};
 use polkadot_node_network_protocol::{
@@ -38,13 +38,14 @@ use sp_runtime::{app_crypto::Pair, traits::Block as BlockT};
 use futures::StreamExt;
 use std::sync::Arc;
 
+mod blockchain_rpc_client;
 mod collator_overseer;
-
 mod network;
 
-mod blockchain_rpc_client;
 pub use blockchain_rpc_client::BlockChainRpcClient;
 
+const LOG_TARGET: &str = "minimal-relaychain-node";
+
 fn build_authority_discovery_service<Block: BlockT>(
 	task_manager: &TaskManager,
 	client: Arc<BlockChainRpcClient>,
@@ -82,16 +83,11 @@ fn build_authority_discovery_service<Block: BlockT>(
 	service
 }
 
-pub async fn build_minimal_relay_chain_node(
+async fn build_interface(
 	polkadot_config: Configuration,
 	task_manager: &mut TaskManager,
-	relay_chain_url: Vec<Url>,
+	client: RelayChainRpcClient,
 ) -> RelayChainResult<(Arc<(dyn RelayChainInterface + 'static)>, Option<CollatorPair>)> {
-	let client = cumulus_relay_chain_rpc_interface::create_client_and_start_worker(
-		relay_chain_url,
-		task_manager,
-	)
-	.await?;
 	let collator_pair = CollatorPair::generate().0;
 	let collator_node = new_minimal_relay_chain(
 		polkadot_config,
@@ -106,6 +102,44 @@ pub async fn build_minimal_relay_chain_node(
 	))
 }
 
+pub async fn build_minimal_relay_chain_node_with_rpc(
+	polkadot_config: Configuration,
+	task_manager: &mut TaskManager,
+	relay_chain_url: Vec<Url>,
+) -> RelayChainResult<(Arc<(dyn RelayChainInterface + 'static)>, Option<CollatorPair>)> {
+	let client = cumulus_relay_chain_rpc_interface::create_client_and_start_worker(
+		relay_chain_url,
+		task_manager,
+	)
+	.await?;
+
+	build_interface(polkadot_config, task_manager, client).await
+}
+
+pub async fn build_minimal_relay_chain_node_light_client(
+	polkadot_config: Configuration,
+	task_manager: &mut TaskManager,
+) -> RelayChainResult<(Arc<(dyn RelayChainInterface + 'static)>, Option<CollatorPair>)> {
+	tracing::info!(
+		target: LOG_TARGET,
+		chain_name = polkadot_config.chain_spec.name(),
+		chain_id = polkadot_config.chain_spec.id(),
+		"Initializing embedded light client with chain spec."
+	);
+
+	let spec = polkadot_config
+		.chain_spec
+		.as_json(false)
+		.map_err(RelayChainError::GenericError)?;
+
+	let client = cumulus_relay_chain_rpc_interface::create_client_and_start_light_client_worker(
+		spec,
+		task_manager,
+	)
+	.await?;
+
+	build_interface(polkadot_config, task_manager, client).await
+}
 /// Builds a minimal relay chain node. Chain data is fetched
 /// via [`BlockChainRpcClient`] and fed into the overseer and its subsystems.
 ///
@@ -117,8 +151,6 @@ pub async fn build_minimal_relay_chain_node(
 /// - NetworkBridgeRx
 /// - NetworkBridgeTx
 /// - RuntimeApi
-/// - ChainApi
-/// - AvailabilityDistribution
 #[sc_tracing::logging::prefix_logs_with("Relaychain")]
 async fn new_minimal_relay_chain(
 	config: Configuration,

client/relay-chain-minimal-node/src/lib.rs

@@ -17,11 +17,13 @@ sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch =
 sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-storage = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-rpc-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 tokio = { version = "1.32.0", features = ["sync"] }
+tokio-util = { version = "0.7.8", features = ["compat"] }
 
 futures = "0.3.28"
 futures-timer = "3.0.2"
@@ -33,3 +35,9 @@ url = "2.4.0"
 serde_json = "1.0.105"
 serde = "1.0.183"
 lru = "0.11.0"
+smoldot = { version = "0.11.0",  default_features = false, features = ["std"]}
+smoldot-light = { version = "0.9.0", default_features = false, features = ["std"] }
+either = "1.8.1"
+thiserror = "1.0.38"
+rand = "0.8.5"
+pin-project = "1.1.3"

client/relay-chain-rpc-interface/Cargo.toml

@@ -38,9 +38,15 @@ use std::pin::Pin;
 use cumulus_primitives_core::relay_chain::BlockId;
 pub use url::Url;
 
+mod light_client_worker;
 mod reconnecting_ws_client;
 mod rpc_client;
-pub use rpc_client::{create_client_and_start_worker, RelayChainRpcClient};
+mod tokio_platform;
+
+pub use rpc_client::{
+	create_client_and_start_light_client_worker, create_client_and_start_worker,
+	RelayChainRpcClient,
+};
 
 const TIMEOUT_IN_SECONDS: u64 = 6;