chain-spec: getting ready for native-runtime-free world (#1256)

This PR prepares chains specs for _native-runtime-free_  world.

This PR has following changes:
- `substrate`:
  - adds support for:
- JSON based `GenesisConfig` to `ChainSpec` allowing interaction with
runtime `GenesisBuilder` API.
- interacting with arbitrary runtime wasm blob to[
`chain-spec-builder`](https://github.com/paritytech/substrate/blob/3ef576eaeb3f42610e85daecc464961cf1295570/bin/utils/chain-spec-builder/src/lib.rs#L46)
command line util,
- removes
[`code`](https://github.com/paritytech/substrate/blob/3ef576eaeb3f42610e85daecc464961cf1295570/frame/system/src/lib.rs#L660)
from `system_pallet`
  - adds `code` to the `ChainSpec`
- deprecates
[`ChainSpec::from_genesis`](https://github.com/paritytech/substrate/blob/3ef576eaeb3f42610e85daecc464961cf1295570/client/chain-spec/src/chain_spec.rs#L263),
but also changes the signature of this method extending it with `code`
argument.
[`ChainSpec::builder()`](https://github.com/paritytech/substrate/blob/20bee680ed098be7239cf7a6b804cd4de267983e/client/chain-spec/src/chain_spec.rs#L507)
should be used instead.
- `polkadot`:
- all references to `RuntimeGenesisConfig` in `node/service` are
removed,
- all
`(kusama|polkadot|versi|rococo|wococo)_(staging|dev)_genesis_config`
functions now return the JSON patch for default runtime `GenesisConfig`,
  - `ChainSpecBuilder` is used, `ChainSpec::from_genesis` is removed,

- `cumulus`:
  - `ChainSpecBuilder` is used, `ChainSpec::from_genesis` is removed,
- _JSON_ patch configuration used instead of `RuntimeGenesisConfig
struct` in all chain specs.
  
---------

Co-authored-by: command-bot <>
Co-authored-by: Javier Viola <javier@parity.io>
Co-authored-by: Davide Galassi <davxy@datawok.net>
Co-authored-by: Francisco Aguirre <franciscoaguirreperez@gmail.com>
Co-authored-by: Kevin Krone <kevin@parity.io>
Co-authored-by: Bastian Köcher <git@kchr.de>

substrate/bin/utils/chain-spec-builder/Cargo.toml

@@ -23,8 +23,12 @@ crate-type = ["rlib"]
 ansi_term = "0.12.1"
 clap = { version = "4.4.6", features = ["derive"] }
 rand = "0.8"
+kitchensink-runtime = { version = "3.0.0-dev", path = "../../node/runtime" }
+log = "0.4.17"
 node-cli = { package = "staging-node-cli", path = "../../node/cli" }
 sc-chain-spec = { path = "../../../client/chain-spec" }
 sc-keystore = { path = "../../../client/keystore" }
+serde_json = "1.0.100"
 sp-core = { path = "../../../primitives/core" }
 sp-keystore = { path = "../../../primitives/keystore" }
+sp-tracing = { version = "10.0.0", path = "../../../primitives/tracing" }

substrate/bin/utils/chain-spec-builder/bin/main.rs

@@ -17,28 +17,40 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use chain_spec_builder::{
-	generate_authority_keys_and_store, generate_chain_spec, print_seeds, ChainSpecBuilder,
+	generate_authority_keys_and_store, generate_chain_spec, generate_chain_spec_for_runtime,
+	print_seeds, ChainSpecBuilder, ChainSpecBuilderCmd, EditCmd, GenerateCmd, NewCmd, VerifyCmd,
 };
 use clap::Parser;
 use node_cli::chain_spec;
 use rand::{distributions::Alphanumeric, rngs::OsRng, Rng};
+use sc_chain_spec::{update_code_in_json_chain_spec, GenericChainSpec};
 use sp_core::{crypto::Ss58Codec, sr25519};
 use staging_chain_spec_builder as chain_spec_builder;
 use std::fs;
 
 fn main() -> Result<(), String> {
-	#[cfg(build_type = "debug")]
-	println!(
-		"The chain spec builder builds a chain specification that includes a Substrate runtime \
-		 compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \
-		 the chain spec builder binary in `--release` mode.\n",
-	);
+	sp_tracing::try_init_simple();
 
 	let builder = ChainSpecBuilder::parse();
-	let chain_spec_path = builder.chain_spec_path().to_path_buf();
+	#[cfg(build_type = "debug")]
+	if matches!(builder.command, ChainSpecBuilderCmd::Generate(_) | ChainSpecBuilderCmd::New(_)) {
+		println!(
+			"The chain spec builder builds a chain specification that includes a Substrate runtime \
+		 compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \
+		 the chain spec builder binary in `--release` mode.\n",
+		 );
+	}
 
-	let (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) = match builder {
-		ChainSpecBuilder::Generate { authorities, nominators, endowed, keystore_path, .. } => {
+	let chain_spec_path = builder.chain_spec_path.to_path_buf();
+	let mut write_chain_spec = true;
+
+	let chain_spec_json = match builder.command {
+		ChainSpecBuilderCmd::Generate(GenerateCmd {
+			authorities,
+			nominators,
+			endowed,
+			keystore_path,
+		}) => {
 			let authorities = authorities.max(1);
 			let rand_str = || -> String {
 				OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect()
@@ -72,19 +84,58 @@ fn main() -> Result<(), String> {
 			let sudo_account =
 				chain_spec::get_account_id_from_seed::<sr25519::Public>(&sudo_seed).to_ss58check();
 
-			(authority_seeds, nominator_accounts, endowed_accounts, sudo_account)
+			generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account)
 		},
-		ChainSpecBuilder::New {
+		ChainSpecBuilderCmd::New(NewCmd {
 			authority_seeds,
 			nominator_accounts,
 			endowed_accounts,
 			sudo_account,
-			..
-		} => (authority_seeds, nominator_accounts, endowed_accounts, sudo_account),
-	};
+		}) =>
+			generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account),
+		ChainSpecBuilderCmd::Runtime(cmd) => generate_chain_spec_for_runtime(&cmd),
+		ChainSpecBuilderCmd::Edit(EditCmd {
+			ref input_chain_spec,
+			ref runtime_wasm_path,
+			convert_to_raw,
+		}) => {
+			let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?;
 
-	let json =
-		generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account)?;
+			let mut chain_spec_json =
+				serde_json::from_str::<serde_json::Value>(&chain_spec.as_json(convert_to_raw)?)
+					.map_err(|e| format!("Conversion to json failed: {e}"))?;
+			if let Some(path) = runtime_wasm_path {
+				update_code_in_json_chain_spec(
+					&mut chain_spec_json,
+					&fs::read(path.as_path())
+						.map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..],
+				);
+			}
 
-	fs::write(chain_spec_path, json).map_err(|err| err.to_string())
+			serde_json::to_string_pretty(&chain_spec_json)
+				.map_err(|e| format!("to pretty failed: {e}"))
+		},
+		ChainSpecBuilderCmd::Verify(VerifyCmd { ref input_chain_spec, ref runtime_wasm_path }) => {
+			write_chain_spec = false;
+			let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?;
+			let mut chain_spec_json =
+				serde_json::from_str::<serde_json::Value>(&chain_spec.as_json(true)?)
+					.map_err(|e| format!("Conversion to json failed: {e}"))?;
+			if let Some(path) = runtime_wasm_path {
+				update_code_in_json_chain_spec(
+					&mut chain_spec_json,
+					&fs::read(path.as_path())
+						.map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..],
+				);
+			};
+			serde_json::to_string_pretty(&chain_spec_json)
+				.map_err(|e| format!("to pretty failed: {e}"))
+		},
+	}?;
+
+	if write_chain_spec {
+		fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string())
+	} else {
+		Ok(())
+	}
 }

substrate/bin/utils/chain-spec-builder/src/lib.rs

@@ -29,96 +29,169 @@
 //! [`sc-chain-spec`]: ../sc_chain_spec/index.html
 //! [`node-cli`]: ../node_cli/index.html
 
-use std::path::{Path, PathBuf};
+use std::{
+	fs,
+	path::{Path, PathBuf},
+};
 
 use ansi_term::Style;
-use clap::Parser;
+use clap::{Parser, Subcommand};
+use sc_chain_spec::GenesisConfigBuilderRuntimeCaller;
 
 use node_cli::chain_spec::{self, AccountId};
 use sc_keystore::LocalKeystore;
+use serde_json::Value;
 use sp_core::crypto::{ByteArray, Ss58Codec};
 use sp_keystore::KeystorePtr;
 
 /// A utility to easily create a testnet chain spec definition with a given set
 /// of authorities and endowed accounts and/or generate random accounts.
-#[derive(Parser)]
+#[derive(Debug, Parser)]
 #[command(rename_all = "kebab-case")]
-pub enum ChainSpecBuilder {
-	/// Create a new chain spec with the given authorities, endowed and sudo
-	/// accounts.
-	New {
-		/// Authority key seed.
-		#[arg(long, short, required = true)]
-		authority_seeds: Vec<String>,
-		/// Active nominators (SS58 format), each backing a random subset of the aforementioned
-		/// authorities.
-		#[arg(long, short, default_value = "0")]
-		nominator_accounts: Vec<String>,
-		/// Endowed account address (SS58 format).
-		#[arg(long, short)]
-		endowed_accounts: Vec<String>,
-		/// Sudo account address (SS58 format).
-		#[arg(long, short)]
-		sudo_account: String,
-		/// The path where the chain spec should be saved.
-		#[arg(long, short, default_value = "./chain_spec.json")]
-		chain_spec_path: PathBuf,
-	},
-	/// Create a new chain spec with the given number of authorities and endowed
-	/// accounts. Random keys will be generated as required.
-	Generate {
-		/// The number of authorities.
-		#[arg(long, short)]
-		authorities: usize,
-		/// The number of nominators backing the aforementioned authorities.
-		///
-		/// Will nominate a random subset of `authorities`.
-		#[arg(long, short, default_value_t = 0)]
-		nominators: usize,
-		/// The number of endowed accounts.
-		#[arg(long, short, default_value_t = 0)]
-		endowed: usize,
-		/// The path where the chain spec should be saved.
-		#[arg(long, short, default_value = "./chain_spec.json")]
-		chain_spec_path: PathBuf,
-		/// Path to use when saving generated keystores for each authority.
-		///
-		/// At this path, a new folder will be created for each authority's
-		/// keystore named `auth-$i` where `i` is the authority index, i.e.
-		/// `auth-0`, `auth-1`, etc.
-		#[arg(long, short)]
-		keystore_path: Option<PathBuf>,
-	},
+pub struct ChainSpecBuilder {
+	#[command(subcommand)]
+	pub command: ChainSpecBuilderCmd,
+	/// The path where the chain spec should be saved.
+	#[arg(long, short, default_value = "./chain_spec.json")]
+	pub chain_spec_path: PathBuf,
 }
 
-impl ChainSpecBuilder {
-	/// Returns the path where the chain spec should be saved.
-	pub fn chain_spec_path(&self) -> &Path {
-		match self {
-			ChainSpecBuilder::New { chain_spec_path, .. } => chain_spec_path.as_path(),
-			ChainSpecBuilder::Generate { chain_spec_path, .. } => chain_spec_path.as_path(),
-		}
-	}
+#[derive(Debug, Subcommand)]
+#[command(rename_all = "kebab-case")]
+pub enum ChainSpecBuilderCmd {
+	New(NewCmd),
+	Generate(GenerateCmd),
+	Runtime(RuntimeCmd),
+	Edit(EditCmd),
+	Verify(VerifyCmd),
 }
 
-fn genesis_constructor(
-	authority_seeds: &[String],
-	nominator_accounts: &[AccountId],
-	endowed_accounts: &[AccountId],
-	sudo_account: &AccountId,
-) -> chain_spec::RuntimeGenesisConfig {
-	let authorities = authority_seeds
-		.iter()
-		.map(AsRef::as_ref)
-		.map(chain_spec::authority_keys_from_seed)
-		.collect::<Vec<_>>();
+/// Create a new chain spec with the given authorities, endowed and sudo
+/// accounts. Only works for kitchen-sink runtime
+#[derive(Parser, Debug)]
+#[command(rename_all = "kebab-case")]
+pub struct NewCmd {
+	/// Authority key seed.
+	#[arg(long, short, required = true)]
+	pub authority_seeds: Vec<String>,
+	/// Active nominators (SS58 format), each backing a random subset of the aforementioned
+	/// authorities.
+	#[arg(long, short, default_value = "0")]
+	pub nominator_accounts: Vec<String>,
+	/// Endowed account address (SS58 format).
+	#[arg(long, short)]
+	pub endowed_accounts: Vec<String>,
+	/// Sudo account address (SS58 format).
+	#[arg(long, short)]
+	pub sudo_account: String,
+}
 
-	chain_spec::testnet_genesis(
-		authorities,
-		nominator_accounts.to_vec(),
-		sudo_account.clone(),
-		Some(endowed_accounts.to_vec()),
-	)
+/// Create a new chain spec with the given number of authorities and endowed
+/// accounts. Random keys will be generated as required.
+#[derive(Parser, Debug)]
+pub struct GenerateCmd {
+	/// The number of authorities.
+	#[arg(long, short)]
+	pub authorities: usize,
+	/// The number of nominators backing the aforementioned authorities.
+	///
+	/// Will nominate a random subset of `authorities`.
+	#[arg(long, short, default_value_t = 0)]
+	pub nominators: usize,
+	/// The number of endowed accounts.
+	#[arg(long, short, default_value_t = 0)]
+	pub endowed: usize,
+	/// Path to use when saving generated keystores for each authority.
+	///
+	/// At this path, a new folder will be created for each authority's
+	/// keystore named `auth-$i` where `i` is the authority index, i.e.
+	/// `auth-0`, `auth-1`, etc.
+	#[arg(long, short)]
+	pub keystore_path: Option<PathBuf>,
+}
+
+/// Create a new chain spec by interacting with the provided runtime wasm blob.
+#[derive(Parser, Debug)]
+pub struct RuntimeCmd {
+	/// The name of chain
+	#[arg(long, short = 'n', default_value = "Custom")]
+	chain_name: String,
+	/// The chain id
+	#[arg(long, short = 'i', default_value = "custom")]
+	chain_id: String,
+	/// The path to runtime wasm blob
+	#[arg(long, short)]
+	runtime_wasm_path: PathBuf,
+	/// Export chainspec as raw storage
+	#[arg(long, short = 's')]
+	raw_storage: bool,
+	/// Verify the genesis config. This silently generates the raw storage from genesis config. Any
+	/// errors will be reported.
+	#[arg(long, short = 'v')]
+	verify: bool,
+	#[command(subcommand)]
+	action: GenesisBuildAction,
+}
+
+#[derive(Subcommand, Debug, Clone)]
+enum GenesisBuildAction {
+	Patch(PatchCmd),
+	Full(FullCmd),
+	Default(DefaultCmd),
+}
+
+/// Patches the runtime's default genesis config with provided patch.
+#[derive(Parser, Debug, Clone)]
+struct PatchCmd {
+	/// The path to the runtime genesis config patch.
+	#[arg(long, short)]
+	patch_path: PathBuf,
+}
+
+/// Build the genesis config for runtime using provided json file. No defaults will be used.
+#[derive(Parser, Debug, Clone)]
+struct FullCmd {
+	/// The path to the full runtime genesis config json file.
+	#[arg(long, short)]
+	config_path: PathBuf,
+}
+
+/// Gets the default genesis config for the runtime and uses it in ChainSpec. Please note that
+/// default genesis config may not be valid. For some runtimes initial values should be added there
+/// (e.g. session keys, babe epoch).
+#[derive(Parser, Debug, Clone)]
+struct DefaultCmd {
+	/// If provided stores the default genesis config json file at given path (in addition to
+	/// chain-spec).
+	#[arg(long, short)]
+	default_config_path: Option<PathBuf>,
+}
+
+/// Edits provided input chain spec. Input can be converted into raw storage chain-spec. The code
+/// can be updated with the runtime provided in the command line.
+#[derive(Parser, Debug, Clone)]
+pub struct EditCmd {
+	/// Chain spec to be edited
+	#[arg(long, short)]
+	pub input_chain_spec: PathBuf,
+	/// The path to new runtime wasm blob to be stored into chain-spec
+	#[arg(long, short = 'r')]
+	pub runtime_wasm_path: Option<PathBuf>,
+	/// Convert genesis spec to raw format
+	#[arg(long, short = 's')]
+	pub convert_to_raw: bool,
+}
+
+/// Verifies provided input chain spec. If the runtime is provided verification is performed against
+/// new runtime.
+#[derive(Parser, Debug, Clone)]
+pub struct VerifyCmd {
+	/// Chain spec to be edited
+	#[arg(long, short)]
+	pub input_chain_spec: PathBuf,
+	/// The path to new runtime wasm blob to be stored into chain-spec
+	#[arg(long, short = 'r')]
+	pub runtime_wasm_path: Option<PathBuf>,
 }
 
 /// Generate the chain spec using the given seeds and accounts.
@@ -145,27 +218,24 @@ pub fn generate_chain_spec(
 
 	let sudo_account = parse_account(sudo_account)?;
 
-	let chain_spec = chain_spec::ChainSpec::from_genesis(
-		"Custom",
-		"custom",
-		sc_chain_spec::ChainType::Live,
-		move || {
-			genesis_constructor(
-				&authority_seeds,
-				&nominator_accounts,
-				&endowed_accounts,
-				&sudo_account,
-			)
-		},
-		vec![],
-		None,
-		None,
-		None,
-		None,
-		Default::default(),
-	);
+	let authorities = authority_seeds
+		.iter()
+		.map(AsRef::as_ref)
+		.map(chain_spec::authority_keys_from_seed)
+		.collect::<Vec<_>>();
 
-	chain_spec.as_json(false)
+	chain_spec::ChainSpec::builder(kitchensink_runtime::wasm_binary_unwrap(), Default::default())
+		.with_name("Custom")
+		.with_id("custom")
+		.with_chain_type(sc_chain_spec::ChainType::Live)
+		.with_genesis_config_patch(chain_spec::testnet_genesis(
+			authorities,
+			nominator_accounts,
+			sudo_account,
+			Some(endowed_accounts),
+		))
+		.build()
+		.as_json(false)
 }
 
 /// Generate the authority keys and store them in the given `keystore_path`.
@@ -241,3 +311,54 @@ pub fn print_seeds(
 	println!("{}", header.paint("Sudo seed"));
 	println!("//{}", sudo_seed);
 }
+
+/// Processes `RuntimeCmd` and returns JSON version of `ChainSpec`
+pub fn generate_chain_spec_for_runtime(cmd: &RuntimeCmd) -> Result<String, String> {
+	let code = fs::read(cmd.runtime_wasm_path.as_path())
+		.map_err(|e| format!("wasm blob shall be readable {e}"))?;
+
+	let builder = chain_spec::ChainSpec::builder(&code[..], Default::default())
+		.with_name(&cmd.chain_name[..])
+		.with_id(&cmd.chain_id[..])
+		.with_chain_type(sc_chain_spec::ChainType::Live);
+
+	let builder = match cmd.action {
+		GenesisBuildAction::Patch(PatchCmd { ref patch_path }) => {
+			let patch = fs::read(patch_path.as_path())
+				.map_err(|e| format!("patch file {patch_path:?} shall be readable: {e}"))?;
+			builder.with_genesis_config_patch(serde_json::from_slice::<Value>(&patch[..]).map_err(
+				|e| format!("patch file {patch_path:?} shall contain a valid json: {e}"),
+			)?)
+		},
+		GenesisBuildAction::Full(FullCmd { ref config_path }) => {
+			let config = fs::read(config_path.as_path())
+				.map_err(|e| format!("config file {config_path:?} shall be readable: {e}"))?;
+			builder.with_genesis_config(serde_json::from_slice::<Value>(&config[..]).map_err(
+				|e| format!("config file {config_path:?} shall contain a valid json: {e}"),
+			)?)
+		},
+		GenesisBuildAction::Default(DefaultCmd { ref default_config_path }) => {
+			let caller = GenesisConfigBuilderRuntimeCaller::new(&code[..]);
+			let default_config = caller
+				.get_default_config()
+				.map_err(|e| format!("getting default config from runtime should work: {e}"))?;
+			default_config_path.clone().map(|path| {
+				fs::write(path.as_path(), serde_json::to_string_pretty(&default_config).unwrap())
+					.map_err(|err| err.to_string())
+			});
+			builder.with_genesis_config(default_config)
+		},
+	};
+
+	let chain_spec = builder.build();
+
+	match (cmd.verify, cmd.raw_storage) {
+		(_, true) => chain_spec.as_json(true),
+		(true, false) => {
+			chain_spec.as_json(true)?;
+			println!("Genesis config verification: OK");
+			chain_spec.as_json(false)
+		},
+		(false, false) => chain_spec.as_json(false),
+	}
+}