update kvdb & co (#6111)

* toml changes

* REVERTME: patch

* adapt parachains db interface

* fix Cargo.toml patch after master rebase

* fix av-store

* fix chain-selection

* fix parachains-db?

* Revert "fix Cargo.toml patch after master rebase"

This reverts commit 3afcbf033c86027b3f2b909d83ec703591bdd287.

* Revert "REVERTME: patch"

This reverts commit 464b717cf4142d3d09c3d77b83700b632d8c5f54.

* Use `Ok` imported from prelude

Co-authored-by: Bastian Köcher <info@kchr.de>

* update lockfile for {"substrate"}

* Revert "update lockfile for {"substrate"}"

This reverts commit fdc623de226f7645741b86c4b1a7d030fed2172d.

* cargo update -p sp-io

Co-authored-by: Bastian Köcher <info@kchr.de>
Co-authored-by: parity-processbot <>

polkadot/Cargo.toml

@@ -15,7 +15,7 @@ readme = "README.md"
 [dependencies]
 polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ]  }
 color-eyre = { version = "0.6.1", default-features = false }
-parity-util-mem = { version = "0.11.0", default-features = false, features = ["jemalloc-global"] }
+parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
 
 [dev-dependencies]
 assert_cmd = "2.0.4"

polkadot/core-primitives/Cargo.toml

@@ -10,7 +10,7 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
 parity-scale-codec = { version = "3.1.5", default-features = false, features = [ "derive" ] }
-parity-util-mem = { version = "0.11.0", default-features = false, optional = true }
+parity-util-mem = { version = "0.12.0", default-features = false, optional = true }
 
 [features]
 default = [ "std" ]

polkadot/node/core/approval-voting/Cargo.toml

@@ -13,7 +13,7 @@ bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
 lru = "0.8"
 merlin = "2.0"
 schnorrkel = "0.9.1"
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 derive_more = "0.99.17"
 thiserror = "1.0.31"
 
@@ -40,5 +40,5 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }

polkadot/node/core/av-store/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 [dependencies]
 futures = "0.3.21"
 futures-timer = "3.0.2"
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 thiserror = "1.0.31"
 gum = { package = "tracing-gum", path = "../../gum" }
 bitvec = "1.0.0"
@@ -24,7 +24,7 @@ polkadot-node-primitives = { path = "../../primitives" }
 log = "0.4.17"
 env_logger = "0.9.0"
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"
 
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }

polkadot/node/core/av-store/src/lib.rs

@@ -792,8 +792,9 @@ fn note_block_included(
 macro_rules! peek_num {
 	($iter:ident) => {
 		match $iter.peek() {
-			Some((k, _)) => decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b),
-			None => None,
+			Some(Ok((k, _))) => Ok(decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b)),
+			Some(Err(_)) => Err($iter.next().expect("peek returned Some(Err); qed").unwrap_err()),
+			None => Ok(None),
 		}
 	};
 }
@@ -819,10 +820,10 @@ async fn process_block_finalized<Context>(
 			let mut iter = subsystem
 				.db
 				.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
-				.take_while(|(k, _)| &k[..] < &end_prefix[..])
+				.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
 				.peekable();
 
-			match peek_num!(iter) {
+			match peek_num!(iter)? {
 				None => break, // end of iterator.
 				Some(n) => n,
 			}
@@ -867,10 +868,10 @@ async fn process_block_finalized<Context>(
 		let iter = subsystem
 			.db
 			.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
-			.take_while(|(k, _)| &k[..] < &end_prefix[..])
+			.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
 			.peekable();
 
-		let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash);
+		let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash)?;
 
 		// Now that we've iterated over the entire batch at this finalized height,
 		// update the meta.
@@ -890,22 +891,22 @@ async fn process_block_finalized<Context>(
 // loads all candidates at the finalized height and maps them to `true` if finalized
 // and `false` if unfinalized.
 fn load_all_at_finalized_height(
-	mut iter: std::iter::Peekable<impl Iterator<Item = (Box<[u8]>, Box<[u8]>)>>,
+	mut iter: std::iter::Peekable<impl Iterator<Item = io::Result<util::database::DBKeyValue>>>,
 	block_number: BlockNumber,
 	finalized_hash: Hash,
-) -> impl IntoIterator<Item = (CandidateHash, bool)> {
+) -> io::Result<impl IntoIterator<Item = (CandidateHash, bool)>> {
 	// maps candidate hashes to true if finalized, false otherwise.
 	let mut candidates = HashMap::new();
 
 	// Load all candidates that were included at this height.
 	loop {
-		match peek_num!(iter) {
+		match peek_num!(iter)? {
 			None => break,                         // end of iterator.
 			Some(n) if n != block_number => break, // end of batch.
 			_ => {},
 		}
 
-		let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed");
+		let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed")?;
 		let (_, block_hash, candidate_hash) =
 			decode_unfinalized_key(&k[..]).expect("`peek_num` checks validity of key; qed");
 
@@ -916,7 +917,7 @@ fn load_all_at_finalized_height(
 		}
 	}
 
-	candidates
+	Ok(candidates)
 }
 
 fn update_blocks_at_finalized_height(
@@ -1214,9 +1215,10 @@ fn prune_all(db: &Arc<dyn Database>, config: &Config, clock: &dyn Clock) -> Resu
 	let mut tx = DBTransaction::new();
 	let iter = db
 		.iter_with_prefix(config.col_meta, &range_start[..])
-		.take_while(|(k, _)| &k[..] < &range_end[..]);
+		.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &range_end[..]));
 
-	for (k, _v) in iter {
+	for r in iter {
+		let (k, _v) = r?;
 		tx.delete(config.col_meta, &k[..]);
 
 		let (_, candidate_hash) = match decode_pruning_key(&k[..]) {

polkadot/node/core/chain-selection/Cargo.toml

@@ -13,7 +13,7 @@ polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-primitives = { path = "../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 thiserror = "1.0.31"
 parity-scale-codec = "3.1.5"
 
@@ -22,4 +22,4 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 parking_lot = "0.12.0"
 assert_matches = "1"
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"

polkadot/node/core/chain-selection/src/db_backend/v1.rs

@@ -235,16 +235,21 @@ impl Backend for DbBackend {
 			self.inner.iter_with_prefix(self.config.col_data, &STAGNANT_AT_PREFIX[..]);
 
 		let val = stagnant_at_iter
-			.filter_map(|(k, v)| {
-				match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok()) {
-					(Some(at), Some(stagnant_at)) => Some((at, stagnant_at)),
-					_ => None,
-				}
+			.filter_map(|r| match r {
+				Ok((k, v)) =>
+					match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok())
+					{
+						(Some(at), Some(stagnant_at)) => Some(Ok((at, stagnant_at))),
+						_ => None,
+					},
+				Err(e) => Some(Err(e)),
 			})
 			.enumerate()
-			.take_while(|(idx, (at, _))| *at <= up_to.into() && *idx < max_elements)
+			.take_while(|(idx, r)| {
+				r.as_ref().map_or(true, |(at, _)| *at <= up_to.into() && *idx < max_elements)
+			})
 			.map(|(_, v)| v)
-			.collect::<Vec<_>>();
+			.collect::<Result<Vec<_>, _>>()?;
 
 		Ok(val)
 	}
@@ -254,10 +259,13 @@ impl Backend for DbBackend {
 			self.inner.iter_with_prefix(self.config.col_data, &BLOCK_HEIGHT_PREFIX[..]);
 
 		let val = blocks_at_height_iter
-			.filter_map(|(k, _)| decode_block_height_key(&k[..]))
+			.filter_map(|r| match r {
+				Ok((k, _)) => decode_block_height_key(&k[..]).map(Ok),
+				Err(e) => Some(Err(e)),
+			})
 			.next();
 
-		Ok(val)
+		val.transpose().map_err(Error::from)
 	}
 
 	fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {

polkadot/node/core/dispute-coordinator/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 futures = "0.3.21"
 gum = { package = "tracing-gum", path = "../../gum" }
 parity-scale-codec = "3.1.5"
-kvdb = "0.11.0"
+kvdb = "0.12.0"
 thiserror = "1.0.31"
 lru = "0.8.0"
 fatality = "0.0.6"
@@ -22,7 +22,7 @@ sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste
 
 
 [dev-dependencies]
-kvdb-memorydb = "0.11.0"
+kvdb-memorydb = "0.12.0"
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }

polkadot/node/core/runtime-api/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 futures = "0.3.21"
 gum = { package = "tracing-gum", path = "../../gum" }
 memory-lru = "0.1.1"
-parity-util-mem = { version = "0.11.0", default-features = false }
+parity-util-mem = { version = "0.12.0", default-features = false }
 
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 

polkadot/node/malus/Cargo.toml

@@ -23,7 +23,7 @@ polkadot-node-core-backing = { path = "../core/backing" }
 polkadot-node-primitives = { path = "../primitives" }
 polkadot-primitives = { path = "../../primitives" }
 polkadot-node-core-pvf = { path = "../core/pvf" }
-parity-util-mem = { version = "0.11.0", default-features = false, features = ["jemalloc-global"] }
+parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
 color-eyre = { version = "0.6.1", default-features = false }
 assert_matches = "1.5"
 async-trait = "0.1.57"

polkadot/node/overseer/Cargo.toml

@@ -18,7 +18,7 @@ polkadot-primitives = { path = "../../primitives" }
 orchestra = "0.0.2"
 gum = { package = "tracing-gum", path = "../gum" }
 lru = "0.8"
-parity-util-mem = { version = "0.11.0", default-features = false }
+parity-util-mem = { version = "0.12.0", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 async-trait = "0.1.57"
 

polkadot/node/service/Cargo.toml

@@ -69,8 +69,8 @@ gum = { package = "tracing-gum", path = "../gum/" }
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
 thiserror = "1.0.31"
-kvdb = "0.11.0"
-kvdb-rocksdb = { version = "0.15.2", optional = true }
+kvdb = "0.12.0"
+kvdb-rocksdb = { version = "0.16.0", optional = true }
 parity-db = { version = "0.3.16", optional = true }
 async-trait = "0.1.57"
 lru = "0.8"

polkadot/node/service/src/parachains_db/upgrade.rs

@@ -121,7 +121,7 @@ fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> {
 		.to_str()
 		.ok_or_else(|| super::other_io_error("Invalid database path".into()))?;
 	let db_cfg = DatabaseConfig::with_columns(super::columns::v0::NUM_COLUMNS);
-	let db = Database::open(&db_cfg, db_path)?;
+	let mut db = Database::open(&db_cfg, db_path)?;
 
 	db.add_column()?;
 	db.add_column()?;

polkadot/node/subsystem-util/Cargo.toml

@@ -32,8 +32,8 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-kvdb = "0.11.0"
-parity-util-mem = { version = "0.11", default-features = false }
+kvdb = "0.12.0"
+parity-util-mem = { version = "0.12.0", default-features = false }
 parity-db = { version = "0.3.13" }
 
 [dev-dependencies]
@@ -44,5 +44,5 @@ log = "0.4.17"
 polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }
 lazy_static = "1.4.0"
 polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" }
-kvdb-shared-tests = "0.9.0"
+kvdb-shared-tests = "0.10.0"
 tempfile = "3.1.0"

polkadot/node/subsystem-util/src/database.rs

@@ -16,7 +16,7 @@
 
 //! Database trait for polkadot db.
 
-pub use kvdb::{DBTransaction, DBValue, KeyValueDB};
+pub use kvdb::{DBKeyValue, DBTransaction, DBValue, KeyValueDB};
 
 /// Database trait with ordered key capacity.
 pub trait Database: KeyValueDB {
@@ -27,7 +27,7 @@ pub trait Database: KeyValueDB {
 
 /// Implementation for database supporting `KeyValueDB` already.
 pub mod kvdb_impl {
-	use super::{DBTransaction, DBValue, Database, KeyValueDB};
+	use super::{DBKeyValue, DBTransaction, DBValue, Database, KeyValueDB};
 	use kvdb::{DBOp, IoStats, IoStatsKind};
 	use parity_util_mem::{MallocSizeOf, MallocSizeOfOps};
 	use std::{collections::BTreeSet, io::Result};
@@ -86,7 +86,7 @@ pub mod kvdb_impl {
 			self.db.get(col, key)
 		}
 
-		fn get_by_prefix(&self, col: u32, prefix: &[u8]) -> Option<Box<[u8]>> {
+		fn get_by_prefix(&self, col: u32, prefix: &[u8]) -> Result<Option<DBValue>> {
 			self.ensure_is_indexed(col);
 			self.db.get_by_prefix(col, prefix)
 		}
@@ -96,7 +96,7 @@ pub mod kvdb_impl {
 			self.db.write(transaction)
 		}
 
-		fn iter<'a>(&'a self, col: u32) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
+		fn iter<'a>(&'a self, col: u32) -> Box<dyn Iterator<Item = Result<DBKeyValue>> + 'a> {
 			self.ensure_is_indexed(col);
 			self.db.iter(col)
 		}
@@ -105,15 +105,11 @@ pub mod kvdb_impl {
 			&'a self,
 			col: u32,
 			prefix: &'a [u8],
-		) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
+		) -> Box<dyn Iterator<Item = Result<DBKeyValue>> + 'a> {
 			self.ensure_is_indexed(col);
 			self.db.iter_with_prefix(col, prefix)
 		}
 
-		fn restore(&self, _new_db: &str) -> Result<()> {
-			unimplemented!("restore is unsupported")
-		}
-
 		fn io_stats(&self, kind: IoStatsKind) -> IoStats {
 			self.db.io_stats(kind)
 		}
@@ -122,7 +118,7 @@ pub mod kvdb_impl {
 			self.db.has_key(col, key)
 		}
 
-		fn has_prefix(&self, col: u32, prefix: &[u8]) -> bool {
+		fn has_prefix(&self, col: u32, prefix: &[u8]) -> Result<bool> {
 			self.ensure_is_indexed(col);
 			self.db.has_prefix(col, prefix)
 		}
@@ -138,8 +134,8 @@ pub mod kvdb_impl {
 
 /// Utilities for using parity-db database.
 pub mod paritydb_impl {
-	use super::{DBTransaction, DBValue, Database, KeyValueDB};
-	use kvdb::{DBOp, IoStats, IoStatsKind};
+	use super::{DBKeyValue, DBTransaction, DBValue, Database, KeyValueDB};
+	use kvdb::DBOp;
 	use parity_db::Db;
 	use parking_lot::Mutex;
 	use std::{collections::BTreeSet, io::Result, sync::Arc};
@@ -179,18 +175,20 @@ pub mod paritydb_impl {
 			map_err(self.db.get(col as u8, key))
 		}
 
-		fn get_by_prefix(&self, col: u32, prefix: &[u8]) -> Option<Box<[u8]>> {
-			self.iter_with_prefix(col, prefix).next().map(|(_, v)| v)
+		fn get_by_prefix(&self, col: u32, prefix: &[u8]) -> Result<Option<DBValue>> {
+			self.iter_with_prefix(col, prefix)
+				.next()
+				.transpose()
+				.map(|mb| mb.map(|(_, v)| v))
 		}
 
-		fn iter<'a>(&'a self, col: u32) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
-			let mut iter = handle_err(self.db.iter(col as u8));
+		fn iter<'a>(&'a self, col: u32) -> Box<dyn Iterator<Item = Result<DBKeyValue>> + 'a> {
+			let mut iter = match self.db.iter(col as u8) {
+				Ok(iter) => iter,
+				Err(e) => return Box::new(std::iter::once(map_err(Err(e)))),
+			};
 			Box::new(std::iter::from_fn(move || {
-				if let Some((key, value)) = handle_err(iter.next()) {
-					Some((key.into_boxed_slice(), value.into_boxed_slice()))
-				} else {
-					None
-				}
+				iter.next().transpose().map(|r| map_err(r.map(|(k, v)| (k.into(), v))))
 			}))
 		}
 
@@ -198,39 +196,26 @@ pub mod paritydb_impl {
 			&'a self,
 			col: u32,
 			prefix: &'a [u8],
-		) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
+		) -> Box<dyn Iterator<Item = Result<DBKeyValue>> + 'a> {
 			if prefix.len() == 0 {
 				return self.iter(col)
 			}
-			let mut iter = handle_err(self.db.iter(col as u8));
-			handle_err(iter.seek(prefix));
+			let mut iter = match self.db.iter(col as u8) {
+				Ok(iter) => iter,
+				Err(e) => return Box::new(std::iter::once(map_err(Err(e)))),
+			};
+			if let Err(e) = iter.seek(prefix) {
+				return Box::new(std::iter::once(map_err(Err(e))))
+			}
 			Box::new(std::iter::from_fn(move || {
-				if let Some((key, value)) = handle_err(iter.next()) {
-					key.starts_with(prefix)
-						.then(|| (key.into_boxed_slice(), value.into_boxed_slice()))
-				} else {
-					None
-				}
+				iter.next().transpose().and_then(|r| {
+					map_err(r.map(|(k, v)| k.starts_with(prefix).then(|| (k.into(), v))))
+						.transpose()
+				})
 			}))
 		}
 
-		fn restore(&self, _new_db: &str) -> Result<()> {
-			unimplemented!("restore is unsupported")
-		}
-
-		fn io_stats(&self, _kind: IoStatsKind) -> IoStats {
-			unimplemented!("io_stats not supported by parity_db");
-		}
-
-		fn has_key(&self, col: u32, key: &[u8]) -> Result<bool> {
-			map_err(self.db.get_size(col as u8, key).map(|r| r.is_some()))
-		}
-
-		fn has_prefix(&self, col: u32, prefix: &[u8]) -> bool {
-			self.get_by_prefix(col, prefix).is_some()
-		}
-
-		fn write(&self, transaction: DBTransaction) -> std::io::Result<()> {
+		fn write(&self, transaction: DBTransaction) -> Result<()> {
 			let mut ops = transaction.ops.into_iter();
 			// TODO using a key iterator or native delete here would be faster.
 			let mut current_prefix_iter: Option<(parity_db::BTreeIterator, u8, Vec<u8>)> = None;

polkadot/parachain/Cargo.toml

@@ -10,7 +10,7 @@ edition = "2021"
 # this crate for WASM. This is critical to avoid forcing all parachain WASM into implementing
 # various unnecessary Substrate-specific endpoints.
 parity-scale-codec = { version = "3.1.5", default-features = false, features = [ "derive" ] }
-parity-util-mem = { version = "0.11.0", default-features = false, optional = true }
+parity-util-mem = { version = "0.12.0", default-features = false, optional = true }
 scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
 sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }

polkadot/primitives/Cargo.toml

@@ -27,7 +27,7 @@ trie = { package = "sp-trie", git = "https://github.com/paritytech/substrate", b
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 hex-literal = "0.3.4"
-parity-util-mem = { version = "0.11.0", default-features = false, optional = true }
+parity-util-mem = { version = "0.12.0", default-features = false, optional = true }
 
 [features]
 default = ["std"]