diff --git a/polkadot/Cargo.lock b/polkadot/Cargo.lock index c107d3399d..dee985c5d6 100644 --- a/polkadot/Cargo.lock +++ b/polkadot/Cargo.lock @@ -124,7 +124,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" dependencies = [ - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -169,8 +169,8 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d0864d84b8e07b145449be9a8537db86bf9de5ce03b913214694643b4743502" dependencies = [ - "quote 1.0.7", - "syn 1.0.60", + "quote", + "syn", ] [[package]] @@ -316,9 +316,9 @@ version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -436,8 +436,8 @@ dependencies = [ "lazycell", "log", "peeking_take_while", - "proc-macro2 1.0.24", - "quote 1.0.7", + "proc-macro2", + "quote", "regex", "rustc-hash", "shlex", @@ -733,7 +733,7 @@ dependencies = [ "js-sys", "libc", "num-integer", - "num-traits 0.2.14", + "num-traits", "time", "wasm-bindgen", "winapi 0.3.9", @@ -1090,8 +1090,8 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fbaabec2c953050352311293be5c6aba8e141ba19d6811862b232d6fd020484" dependencies = [ - "quote 1.0.7", - "syn 1.0.60", + "quote", + "syn", ] [[package]] @@ -1154,7 +1154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f83e699727abca3c56e187945f303389590305ab2f0185ea445aa66e8d5f2a" dependencies = [ "data-encoding", - "syn 1.0.60", + "syn", ] [[package]] @@ -1163,9 +1163,9 @@ version = "0.99.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1274,9 +1274,9 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "558e40ea573c374cf53507fd240b7ee2f5477df7cfebdb97323ec61c719399c5" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1314,15 +1314,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" -[[package]] -name = "enum_primitive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" -dependencies = [ - "num-traits 0.1.43", -] - [[package]] name = "enumflags2" version = "0.6.4" @@ -1338,9 +1329,9 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1473,9 +1464,9 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", "synstructure", ] @@ -1545,7 +1536,7 @@ dependencies = [ "futures 0.3.12", "futures-timer 3.0.2", "log", - "num-traits 0.2.14", + "num-traits", "parity-scale-codec", "parking_lot 0.11.1", ] @@ -1706,9 +1697,9 @@ source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a dependencies = [ "Inflector", "frame-support-procedural-tools", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1718,9 +1709,9 @@ source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1728,9 +1719,9 @@ name = "frame-support-procedural-tools-derive" version = "3.0.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1921,9 +1912,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2490,9 +2481,9 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2665,9 +2656,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99a847f9ec7bb52149b2786a17c9cb260d6effc6b8eeb8c16b343a487a7563a3" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -3019,8 +3010,8 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4bc40943156e42138d22ed3c57ff0e1a147237742715937622a99b10fbe0156" dependencies = [ - "quote 1.0.7", - "syn 1.0.60", + "quote", + "syn", ] [[package]] @@ -3566,15 +3557,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" -[[package]] -name = "memrange" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc29ba65898edc4fdc252cb31cd3925f37c1a8ba25bb46eec883569984976530" -dependencies = [ - "rustc-serialize", -] - [[package]] name = "merlin" version = "2.0.0" @@ -3622,9 +3604,9 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e071b3159835ee91df62dbdbfdd7ec366b7ea77c838f43aff4acda6b61bcfb9" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -3754,9 +3736,9 @@ checksum = "2f5653449cd45d502a53480ee08d7a599e8f4893d2bacb33c63d65bc20af6c1a" dependencies = [ "proc-macro-crate", "proc-macro-error", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", "synstructure", ] @@ -3791,7 +3773,7 @@ dependencies = [ "matrixmultiply", "num-complex", "num-rational", - "num-traits 0.2.14", + "num-traits", "rand 0.7.3", "rand_distr", "simba", @@ -3828,20 +3810,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "nix" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7fd5681d13fda646462cfbd4e5f2051279a89a544d50eb98c365b507246839f" -dependencies = [ - "bitflags", - "bytes 0.4.12", - "cfg-if 0.1.10", - "gcc", - "libc", - "void", -] - [[package]] name = "nix" version = "0.19.1" @@ -3884,7 +3852,7 @@ checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -3894,7 +3862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ "autocfg", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -3904,7 +3872,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ "autocfg", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -3916,16 +3884,7 @@ dependencies = [ "autocfg", "num-bigint", "num-integer", - "num-traits 0.2.14", -] - -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -dependencies = [ - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -4003,7 +3962,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3741934be594d77de1c8461ebcbbe866f585ea616a9753aa78f2bdc69f0e4579" dependencies = [ - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -4451,9 +4410,9 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -4648,9 +4607,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9029e65297c7fd6d7013f0579e193ec2b34ae78eabca854c9417504ad8a2d214" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -4703,8 +4662,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" dependencies = [ - "proc-macro2 1.0.24", - "syn 1.0.60", + "proc-macro2", + "syn", "synstructure", ] @@ -4918,9 +4877,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -4968,9 +4927,9 @@ version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -4979,9 +4938,9 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -5021,7 +4980,7 @@ dependencies = [ "assert_cmd", "color-eyre", "futures 0.3.12", - "nix 0.19.1", + "nix", "parity-util-mem", "polkadot-cli", "polkadot-service", @@ -5602,11 +5561,13 @@ version = "0.8.28" dependencies = [ "derive_more", "futures 0.3.12", + "libc", "log", "parity-scale-codec", "parity-util-mem", "parking_lot 0.11.1", "polkadot-core-primitives", + "raw_sync", "sc-executor", "serde", "shared_memory", @@ -5616,6 +5577,7 @@ dependencies = [ "sp-runtime", "sp-std", "sp-wasm-interface", + "static_assertions", "thiserror", ] @@ -6227,9 +6189,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", "version_check", ] @@ -6239,8 +6201,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", + "proc-macro2", + "quote", "version_check", ] @@ -6256,22 +6218,13 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] - [[package]] name = "proc-macro2" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid 0.2.1", + "unicode-xid", ] [[package]] @@ -6324,9 +6277,9 @@ checksum = "169a15f3008ecb5160cba7d37bcd690a7601b6d30cfb87a117d45e59d52af5d4" dependencies = [ "anyhow", "itertools", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -6382,22 +6335,13 @@ dependencies = [ "pin-project-lite 0.1.7", ] -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - [[package]] name = "quote" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2", ] [[package]] @@ -6555,6 +6499,19 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "raw_sync" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a34bde3561f980a51c70495164200569a11662644fe5af017f0b5d7015688cc" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "nix", + "rand 0.8.3", + "winapi 0.3.9", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -6655,9 +6612,9 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d21b475ab879ef0e315ad99067fa25778c3b0377f57f1b00207448dac1a3144" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -6853,12 +6810,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" -[[package]] -name = "rustc-serialize" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" - [[package]] name = "rustc_version" version = "0.2.3" @@ -7045,9 +6996,9 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -7176,7 +7127,7 @@ dependencies = [ "merlin", "num-bigint", "num-rational", - "num-traits 0.2.14", + "num-traits", "parity-scale-codec", "parking_lot 0.11.1", "pdqselect", @@ -7435,7 +7386,7 @@ dependencies = [ "derive_more", "futures 0.3.12", "log", - "num-traits 0.2.14", + "num-traits", "parity-scale-codec", "parking_lot 0.11.1", "prost", @@ -7852,9 +7803,9 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -7961,9 +7912,9 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e367622f934864ffa1c704ba2b82280aab856e3d8213c84c5720257eb34b15b9" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -8085,9 +8036,9 @@ version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -8162,34 +8113,18 @@ dependencies = [ [[package]] name = "shared_memory" -version = "0.10.0" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3ab0cdff84d6c66fc9e268010ea6508e58ee942575afb66f2cf194bb218bb4" +checksum = "b854a362375dfe8ab12ea8a98228040d37293c988f85fbac9fa0f83336387966" dependencies = [ "cfg-if 0.1.10", - "enum_primitive", "libc", - "log", - "memrange", - "nix 0.10.0", - "quick-error 1.2.3", - "rand 0.4.6", - "shared_memory_derive", - "theban_interval_tree", + "nix", + "quick-error 2.0.0", + "rand 0.8.3", "winapi 0.3.9", ] -[[package]] -name = "shared_memory_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "767a14f1304be2f0b04e69860252f8ae9cfae0afaa9cc07b675147c43425dd3a" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "syn 0.15.44", -] - [[package]] name = "shlex" version = "0.1.1" @@ -8230,7 +8165,7 @@ checksum = "fb931b1367faadea6b1ab1c306a860ec17aaa5fa39f367d0c744e69d971a1fb2" dependencies = [ "approx", "num-complex", - "num-traits 0.2.14", + "num-traits", "paste 0.1.18", ] @@ -8336,9 +8271,9 @@ source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a dependencies = [ "blake2-rfc", "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -8359,7 +8294,7 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ "integer-sqrt", - "num-traits 0.2.14", + "num-traits", "parity-scale-codec", "serde", "sp-debug-derive", @@ -8515,7 +8450,7 @@ dependencies = [ "libsecp256k1", "log", "merlin", - "num-traits 0.2.14", + "num-traits", "parity-scale-codec", "parity-util-mem", "parking_lot 0.11.1", @@ -8554,9 +8489,9 @@ name = "sp-debug-derive" version = "3.0.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -8670,9 +8605,9 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -8747,9 +8682,9 @@ source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a dependencies = [ "Inflector", "proc-macro-crate", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -8791,7 +8726,7 @@ source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a dependencies = [ "hash-db", "log", - "num-traits 0.2.14", + "num-traits", "parity-scale-codec", "parking_lot 0.11.1", "rand 0.7.3", @@ -9011,9 +8946,9 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -9032,9 +8967,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149" dependencies = [ "heck", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -9165,8 +9100,8 @@ version = "0.9.0" source = "git+https://github.com/paritytech/substrate#e03ca38d45f438932ec92bf69a40b6b16b6ec643" dependencies = [ "proc-macro-crate", - "quote 1.0.7", - "syn 1.0.60", + "quote", + "syn", ] [[package]] @@ -9197,26 +9132,15 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid 0.1.0", -] - [[package]] name = "syn" version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "unicode-xid 0.2.1", + "proc-macro2", + "quote", + "unicode-xid", ] [[package]] @@ -9225,10 +9149,10 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", - "unicode-xid 0.2.1", + "proc-macro2", + "quote", + "syn", + "unicode-xid", ] [[package]] @@ -9340,17 +9264,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "theban_interval_tree" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b42a5385db9a651628091edcd1d58ac9cb1c92327d8cd2a29bf8e35bdfe4ea" -dependencies = [ - "memrange", - "rand 0.3.23", - "time", -] - [[package]] name = "thiserror" version = "1.0.23" @@ -9366,9 +9279,9 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -9562,9 +9475,9 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -9754,9 +9667,9 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -9937,12 +9850,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode-xid" version = "0.2.1" @@ -10116,9 +10023,9 @@ dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-shared", ] @@ -10140,7 +10047,7 @@ version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" dependencies = [ - "quote 1.0.7", + "quote", "wasm-bindgen-macro-support", ] @@ -10150,9 +10057,9 @@ version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10198,7 +10105,7 @@ dependencies = [ "libc", "memory_units", "num-rational", - "num-traits 0.2.14", + "num-traits", "parity-wasm 0.41.0", "wasmi-validation", ] @@ -10703,9 +10610,9 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.60", + "proc-macro2", + "quote", + "syn", "synstructure", ] diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 9ff4a7ed5a..f6ca38437b 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -445,9 +445,9 @@ fn validate_candidate_exhaustive( match B::validate(backend_arg, &validation_code, params, spawn) { Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::Timeout)) => Ok(ValidationResult::Invalid(InvalidCandidate::Timeout)), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::ParamsTooLarge(l))) => + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::ParamsTooLarge(l, _))) => Ok(ValidationResult::Invalid(InvalidCandidate::ParamsTooLarge(l as u64))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::CodeTooLarge(l))) => + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::CodeTooLarge(l, _))) => Ok(ValidationResult::Invalid(InvalidCandidate::CodeTooLarge(l as u64))), Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::BadReturn)) => Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn)), diff --git a/polkadot/parachain/Cargo.toml b/polkadot/parachain/Cargo.toml index 232d2d322a..f1dd0076a2 100644 --- a/polkadot/parachain/Cargo.toml +++ b/polkadot/parachain/Cargo.toml @@ -27,9 +27,12 @@ sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", op parking_lot = { version = "0.11.1", optional = true } log = { version = "0.4.11", optional = true } futures = { version = "0.3.8", optional = true } +static_assertions = { version = "1.1", optional = true } +libc = { version = "0.2.81", optional = true } [target.'cfg(not(any(target_os = "android", target_os = "unknown")))'.dependencies] -shared_memory = { version = "0.10.0", optional = true } +shared_memory = { version = "0.11.0", optional = true } +raw_sync = { version = "0.1", optional = true } [features] default = ["std"] @@ -42,9 +45,12 @@ std = [ "sp-std/std", "sp-runtime/std", "shared_memory", + "raw_sync", "sp-core/std", "parking_lot", + "static_assertions", "log", + "libc", "parity-util-mem", "sp-externalities", "sc-executor", diff --git a/polkadot/parachain/src/wasm_executor/mod.rs b/polkadot/parachain/src/wasm_executor/mod.rs index fe44f41513..3e8fe6d380 100644 --- a/polkadot/parachain/src/wasm_executor/mod.rs +++ b/polkadot/parachain/src/wasm_executor/mod.rs @@ -32,11 +32,6 @@ pub use validation_host::{run_worker, ValidationPool, EXECUTION_TIMEOUT_SEC, WOR mod validation_host; -// maximum memory in bytes -const MAX_RUNTIME_MEM: usize = 1024 * 1024 * 1024; // 1 GiB -const MAX_CODE_MEM: usize = 16 * 1024 * 1024; // 16 MiB -const MAX_VALIDATION_RESULT_HEADER_MEM: usize = MAX_CODE_MEM + 1024; // 16.001 MiB - /// The strategy we employ for isolating execution of wasm parachain validation function (PVF). /// /// For a typical validator an external process is the default way to run PVF. The rationale is based @@ -126,11 +121,11 @@ pub enum InvalidCandidate { #[error("WASM executor error")] WasmExecutor(#[from] sc_executor::error::Error), /// Call data is too large. - #[error("Validation parameters are {0} bytes, max allowed is {}", MAX_RUNTIME_MEM)] - ParamsTooLarge(usize), + #[error("Validation parameters are {0} bytes, max allowed is {1}")] + ParamsTooLarge(usize, usize), /// Code size it too large. - #[error("WASM code is {0} bytes, max allowed is {}", MAX_CODE_MEM)] - CodeTooLarge(usize), + #[error("WASM code is {0} bytes, max allowed is {1}")] + CodeTooLarge(usize, usize), /// Error decoding returned data. #[error("Validation function returned invalid data.")] BadReturn, @@ -156,8 +151,20 @@ pub enum InternalError { System(#[from] Box), #[cfg(not(any(target_os = "android", target_os = "unknown")))] - #[error("Shared memory error: {0}")] - SharedMem(#[from] shared_memory::SharedMemError), + #[error("Failed to create shared memory: {0}")] + WorkerStartTimeout(String), + + #[cfg(not(any(target_os = "android", target_os = "unknown")))] + #[error("Failed to create shared memory: {0}")] + FailedToCreateSharedMemory(String), + + #[cfg(not(any(target_os = "android", target_os = "unknown")))] + #[error("Failed to send a singal to worker: {0}")] + FailedToSignal(String), + + #[cfg(not(any(target_os = "android", target_os = "unknown")))] + #[error("Failed to send data to worker: {0}")] + FailedToWriteData(&'static str), #[error("WASM worker error: {0}")] WasmWorker(String), diff --git a/polkadot/parachain/src/wasm_executor/validation_host.rs b/polkadot/parachain/src/wasm_executor/validation_host.rs index cd58ec5971..9f466b82c5 100644 --- a/polkadot/parachain/src/wasm_executor/validation_host.rs +++ b/polkadot/parachain/src/wasm_executor/validation_host.rs @@ -16,14 +16,9 @@ #![cfg(not(any(target_os = "android", target_os = "unknown")))] -use std::{process, env, sync::Arc, sync::atomic, path::PathBuf}; -use parity_scale_codec::{Decode, Encode}; +use std::{env, path::PathBuf, process, sync::Arc, sync::atomic}; use crate::primitives::{ValidationParams, ValidationResult}; -use super::{ - validate_candidate_internal, ValidationError, InvalidCandidate, InternalError, - MAX_CODE_MEM, MAX_RUNTIME_MEM, MAX_VALIDATION_RESULT_HEADER_MEM, -}; -use shared_memory::{SharedMem, SharedMemConf, EventState, WriteLockable, EventWait, EventSet}; +use super::{validate_candidate_internal, ValidationError, InvalidCandidate, InternalError}; use parking_lot::Mutex; use log::{debug, trace}; use futures::executor::ThreadPool; @@ -35,18 +30,14 @@ pub const WORKER_ARGS: &[&'static str] = &[WORKER_ARG]; const LOG_TARGET: &'static str = "validation-worker"; +mod workspace; + /// Execution timeout in seconds; #[cfg(debug_assertions)] -pub const EXECUTION_TIMEOUT_SEC: u64 = 30; +pub const EXECUTION_TIMEOUT_SEC: u64 = 30; #[cfg(not(debug_assertions))] -pub const EXECUTION_TIMEOUT_SEC: u64 = 5; - -enum Event { - CandidateReady = 0, - ResultReady = 1, - WorkerReady = 2, -} +pub const EXECUTION_TIMEOUT_SEC: u64 = 5; #[derive(Clone)] struct TaskExecutor(ThreadPool); @@ -99,16 +90,14 @@ impl ValidationPool { let worker_cli_args = match cache_base_path { Some(cache_base_path) => { - let worker_cli_args: Vec<&str> = - WORKER_ARGS.into_iter() - .cloned() - .chain(iter::once(cache_base_path)) - .collect(); + let worker_cli_args: Vec<&str> = WORKER_ARGS + .into_iter() + .cloned() + .chain(iter::once(cache_base_path)) + .collect(); Cow::from(worker_cli_args) } - None => { - Cow::from(WORKER_ARGS) - }, + None => Cow::from(WORKER_ARGS), }; self.validate_candidate_custom( @@ -133,24 +122,31 @@ impl ValidationPool { ) -> Result { for host in self.hosts.iter() { if let Some(mut host) = host.try_lock() { - return host.validate_candidate(validation_code, params, command, args) + return host.validate_candidate(validation_code, params, command, args); } } // all workers are busy, just wait for the first one - self.hosts[0].lock().validate_candidate(validation_code, params, command, args) + self.hosts[0] + .lock() + .validate_candidate(validation_code, params, command, args) } } /// Validation worker process entry point. Runs a loop waiting for candidates to validate /// and sends back results via shared memory. pub fn run_worker(mem_id: &str, cache_base_path: Option) -> Result<(), String> { - let mut memory = match SharedMem::open(mem_id) { - Ok(memory) => memory, + let mut worker_handle = match workspace::open(mem_id) { Err(e) => { - debug!(target: LOG_TARGET, "{} Error opening shared memory: {:?}", process::id(), e); - return Err(format!("Error opening shared memory: {:?}", e)); + debug!( + target: LOG_TARGET, + "{} Error opening shared memory: {:?}", + process::id(), + e + ); + return Err(e); } + Ok(h) => h, }; let exit = Arc::new(atomic::AtomicBool::new(false)); @@ -162,12 +158,15 @@ pub fn run_worker(mem_id: &str, cache_base_path: Option) -> Result<(), let mut in_data = Vec::new(); // pipe terminates when parent process exits std::io::stdin().read_to_end(&mut in_data).ok(); - debug!(target: LOG_TARGET, "{} Parent process is dead. Exiting", process::id()); + debug!( + target: LOG_TARGET, + "{} Parent process is dead. Exiting", + process::id() + ); exit.store(true, atomic::Ordering::Relaxed); }); - memory.set(Event::WorkerReady as usize, EventState::Signaled) - .map_err(|e| format!("{} Error setting shared event: {:?}", process::id(), e))?; + worker_handle.signal_ready()?; let executor = super::ExecutorCache::new(cache_base_path); @@ -176,102 +175,64 @@ pub fn run_worker(mem_id: &str, cache_base_path: Option) -> Result<(), break; } - debug!(target: LOG_TARGET, "{} Waiting for candidate", process::id()); - match memory.wait(Event::CandidateReady as usize, shared_memory::Timeout::Sec(3)) { - Err(e) => { - // Timeout - trace!(target: LOG_TARGET, "{} Timeout waiting for candidate: {:?}", process::id(), e); + debug!( + target: LOG_TARGET, + "{} Waiting for candidate", + process::id() + ); + let work_item = match worker_handle.wait_for_work(3) { + Err(workspace::WaitForWorkErr::Wait(e)) => { + trace!( + target: LOG_TARGET, + "{} Timeout waiting for candidate: {:?}", + process::id(), + e + ); continue; } - Ok(()) => {} - } + Err(workspace::WaitForWorkErr::FailedToDecode(e)) => { + return Err(e); + } + Ok(work_item) => work_item, + }; - { - debug!(target: LOG_TARGET, "{} Processing candidate", process::id()); - // we have candidate data - let mut slice = memory.wlock_as_slice(0) - .map_err(|e| format!("Error locking shared memory: {:?}", e))?; + debug!(target: LOG_TARGET, "{} Processing candidate", process::id()); + let result = validate_candidate_internal( + &executor, + work_item.code, + work_item.params, + task_executor.clone(), + ); - let result = { - let data: &mut[u8] = &mut **slice; - let (header_buf, rest) = data.split_at_mut(1024); - let mut header_buf: &[u8] = header_buf; - let header = ValidationHeader::decode(&mut header_buf) - .map_err(|_| format!("Error decoding validation request."))?; - debug!(target: LOG_TARGET, "{} Candidate header: {:?}", process::id(), header); - let (code, rest) = rest.split_at_mut(MAX_CODE_MEM); - let (code, _) = code.split_at_mut(header.code_size as usize); - let (call_data, _) = rest.split_at_mut(MAX_RUNTIME_MEM); - let (call_data, _) = call_data.split_at_mut(header.params_size as usize); + debug!( + target: LOG_TARGET, + "{} Candidate validated: {:?}", + process::id(), + result + ); - let result = validate_candidate_internal(&executor, code, call_data, task_executor.clone()); - debug!(target: LOG_TARGET, "{} Candidate validated: {:?}", process::id(), result); - - match result { - Ok(r) => ValidationResultHeader::Ok(r), - Err(ValidationError::Internal(e)) => - ValidationResultHeader::Error(WorkerValidationError::InternalError(e.to_string())), - Err(ValidationError::InvalidCandidate(e)) => - ValidationResultHeader::Error(WorkerValidationError::ValidationError(e.to_string())), - } - }; - let mut data: &mut[u8] = &mut **slice; - result.encode_to(&mut data); - } - debug!(target: LOG_TARGET, "{} Signaling result", process::id()); - memory.set(Event::ResultReady as usize, EventState::Signaled) - .map_err(|e| format!("Error setting shared event: {:?}", e))?; + let result_header = match result { + Ok(r) => workspace::ValidationResultHeader::Ok(r), + Err(ValidationError::Internal(e)) => workspace::ValidationResultHeader::Error( + workspace::WorkerValidationError::InternalError(e.to_string()), + ), + Err(ValidationError::InvalidCandidate(e)) => workspace::ValidationResultHeader::Error( + workspace::WorkerValidationError::ValidationError(e.to_string()), + ), + }; + worker_handle + .report_result(result_header) + .map_err(|e| format!("error reporting result: {:?}", e))?; } Ok(()) } -/// Params header in shared memory. All offsets should be aligned to WASM page size. -#[derive(Encode, Decode, Debug)] -struct ValidationHeader { - code_size: u64, - params_size: u64, -} - -#[derive(Encode, Decode, Debug)] -enum WorkerValidationError { - InternalError(String), - ValidationError(String), -} - -#[derive(Encode, Decode, Debug)] -enum ValidationResultHeader { - Ok(ValidationResult), - Error(WorkerValidationError), -} - unsafe impl Send for ValidationHost {} -struct ValidationHostMemory(SharedMem); - -impl std::fmt::Debug for ValidationHostMemory { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "ValidationHostMemory") - } -} - -impl std::ops::Deref for ValidationHostMemory { - type Target = SharedMem; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl std::ops::DerefMut for ValidationHostMemory { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - #[derive(Default, Debug)] struct ValidationHost { worker: Option, - memory: Option, + host_handle: Option, id: u32, } @@ -284,18 +245,6 @@ impl Drop for ValidationHost { } impl ValidationHost { - fn create_memory() -> Result { - let mem_size = MAX_RUNTIME_MEM + MAX_CODE_MEM + MAX_VALIDATION_RESULT_HEADER_MEM; - let mem_config = SharedMemConf::default() - .set_size(mem_size) - .add_lock(shared_memory::LockType::Mutex, 0, mem_size)? - .add_event(shared_memory::EventType::Auto)? // Event::CandidateReady - .add_event(shared_memory::EventType::Auto)? // Event::ResultReady - .add_event(shared_memory::EventType::Auto)?; // Event::WorkerReady - - Ok(mem_config.create()?) - } - fn start_worker(&mut self, cmd: &PathBuf, args: &[&str]) -> Result<(), InternalError> { if let Some(ref mut worker) = self.worker { // Check if still alive @@ -305,28 +254,28 @@ impl ValidationHost { } } - let memory = Self::create_memory()?; + let host_handle = + workspace::create().map_err(|msg| InternalError::FailedToCreateSharedMemory(msg))?; debug!( target: LOG_TARGET, "Starting worker at {:?} with arguments: {:?} and {:?}", cmd, args, - memory.get_os_path(), + host_handle.id(), ); let worker = process::Command::new(cmd) .args(args) - .arg(memory.get_os_path()) + .arg(host_handle.id()) .stdin(process::Stdio::piped()) .spawn()?; self.id = worker.id(); self.worker = Some(worker); - memory.wait( - Event::WorkerReady as usize, - shared_memory::Timeout::Sec(EXECUTION_TIMEOUT_SEC as usize), - )?; - self.memory = Some(ValidationHostMemory(memory)); + host_handle + .wait_until_ready(EXECUTION_TIMEOUT_SEC) + .map_err(|e| InternalError::WorkerStartTimeout(format!("{:?}", e)))?; + self.host_handle = Some(host_handle); Ok(()) } @@ -340,76 +289,68 @@ impl ValidationHost { binary: &PathBuf, args: &[&str], ) -> Result { - if validation_code.len() > MAX_CODE_MEM { - return Err(ValidationError::InvalidCandidate(InvalidCandidate::CodeTooLarge(validation_code.len()))); - } // First, check if need to spawn the child process self.start_worker(binary, args)?; - let memory = self.memory.as_mut() - .expect("memory is always `Some` after `start_worker` completes successfully"); - { - // Put data in shared mem - let data: &mut[u8] = &mut **memory.wlock_as_slice(0) - .map_err(|e|ValidationError::Internal(e.into()))?; - let (mut header_buf, rest) = data.split_at_mut(1024); - let (code, rest) = rest.split_at_mut(MAX_CODE_MEM); - let (code, _) = code.split_at_mut(validation_code.len()); - let (call_data, _) = rest.split_at_mut(MAX_RUNTIME_MEM); - code[..validation_code.len()].copy_from_slice(validation_code); - let encoded_params = params.encode(); - if encoded_params.len() >= MAX_RUNTIME_MEM { - return Err(ValidationError::InvalidCandidate(InvalidCandidate::ParamsTooLarge(MAX_RUNTIME_MEM))); - } - call_data[..encoded_params.len()].copy_from_slice(&encoded_params); - let header = ValidationHeader { - code_size: validation_code.len() as u64, - params_size: encoded_params.len() as u64, - }; - - header.encode_to(&mut header_buf); - } + let host_handle = self + .host_handle + .as_mut() + .expect("host_handle is always `Some` after `start_worker` completes successfully"); debug!(target: LOG_TARGET, "{} Signaling candidate", self.id); - memory.set(Event::CandidateReady as usize, EventState::Signaled) - .map_err(|e| ValidationError::Internal(e.into()))?; + match host_handle.request_validation(validation_code, params) { + Ok(()) => {} + Err(workspace::RequestValidationErr::CodeTooLarge { actual, max }) => { + return Err(ValidationError::InvalidCandidate( + InvalidCandidate::CodeTooLarge(actual, max), + )); + } + Err(workspace::RequestValidationErr::ParamsTooLarge { actual, max }) => { + return Err(ValidationError::InvalidCandidate( + InvalidCandidate::ParamsTooLarge(actual, max), + )); + } + Err(workspace::RequestValidationErr::Signal(msg)) => { + return Err(ValidationError::Internal(InternalError::FailedToSignal(msg))); + } + Err(workspace::RequestValidationErr::WriteData(msg)) => { + return Err(ValidationError::Internal(InternalError::FailedToWriteData(msg))); + } + } debug!(target: LOG_TARGET, "{} Waiting for results", self.id); - match memory.wait(Event::ResultReady as usize, shared_memory::Timeout::Sec(EXECUTION_TIMEOUT_SEC as usize)) { - Err(e) => { - debug!(target: LOG_TARGET, "Worker timeout: {:?}", e); + let result_header = match host_handle.wait_for_result(EXECUTION_TIMEOUT_SEC) { + Ok(inner_result) => inner_result, + Err(assumed_timeout) => { + debug!(target: LOG_TARGET, "Worker timeout: {:?}", assumed_timeout); if let Some(mut worker) = self.worker.take() { worker.kill().ok(); } return Err(ValidationError::InvalidCandidate(InvalidCandidate::Timeout)); } - Ok(()) => {} - } + }; - { - debug!(target: LOG_TARGET, "{} Reading results", self.id); - let data: &[u8] = &**memory.wlock_as_slice(0) - .map_err(|e| ValidationError::Internal(e.into()))?; - let (header_buf, _) = data.split_at(MAX_VALIDATION_RESULT_HEADER_MEM); - let mut header_buf: &[u8] = header_buf; - let header = ValidationResultHeader::decode(&mut header_buf) - .map_err(|e| - InternalError::System( - Box::::from( - format!("Failed to decode `ValidationResultHeader`: {:?}", e) - ) as Box<_> - ) - )?; - match header { - ValidationResultHeader::Ok(result) => Ok(result), - ValidationResultHeader::Error(WorkerValidationError::InternalError(e)) => { - debug!(target: LOG_TARGET, "{} Internal validation error: {}", self.id, e); - Err(ValidationError::Internal(InternalError::WasmWorker(e))) - }, - ValidationResultHeader::Error(WorkerValidationError::ValidationError(e)) => { - debug!(target: LOG_TARGET, "{} External validation error: {}", self.id, e); - Err(ValidationError::InvalidCandidate(InvalidCandidate::ExternalWasmExecutor(e))) - } + match result_header { + workspace::ValidationResultHeader::Ok(result) => Ok(result), + workspace::ValidationResultHeader::Error( + workspace::WorkerValidationError::InternalError(e), + ) => { + debug!( + target: LOG_TARGET, + "{} Internal validation error: {}", self.id, e + ); + Err(ValidationError::Internal(InternalError::WasmWorker(e))) + } + workspace::ValidationResultHeader::Error( + workspace::WorkerValidationError::ValidationError(e), + ) => { + debug!( + target: LOG_TARGET, + "{} External validation error: {}", self.id, e + ); + Err(ValidationError::InvalidCandidate( + InvalidCandidate::ExternalWasmExecutor(e), + )) } } } diff --git a/polkadot/parachain/src/wasm_executor/validation_host/workspace.rs b/polkadot/parachain/src/wasm_executor/validation_host/workspace.rs new file mode 100644 index 0000000000..a65c2d6597 --- /dev/null +++ b/polkadot/parachain/src/wasm_executor/validation_host/workspace.rs @@ -0,0 +1,556 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! This module implements a "workspace" - basically a wrapper around a shared memory that +//! is used as an IPC channel for communication between the validation host and it's validation +//! worker. + +use crate::primitives::{ValidationParams, ValidationResult}; +use super::LOG_TARGET; +use parity_scale_codec::{Decode, Encode}; +use raw_sync::{ + events::{Event, EventImpl, EventInit, EventState}, + Timeout, +}; +use shared_memory::{Shmem, ShmemConf}; +use std::{ + error::Error, + fmt, + io::{Cursor, Write}, + slice, + sync::atomic::AtomicBool, + time::Duration, +}; + +// maximum memory in bytes +const MAX_PARAMS_MEM: usize = 1024 * 1024; // 1 MiB +const MAX_CODE_MEM: usize = 16 * 1024 * 1024; // 16 MiB +const MAX_VALIDATION_RESULT_HEADER_MEM: usize = MAX_CODE_MEM + 1024; // 16.001 MiB + +/// Params header in shared memory. All offsets should be aligned to WASM page size. +#[derive(Encode, Decode, Debug)] +struct ValidationHeader { + code_size: u64, + params_size: u64, +} + +/// An error that could happen during validation of a candidate. +#[derive(Encode, Decode, Debug)] +pub enum WorkerValidationError { + InternalError(String), + ValidationError(String), +} + +/// An enum that is used to marshal a validation result in order to pass it through the shared memory. +#[derive(Encode, Decode, Debug)] +pub enum ValidationResultHeader { + Ok(ValidationResult), + Error(WorkerValidationError), +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +enum Mode { + Initialize, + Attach, +} + +fn stringify_err(err: Box) -> String { + format!("{:?}", err) +} + +struct Inner { + shmem: Shmem, + candidate_ready_ev: Box, + result_ready_ev: Box, + worker_ready_ev: Box, + + /// Flag that indicates that the worker side is attached to this workspace. + /// + /// While there are apparent problems attaching multiple workers to the same workspace, we don't + /// need that anyway. So to make our reasoning a little bit simpler just add a flag and check + /// it before attaching. + attached: *mut AtomicBool, + + /// The number of bytes reserved by the auxilary stuff like events from the beginning of the + /// shared memory area. + /// + /// We expect this to be way smaller than the whole shmem size. + consumed: usize, +} + +impl Inner { + fn layout(shmem: Shmem, mode: Mode) -> Self { + unsafe { + let base_ptr = shmem.as_ptr(); + let mut consumed = 0; + + let candidate_ready_ev = add_event(base_ptr, &mut consumed, mode); + let result_ready_ev = add_event(base_ptr, &mut consumed, mode); + let worker_ready_ev = add_event(base_ptr, &mut consumed, mode); + + // The size of AtomicBool is guaranteed to be the same as the bool, however, docs + // on the bool primitve doesn't actually state that the in-memory size is equal to 1 byte. + // + // AtomicBool requires hardware support of 1 byte width of atomic operations though, so + // that should be fine. + // + // We still assert here to be safe than sorry. + static_assertions::assert_eq_size!(AtomicBool, u8); + // SAFETY: `AtomicBool` is represented by an u8 thus will be happy to take any alignment. + let attached = base_ptr.add(consumed) as *mut AtomicBool; + consumed += 1; + + let consumed = align_up_to(consumed, 64); + + Self { + shmem, + attached, + consumed, + candidate_ready_ev, + result_ready_ev, + worker_ready_ev, + } + } + } + + fn as_slice(&self) -> &[u8] { + unsafe { + let base_ptr = self.shmem.as_ptr().add(self.consumed); + let remaining = self.shmem.len() - self.consumed; + slice::from_raw_parts(base_ptr, remaining) + } + } + + fn as_slice_mut(&mut self) -> &mut [u8] { + unsafe { + let base_ptr = self.shmem.as_ptr().add(self.consumed); + let remaining = self.shmem.len() - self.consumed; + slice::from_raw_parts_mut(base_ptr, remaining) + } + } + + /// Mark that this workspace has an attached worker already. Returning `true` means that this + /// was the first worker attached. + fn declare_exclusive_attached(&self) -> bool { + unsafe { + // If this succeeded then the value was `false`, thus, we managed to attach exclusively. + (&*self.attached) + .compare_exchange_weak( + false, + true, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) + .is_ok() + } + } +} + +fn align_up_to(v: usize, alignment: usize) -> usize { + ((v + alignment - 1) / alignment) * alignment +} + +/// Initializes a new or attaches to an exising event. +/// +/// # Safety +/// +/// This function should be called with the combination of `base_ptr` and `consumed` so that `base_ptr + consumed` +/// points on the memory area that is allocated and accessible. +/// +/// This function should be called only once for the same combination of the `base_ptr + consumed` and the mode. +/// Furthermore, this function should be called once for initialization. +/// +/// Specifically, `consumed` should not be modified by the caller, it should be passed as is to this function. +unsafe fn add_event(base_ptr: *mut u8, consumed: &mut usize, mode: Mode) -> Box { + // SAFETY: there is no safety proof since the documentation doesn't specify the particular constraints + // besides requiring the pointer to be valid. AFAICT, the pointer is valid. + let ptr = base_ptr.add(*consumed); + + const EXPECTATION: &str = + "given that the preconditions were fulfilled, the creation of the event should succeed"; + let (ev, used_bytes) = match mode { + Mode::Initialize => Event::new(ptr, true).expect(EXPECTATION), + Mode::Attach => Event::from_existing(ptr).expect(EXPECTATION), + }; + *consumed += used_bytes; + ev +} + +/// A message received by the worker that specifies a candidate validation work. +pub struct WorkItem<'handle> { + pub params: &'handle [u8], + pub code: &'handle [u8], +} + +/// An error that could be returned from [`WorkerHandle::wait_for_work`]. +#[derive(Debug)] +pub enum WaitForWorkErr { + /// An error occured during waiting for work. Typically a timeout. + Wait(String), + /// An error ocurred when trying to decode the validation request from the host. + FailedToDecode(String), +} + +/// An error that could be returned from [`WorkerHandle::report_result`]. +#[derive(Debug)] +pub enum ReportResultErr { + /// An error occured during signalling to the host that the result is ready. + Signal(String), +} + +/// A worker side handle to a workspace. +pub struct WorkerHandle { + inner: Inner, +} + +impl WorkerHandle { + /// Signals to the validation host that this worker is ready to accept new work requests. + pub fn signal_ready(&self) -> Result<(), String> { + self.inner + .worker_ready_ev + .set(EventState::Signaled) + .map_err(stringify_err)?; + Ok(()) + } + + /// Waits until a new piece of work. Returns `Err` if the work doesn't come within the given + /// timeout. + pub fn wait_for_work(&mut self, timeout_secs: u64) -> Result { + self.inner + .candidate_ready_ev + .wait(Timeout::Val(Duration::from_secs(timeout_secs))) + .map_err(stringify_err) + .map_err(WaitForWorkErr::Wait)?; + + let mut cur = self.inner.as_slice(); + let header = ValidationHeader::decode(&mut cur) + .map_err(|e| format!("{:?}", e)) + .map_err(WaitForWorkErr::FailedToDecode)?; + + let (params, cur) = cur.split_at(header.params_size as usize); + let (code, _) = cur.split_at(header.code_size as usize); + + Ok(WorkItem { params, code }) + } + + /// Report back the result of validation. + pub fn report_result(&mut self, result: ValidationResultHeader) -> Result<(), ReportResultErr> { + let mut cur = self.inner.as_slice_mut(); + result.encode_to(&mut cur); + self.inner + .result_ready_ev + .set(EventState::Signaled) + .map_err(stringify_err) + .map_err(ReportResultErr::Signal)?; + + Ok(()) + } +} + +/// An error that could be returned from [`HostHandle::wait_until_ready`]. +#[derive(Debug)] +pub enum WaitUntilReadyErr { + /// An error occured during waiting for the signal from the worker. + Wait(String), +} + +/// An error that could be returned from [`HostHandle::request_validation`]. +#[derive(Debug)] +pub enum RequestValidationErr { + /// The code passed exceeds the maximum allowed limit. + CodeTooLarge { actual: usize, max: usize }, + /// The call parameters exceed the maximum allowed limit. + ParamsTooLarge { actual: usize, max: usize }, + /// An error occured during writing either the code or the call params (the inner string specifies which) + WriteData(&'static str), + /// An error occured during signalling that the request is ready. + Signal(String), +} + +/// An error that could be returned from [`HostHandle::wait_for_result`] +#[derive(Debug)] +pub enum WaitForResultErr { + /// A error happened during waiting for the signal. Typically a timeout. + Wait(String), + /// Failed to decode the result header sent by the worker. + HeaderDecodeErr(String), +} + +/// A worker side handle to a workspace. +pub struct HostHandle { + inner: Inner, +} + +impl fmt::Debug for HostHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "HostHandle") + } +} + +impl HostHandle { + /// Returns the OS specific ID for this workspace. + pub fn id(&self) -> &str { + self.inner.shmem.get_os_id() + } + + /// Wait until the worker is online and ready for accepting validation requests. + pub fn wait_until_ready(&self, timeout_secs: u64) -> Result<(), WaitUntilReadyErr> { + self.inner + .worker_ready_ev + .wait(Timeout::Val(Duration::from_secs(timeout_secs))) + .map_err(stringify_err) + .map_err(WaitUntilReadyErr::Wait)?; + Ok(()) + } + + /// Request validation with the given code and parameters. + pub fn request_validation( + &mut self, + code: &[u8], + params: ValidationParams, + ) -> Result<(), RequestValidationErr> { + if code.len() > MAX_CODE_MEM { + return Err(RequestValidationErr::CodeTooLarge { + actual: code.len(), + max: MAX_CODE_MEM, + }); + } + + let params = params.encode(); + if params.len() > MAX_PARAMS_MEM { + return Err(RequestValidationErr::ParamsTooLarge { + actual: params.len(), + max: MAX_PARAMS_MEM, + }); + } + + let mut cur = Cursor::new(self.inner.as_slice_mut()); + ValidationHeader { + code_size: code.len() as u64, + params_size: params.len() as u64, + } + .encode_to(&mut cur); + cur.write_all(¶ms) + .map_err(|_| RequestValidationErr::WriteData("params"))?; + cur.write_all(code) + .map_err(|_| RequestValidationErr::WriteData("code"))?; + + self.inner + .candidate_ready_ev + .set(EventState::Signaled) + .map_err(stringify_err) + .map_err(RequestValidationErr::Signal)?; + + Ok(()) + } + + /// Wait for the validation result from the worker with the given timeout. + /// + /// Returns `Ok` if the response was received within the deadline or error otherwise. An error + /// could also occur because of failing decoding the result from the worker. Returning + /// `Ok` doesn't mean that the candidate was successfully validated though, for that the client + /// needs to inspect the returned validation result header. + pub fn wait_for_result( + &self, + execution_timeout: u64, + ) -> Result { + self.inner + .result_ready_ev + .wait(Timeout::Val(Duration::from_secs(execution_timeout))) + .map_err(|e| WaitForResultErr::Wait(format!("{:?}", e)))?; + + let mut cur = self.inner.as_slice(); + let header = ValidationResultHeader::decode(&mut cur) + .map_err(|e| WaitForResultErr::HeaderDecodeErr(format!("{:?}", e)))?; + Ok(header) + } +} + +/// Create a new workspace and return a handle to it. +pub fn create() -> Result { + // We actually don't need even that much, because e.g. validation result header will be + // written on top clobbering the params and code. We still over allocate just to be safe. + let mem_size = MAX_PARAMS_MEM + MAX_CODE_MEM + MAX_VALIDATION_RESULT_HEADER_MEM; + let shmem = ShmemConf::new() + .size(mem_size) + .create() + .map_err(|e| format!("Error creating shared memory: {:?}", e))?; + + Ok(HostHandle { + inner: Inner::layout(shmem, Mode::Initialize), + }) +} + +/// Open a workspace with the given `id`. +/// +/// You can attach only once to a single workspace. +pub fn open(id: &str) -> Result { + let shmem = ShmemConf::new() + .os_id(id) + .open() + .map_err(|e| format!("Error opening shared memory: {:?}", e))?; + + #[cfg(unix)] + unlink_shmem(&id); + + let inner = Inner::layout(shmem, Mode::Attach); + if !inner.declare_exclusive_attached() { + return Err(format!("The workspace has been already attached to")); + } + + return Ok(WorkerHandle { inner }); + + #[cfg(unix)] + fn unlink_shmem(shmem_id: &str) { + // Unlink the shmem. Unlinking it from the filesystem will make it unaccessible for further + // opening, however, the kernel will still let the object live until the last reference dies + // out. + // + // There is still a chance that the shm stays on the fs, but that's a highly unlikely case + // that we don't address at this time. + + // shared-memory doesn't return file path to the shmem if get_flink_path is called, so we + // resort to `shm_unlink`. + // + // Additionally, even thouygh `fs::remove_file` is said to use `unlink` we still avoid relying on it, + // because the stdlib doesn't actually provide any gurantees on what syscalls will be called. + // (Not sure, what alternative it has though). + unsafe { + // must be in a local var in order to be not deallocated. + let shmem_id_cstr = + std::ffi::CString::new(shmem_id).expect("the shmmem id cannot have NUL in it; qed"); + + if libc::shm_unlink(shmem_id_cstr.as_ptr()) == -1 { + // failed to remove the shmem file nothing we can do ¯\_(ツ)_/¯ + log::warn!( + target: LOG_TARGET, + "failed to remove the shmem with id {}", + shmem_id, + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::primitives::BlockData; + + use super::*; + use std::thread; + + #[test] + fn wait_until_ready() { + let host = create().unwrap(); + + let worker_handle = thread::spawn({ + let id = host.id().to_string(); + move || { + let worker = open(&id).unwrap(); + worker.signal_ready().unwrap(); + } + }); + + host.wait_until_ready(1).unwrap(); + + worker_handle.join().unwrap(); + } + + #[test] + fn wait_until_ready_timeout() { + let host = create().unwrap(); + + let _worker_handle = thread::spawn({ + let id = host.id().to_string(); + move || { + let _worker = open(&id).unwrap(); + } + }); + + assert!(matches!( + host.wait_until_ready(1), + Err(WaitUntilReadyErr::Wait(_)) + )); + } + + #[test] + fn open_junk_id() { + assert!(open("").is_err()); + assert!(open("non_existent").is_err()); + assert!(open("☭").is_err()); + } + + #[test] + fn attach_twice() { + let host = create().unwrap(); + + thread::spawn({ + let id = host.id().to_string(); + move || { + let _worker1 = open(&id).unwrap(); + assert!(open(&id).is_err()); + } + }); + } + + #[test] + fn validation_works() { + let mut host = create().unwrap(); + + let worker_handle = thread::spawn({ + let id = host.id().to_string(); + move || { + let mut worker = open(&id).unwrap(); + worker.signal_ready().unwrap(); + + let work = worker.wait_for_work(3).unwrap(); + assert_eq!(work.code, b"\0asm\01\00\00\00"); + + worker + .report_result(ValidationResultHeader::Ok(ValidationResult { + head_data: Default::default(), + new_validation_code: None, + upward_messages: vec![], + horizontal_messages: vec![], + processed_downward_messages: 322, + hrmp_watermark: 0, + })) + .unwrap(); + } + }); + + host.wait_until_ready(1).unwrap(); + host.request_validation( + b"\0asm\01\00\00\00", + ValidationParams { + parent_head: Default::default(), + block_data: BlockData(b"hello world".to_vec()), + relay_parent_number: 228, + relay_parent_storage_root: Default::default(), + }, + ) + .unwrap(); + + match host.wait_for_result(3).unwrap() { + ValidationResultHeader::Ok(r) => { + assert_eq!(r.processed_downward_messages, 322); + } + _ => panic!(), + } + + worker_handle.join().unwrap(); + } +} diff --git a/polkadot/parachain/test-parachains/tests/lib.rs b/polkadot/parachain/test-parachains/tests/lib.rs index e2c7c9a57d..aef9877ffa 100644 --- a/polkadot/parachain/test-parachains/tests/lib.rs +++ b/polkadot/parachain/test-parachains/tests/lib.rs @@ -24,7 +24,7 @@ use parachain::wasm_executor::run_worker; // It then passes "validation_worker" on CLI effectivly making rust test executor to run this single test. #[test] fn validation_worker() { - if let Some(id) = std::env::args().find(|a| a.starts_with("/shmem_rs_")) { + if let Some(id) = std::env::args().find(|a| a.starts_with("/shmem_")) { run_worker(&id, None).unwrap() } }