Move vulnerable dependencies to patched versions (#7)

* Move vulnerable dependencies to patched versions

Backend (Cargo.lock): openssl 0.10.55 -> 0.10.81, bytes 1.12.1, h2 0.3.27,
tokio 1.26 -> 1.47.1 (mio 1.x), url 2.3.1 -> 2.5.7 (idna 1.1), rand 0.8.8,
spin 0.9.9, time 0.3.55, ring 0.17.14.

Three alerts needed manifest changes:

- tokio-rustls 0.23 -> 0.26 (rustls 0.20 -> 0.23), which is what moves
  ring off 0.16. The ring provider is selected explicitly instead of the
  default aws-lc-rs, so the image build needs no cmake/C toolchain; roots
  still come from webpki-roots (now 1.0). Checked against live hosts: a
  valid certificate connects, an expired one and a name mismatch are
  refused.
- maxminddb 0.23 -> 0.27: lookup() now returns a result that is decoded,
  and the City model's fields are no longer Options. The existing
  locate_random_ip test (12.5.56.25 -> "Gardena") passes.
- structopt (clap 2, which pulled atty) -> clap 4 derive in telemetry_core,
  telemetry_shard and the soak tests; criterion 0.4 -> 0.5 for the same
  reason. Both binaries' arguments were parsed before and after for the
  docker-compose invocations and a set of other cases: identical results,
  except that `--feed-timeout` given without a value is now an error where
  structopt silently used the default. --denylist keeps taking every value
  up to the next flag (num_args = 1..), as its help text describes.

Frontend: npm audit fix within ranges, and webpack-dev-server 4 -> 5.2.6
(no devServer options are configured, so nothing to migrate). Left:
sockjs -> uuid 8 inside webpack-dev-server 5; the fix is 6.0, which needs
Node 22 while CI runs 20, and it is the local dev server only.

Checked: backend cargo test (58 passed), clippy, fmt; frontend lint,
prettier, test and build.

* Reject an invalid byte size instead of panicking

ByteSize::from_str called expect("all ascii digits") on the numeric part,
so a value with no digits ("abc", "k") or one too large for usize
aborted the shard at startup with a panic instead of a usage error, and
the suffix multiplication could overflow silently in release builds.
Parse errors now return Err and the multiplication is checked.

The new test fails on the old code (panic) and passes on this one.

backend/common/Cargo.toml

@@ -29,8 +29,9 @@ thiserror = "1.0.24"
 tokio = { version = "1.8.2", features = ["full"] }
 tokio-util = { version = "0.7.4", features = ["compat"] }
 arrayvec = { version = "0.7.1", features = ["serde"] }
-tokio-rustls = "0.23.4"
+# ring rather than the default aws-lc-rs provider: no cmake/C toolchain in the image build.
-webpki-roots = "0.22.4"
+tokio-rustls = { version = "0.26.6", default-features = false, features = ["logging", "ring", "tls12"] }
+webpki-roots = "1.0.9"
 
 [dev-dependencies]
 bincode = "1.3.3"

backend/common/src/byte_size.rs

@@ -40,33 +40,32 @@ impl std::str::FromStr for ByteSize {
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let s = s.trim();
-        match s.find(|c| !char::is_ascii_digit(&c)) {
+        let idx = s.find(|c| !char::is_ascii_digit(&c)).unwrap_or(s.len());
-            // No non-numeric chars; assume bytes then
+        // Empty (no digits at all) or too large for usize: a usage error, not a panic.
-            None => Ok(ByteSize(s.parse().expect("all ascii digits"))),
+        let n: usize = s[..idx]
-            // First non-numeric char
+            .parse()
-            Some(idx) => {
+            .map_err(|e| anyhow!("Cannot parse '{}' into bytes: {}", s, e))?;
-                let n = s[..idx].parse().expect("all ascii digits");
+        let suffix = s[idx..].trim();
-                let suffix = s[idx..].trim();
+        let multiplier: usize = match suffix {
-                let n = match suffix {
+            "" | "B" | "b" => 1,
-                    "B" | "b" => n,
+            "kB" | "K" | "k" => 1000,
-                    "kB" | "K" | "k" => n * 1000,
+            "MB" | "M" | "m" => 1000 * 1000,
-                    "MB" | "M" | "m" => n * 1000 * 1000,
+            "GB" | "G" | "g" => 1000 * 1000 * 1000,
-                    "GB" | "G" | "g" => n * 1000 * 1000 * 1000,
+            "KiB" | "Ki" => 1024,
-                    "KiB" | "Ki" => n * 1024,
+            "MiB" | "Mi" => 1024 * 1024,
-                    "MiB" | "Mi" => n * 1024 * 1024,
+            "GiB" | "Gi" => 1024 * 1024 * 1024,
-                    "GiB" | "Gi" => n * 1024 * 1024 * 1024,
+            _ => {
-                    _ => {
+                return Err(anyhow!(
-                        return Err(anyhow!(
+                    "\
-                            "\
+                Cannot parse into bytes; suffix is '{}', but expecting one of \
-                        Cannot parse into bytes; suffix is '{}', but expecting one of \
+                B,b, kB,K,k, MB,M,m, GB,G,g, KiB,Ki, MiB,Mi, GiB,Gi",
-                        B,b, kB,K,k, MB,M,m, GB,G,g, KiB,Ki, MiB,Mi, GiB,Gi",
+                    suffix
-                            suffix
+                ))
-                        ))
-                    }
-                };
-                Ok(ByteSize(n))
             }
-        }
+        };
+        n.checked_mul(multiplier)
+            .map(ByteSize)
+            .ok_or_else(|| anyhow!("Cannot parse '{}' into bytes: too large", s))
     }
 }
 
@@ -105,4 +104,23 @@ mod test {
             assert_eq!(b.num_bytes(), expected);
         }
     }
+
+    #[test]
+    fn rejects_invalid_strings_instead_of_panicking() {
+        // A typo in --max-node-data-per-second used to abort the shard with a panic
+        // ("all ascii digits") instead of a usage error.
+        let cases = vec![
+            "",
+            "abc",
+            "k",
+            " MB",
+            "1.5MB",
+            "99999999999999999999",
+            "18446744073709551615GiB",
+        ];
+
+        for s in cases {
+            assert!(s.parse::<ByteSize>().is_err(), "{s:?} should not parse");
+        }
+    }
 }

backend/common/src/ws_client/connect.rs

@@ -20,8 +20,8 @@ use std::io;
 use std::sync::Arc;
 use tokio::io::{AsyncRead, AsyncWrite};
 use tokio::net::TcpStream;
-use tokio_rustls::rustls::{OwnedTrustAnchor, ServerName};
+use tokio_rustls::rustls::{self, pki_types::ServerName};
-use tokio_rustls::{rustls, TlsConnector};
+use tokio_rustls::TlsConnector;
 use tokio_util::compat::TokioAsyncReadCompatExt;
 
 use super::{
@@ -274,20 +274,18 @@ async fn may_connect_tls(
     if !use_https {
         return Ok(Box::new(socket));
     };
-    let mut root_cert_store = rustls::RootCertStore::empty();
+    let root_cert_store = rustls::RootCertStore {
-    root_cert_store.add_server_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| {
+        roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(),
-        OwnedTrustAnchor::from_subject_spki_name_constraints(
+    };
-            ta.subject,
+    let config = rustls::ClientConfig::builder_with_provider(Arc::new(
-            ta.spki,
+        rustls::crypto::ring::default_provider(),
-            ta.name_constraints,
+    ))
-        )
+    .with_safe_default_protocol_versions()
-    }));
+    .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?
-    let config = rustls::ClientConfig::builder()
+    .with_root_certificates(root_cert_store)
-        .with_safe_defaults()
+    .with_no_client_auth();
-        .with_root_certificates(root_cert_store)
-        .with_no_client_auth();
     let connector = TlsConnector::from(Arc::new(config));
-    let domain = ServerName::try_from(host)
+    let domain = ServerName::try_from(host.to_owned())
         .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid dns name"))?;
     let socket = connector.connect(domain, socket).await?;
     Ok(Box::new(socket))

backend/telemetry_core/Cargo.toml

@@ -17,7 +17,7 @@ hex = "0.4.3"
 http = "0.2.4"
 hyper = "0.14.11"
 log = "0.4.14"
-maxminddb = "0.23.0"
+maxminddb = "0.27.0"
 num_cpus = "1.13.0"
 once_cell = "1.8.0"
 parking_lot = "0.12.1"
@@ -30,7 +30,7 @@ serde_json = "1.0.64"
 simple_logger = "4.0.0"
 smallvec = "1.6.1"
 soketto = "0.7.1"
-structopt = "0.3.21"
+clap = { version = "4.5", features = ["derive"] }
 thiserror = "1.0.25"
 tokio = { version = "1.10.1", features = ["full"] }
 tokio-util = { version = "0.7.4", features = ["compat"] }
@@ -41,7 +41,7 @@ jemallocator = "0.5.0"
 [dev-dependencies]
 shellwords = "1.1.0"
 test_utils = { path = "../test_utils" }
-criterion = { version = "0.4.0", features = ["async", "async_tokio"] }
+criterion = { version = "0.5", features = ["async", "async_tokio"] }
 
 [[bench]]
 name = "subscribe"

backend/telemetry_core/src/find_location.rs

@@ -111,16 +111,10 @@ impl Locator {
             return cached_loc;
         }
 
-        let City { city, location, .. } = self.city.lookup(ip.into()).ok()?;
+        let City { city, location, .. } = self.city.lookup(ip.into()).ok()?.decode().ok()??;
-        let city = city
+        let city = city.names.english?.to_string().into_boxed_str();
-            .as_ref()?
+        let latitude = location.latitude? as f32;
-            .names
+        let longitude = location.longitude? as f32;
-            .as_ref()?
-            .get("en")?
-            .to_string()
-            .into_boxed_str();
-        let latitude = location.as_ref()?.latitude? as f32;
-        let longitude = location?.longitude? as f32;
 
         let location = Arc::new(NodeLocation {
             city,

backend/telemetry_core/src/main.rs

@@ -26,13 +26,13 @@ use aggregator::{
     ToShardWebsocket,
 };
 use bincode::Options;
+use clap::Parser;
 use common::http_utils;
 use common::internal_messages;
 use common::ready_chunks_all::ReadyChunksAll;
 use futures::{SinkExt, StreamExt};
 use hyper::{Method, Response};
 use simple_logger::SimpleLogger;
-use structopt::StructOpt;
 
 #[cfg(not(target_env = "msvc"))]
 use jemallocator::Jemalloc;
@@ -48,49 +48,50 @@ const NAME: &str = "Substrate Telemetry Backend Core";
 const ABOUT: &str = "This is the Telemetry Backend Core that receives telemetry messages \
                      from Substrate/Polkadot nodes and provides the data to a subscribed feed";
 
-#[derive(StructOpt, Debug)]
+#[derive(Parser, Debug)]
-#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+#[command(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
 struct Opts {
     /// This is the socket address that Telemetry is listening to. This is restricted to
     /// localhost (127.0.0.1) by default and should be fine for most use cases. If
     /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
-    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8000")]
+    #[arg(short = 'l', long = "listen", default_value = "127.0.0.1:8000")]
     socket: std::net::SocketAddr,
     /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
     /// 'error' only logs errors and 'trace' logs everything.
-    #[structopt(long = "log", default_value = "info")]
+    #[arg(long = "log", default_value = "info")]
     log_level: log::LevelFilter,
     /// Space delimited list of the names of chains that are not allowed to connect to
     /// telemetry. Case sensitive.
-    #[structopt(long, required = false)]
+    // Takes every value up to the next flag (`--denylist a b c`), as structopt did.
+    #[arg(long, required = false, num_args = 1..)]
     denylist: Vec<String>,
     /// If it takes longer than this number of seconds to send the current batch of messages
     /// to a feed, the feed connection will be closed.
-    #[structopt(long, default_value = "10")]
+    #[arg(long, default_value = "10")]
     feed_timeout: u64,
     /// Number of worker threads to spawn. If "0" is given, use the number of CPUs available
     /// on the machine. If no value is given, use an internal default that we have deemed sane.
-    #[structopt(long)]
+    #[arg(long)]
     worker_threads: Option<usize>,
     /// Each aggregator keeps track of the entire node state. Feed subscriptions are split across
     /// aggregators.
-    #[structopt(long)]
+    #[arg(long)]
     num_aggregators: Option<usize>,
     /// How big can the message queue for each aggregator grow before we start dropping non-essential
     /// messages in an attempt to let it reduce?
-    #[structopt(long)]
+    #[arg(long)]
     aggregator_queue_len: Option<usize>,
     /// How many nodes from third party chains are allowed to connect before we prevent connections from them.
-    #[structopt(long, default_value = "1000")]
+    #[arg(long, default_value = "1000")]
     max_third_party_nodes: usize,
     /// Flag to expose the node's details (IP address, SysInfo, HwBench) of all connected
     /// nodes to the feed subscribers.
-    #[structopt(long)]
+    #[arg(long)]
     pub expose_node_details: bool,
 }
 
 fn main() {
-    let opts = Opts::from_args();
+    let opts = Opts::parse();
 
     SimpleLogger::new()
         .with_level(opts.log_level)

backend/telemetry_core/tests/soak_tests.rs

@@ -34,13 +34,13 @@ In general, if you run into issues, it may be better to run this on a linux
 box; MacOS seems to hit limits quicker in general.
 */
 
+use clap::Parser;
 use common::node_types::BlockHash;
 use common::ws_client::SentMessage;
 use futures::{future, StreamExt};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
-use structopt::StructOpt;
 use test_utils::workspace::{start_server, CoreOpts, ServerOpts, ShardOpts};
 
 /// A test runner which sends realistic(ish) messages from fake nodes to a telemetry server.
@@ -266,42 +266,42 @@ fn chain_names(total: usize) -> impl Iterator<Item = String> {
 /// General arguments that are used to start a soak test. Run `soak_test` as
 /// instructed by its documentation for full control over what is ran, or run
 /// preconfigured variants.
-#[derive(StructOpt)]
+#[derive(Parser)]
 struct SoakTestOpts {
     /// The number of shards to run this test with
-    #[structopt(long)]
+    #[arg(long)]
     shards: usize,
     /// The number of feeds to connect to the core
-    #[structopt(long)]
+    #[arg(long)]
     feeds: usize,
     /// The number of chains that nodes will pretend to belong to
-    #[structopt(long, default_value = "1")]
+    #[arg(long, default_value = "1")]
     chains: usize,
     /// The number of nodes to connect to each shard * chain combo.
     /// If we have 10 chains and 4 shards, setting this to 1 will connect `10 x 4 x 1 = 40` nodes.
-    #[structopt(long)]
+    #[arg(long)]
     nodes: usize,
     /// The number of different virtual nodes to connect per actual node socket connection
-    #[structopt(long, default_value = "1")]
+    #[arg(long, default_value = "1")]
     ids_per_node: usize,
     /// Number of aggregator loops to use in the core
-    #[structopt(long)]
+    #[arg(long)]
     core_num_aggregators: Option<usize>,
     /// Number of worker threads the core will use
-    #[structopt(long)]
+    #[arg(long)]
     core_worker_threads: Option<usize>,
     /// Number of worker threads each shard will use
-    #[structopt(long)]
+    #[arg(long)]
     shard_worker_threads: Option<usize>,
     /// Should we log output from the core/shards to stdout?
-    #[structopt(long)]
+    #[arg(long)]
     log_output: bool,
     /// How many worker threads should the soak test runner use?
-    #[structopt(long, default_value = "4")]
+    #[arg(long, default_value = "4")]
     test_worker_threads: usize,
 }
 
-/// Get soak test args from an envvar and parse them via structopt.
+/// Get soak test args from an envvar and parse them via clap.
 fn get_soak_test_opts() -> SoakTestOpts {
     let arg_string = std::env::var("SOAK_TEST_ARGS")
         .expect("Expecting args to be provided in the env var SOAK_TEST_ARGS");
@@ -311,5 +311,5 @@ fn get_soak_test_opts() -> SoakTestOpts {
     // The binary name is expected to be the first arg, so fake it:
     let all_args = std::iter::once("soak_test".to_owned()).chain(args.into_iter());
 
-    SoakTestOpts::from_iter(all_args)
+    SoakTestOpts::parse_from(all_args)
 }

backend/telemetry_shard/Cargo.toml

@@ -21,7 +21,7 @@ serde = { version = "1.0.126", features = ["derive"] }
 serde_json = "1.0.64"
 simple_logger = "4.0.0"
 soketto = "0.7.1"
-structopt = "0.3.21"
+clap = { version = "4.5", features = ["derive"] }
 thiserror = "1.0.25"
 tokio = { version = "1.10.1", features = ["full"] }
 tokio-util = { version = "0.7.4", features = ["compat"] }

backend/telemetry_shard/src/main.rs

@@ -29,6 +29,7 @@ use std::{
 
 use aggregator::{Aggregator, FromWebsocket};
 use blocked_addrs::BlockedAddrs;
+use clap::Parser;
 use common::byte_size::ByteSize;
 use common::http_utils;
 use common::node_message;
@@ -38,7 +39,6 @@ use futures::{SinkExt, StreamExt};
 use http::Uri;
 use hyper::{Method, Response};
 use simple_logger::SimpleLogger;
-use structopt::StructOpt;
 
 #[cfg(not(target_env = "msvc"))]
 use jemallocator::Jemalloc;
@@ -53,21 +53,21 @@ const NAME: &str = "Substrate Telemetry Backend Shard";
 const ABOUT: &str = "This is the Telemetry Backend Shard that forwards the \
                      data sent by Substrate/Polkadot nodes to the Backend Core";
 
-#[derive(StructOpt, Debug)]
+#[derive(Parser, Debug)]
-#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+#[command(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
 struct Opts {
     /// This is the socket address that this shard is listening to. This is restricted to
     /// localhost (127.0.0.1) by default and should be fine for most use cases. If
     /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
-    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8001")]
+    #[arg(short = 'l', long = "listen", default_value = "127.0.0.1:8001")]
     socket: std::net::SocketAddr,
     /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
     /// 'error' only logs errors and 'trace' logs everything.
-    #[structopt(long = "log", default_value = "info")]
+    #[arg(long = "log", default_value = "info")]
     log_level: log::LevelFilter,
     /// Url to the Backend Core endpoint accepting shard connections
-    #[structopt(
+    #[arg(
-        short = "c",
+        short = 'c',
         long = "core",
         default_value = "ws://127.0.0.1:8000/shard_submit/"
     )]
@@ -77,7 +77,7 @@ struct Opts {
     ///
     /// This is important because without a limit, a single connection could exhaust
     /// RAM by suggesting that it accounts for billions of nodes.
-    #[structopt(long, default_value = "20")]
+    #[arg(long, default_value = "20")]
     max_nodes_per_connection: usize,
     /// What is the maximum number of bytes per second, on average, that a connection from a
     /// node is allowed to send to a shard before it gets booted. This is averaged over a
@@ -86,26 +86,26 @@ struct Opts {
     ///
     /// As a reference point, syncing a new Polkadot node leads to a maximum of about 25k of
     /// traffic on average (at least initially).
-    #[structopt(long, default_value = "256k")]
+    #[arg(long, default_value = "256k")]
     max_node_data_per_second: ByteSize,
     /// How many seconds is a "/feed" connection that violates the '--max-node-data-per-second'
     /// value prevented from reconnecting to this shard for, in seconds.
-    #[structopt(long, default_value = "600")]
+    #[arg(long, default_value = "600")]
     node_block_seconds: u64,
     /// Number of worker threads to spawn. If "0" is given, use the number of CPUs available
     /// on the machine. If no value is given, use an internal default that we have deemed sane.
-    #[structopt(long)]
+    #[arg(long)]
     worker_threads: Option<usize>,
     /// Roughly how long to wait in seconds for new telemetry data to arrive from a node. If
     /// telemetry for a node does not arrive in this time frame, we remove the corresponding node
     /// state, and if no messages are received on the connection at all in this time, it will be
     /// dropped.
-    #[structopt(long, default_value = "60")]
+    #[arg(long, default_value = "60")]
     stale_node_timeout: u64,
 }
 
 fn main() {
-    let opts = Opts::from_args();
+    let opts = Opts::parse();
 
     SimpleLogger::new()
         .with_level(opts.log_level)

frontend/package.json

@@ -42,7 +42,7 @@
     "typescript": "^4.9.4",
     "webpack": "^5.76.0",
     "webpack-cli": "^5.0.1",
-    "webpack-dev-server": "^4.11.1",
+    "webpack-dev-server": "^5.2.6",
     "whatwg-fetch": "^3.6.2"
   },
   "lint-staged": {