Move vulnerable dependencies to patched versions

Backend (Cargo.lock): openssl 0.10.55 -> 0.10.81, bytes 1.12.1, h2 0.3.27,
tokio 1.26 -> 1.47.1 (mio 1.x), url 2.3.1 -> 2.5.7 (idna 1.1), rand 0.8.8,
spin 0.9.9, time 0.3.55, ring 0.17.14.

Three alerts needed manifest changes:

- tokio-rustls 0.23 -> 0.26 (rustls 0.20 -> 0.23), which is what moves
  ring off 0.16. The ring provider is selected explicitly instead of the
  default aws-lc-rs, so the image build needs no cmake/C toolchain; roots
  still come from webpki-roots (now 1.0). Checked against live hosts: a
  valid certificate connects, an expired one and a name mismatch are
  refused.
- maxminddb 0.23 -> 0.27: lookup() now returns a result that is decoded,
  and the City model's fields are no longer Options. The existing
  locate_random_ip test (12.5.56.25 -> "Gardena") passes.
- structopt (clap 2, which pulled atty) -> clap 4 derive in telemetry_core,
  telemetry_shard and the soak tests; criterion 0.4 -> 0.5 for the same
  reason. Both binaries' arguments were parsed before and after for the
  docker-compose invocations and a set of other cases: identical results,
  except that `--feed-timeout` given without a value is now an error where
  structopt silently used the default. --denylist keeps taking every value
  up to the next flag (num_args = 1..), as its help text describes.

Frontend: npm audit fix within ranges, and webpack-dev-server 4 -> 5.2.6
(no devServer options are configured, so nothing to migrate). Left:
sockjs -> uuid 8 inside webpack-dev-server 5; the fix is 6.0, which needs
Node 22 while CI runs 20, and it is the local dev server only.

Checked: backend cargo test (58 passed), clippy, fmt; frontend lint,
prettier, test and build.

backend/common/Cargo.toml

@@ -29,8 +29,9 @@ thiserror = "1.0.24"
 tokio = { version = "1.8.2", features = ["full"] }
 tokio-util = { version = "0.7.4", features = ["compat"] }
 arrayvec = { version = "0.7.1", features = ["serde"] }
-tokio-rustls = "0.23.4"
-webpki-roots = "0.22.4"
+# ring rather than the default aws-lc-rs provider: no cmake/C toolchain in the image build.
+tokio-rustls = { version = "0.26.6", default-features = false, features = ["logging", "ring", "tls12"] }
+webpki-roots = "1.0.9"
 
 [dev-dependencies]
 bincode = "1.3.3"

backend/common/src/ws_client/connect.rs

@@ -20,8 +20,8 @@ use std::io;
 use std::sync::Arc;
 use tokio::io::{AsyncRead, AsyncWrite};
 use tokio::net::TcpStream;
-use tokio_rustls::rustls::{OwnedTrustAnchor, ServerName};
-use tokio_rustls::{rustls, TlsConnector};
+use tokio_rustls::rustls::{self, pki_types::ServerName};
+use tokio_rustls::TlsConnector;
 use tokio_util::compat::TokioAsyncReadCompatExt;
 
 use super::{
@@ -274,20 +274,18 @@ async fn may_connect_tls(
     if !use_https {
         return Ok(Box::new(socket));
     };
-    let mut root_cert_store = rustls::RootCertStore::empty();
-    root_cert_store.add_server_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| {
-        OwnedTrustAnchor::from_subject_spki_name_constraints(
-            ta.subject,
-            ta.spki,
-            ta.name_constraints,
-        )
-    }));
-    let config = rustls::ClientConfig::builder()
-        .with_safe_defaults()
-        .with_root_certificates(root_cert_store)
-        .with_no_client_auth();
+    let root_cert_store = rustls::RootCertStore {
+        roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(),
+    };
+    let config = rustls::ClientConfig::builder_with_provider(Arc::new(
+        rustls::crypto::ring::default_provider(),
+    ))
+    .with_safe_default_protocol_versions()
+    .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?
+    .with_root_certificates(root_cert_store)
+    .with_no_client_auth();
     let connector = TlsConnector::from(Arc::new(config));
-    let domain = ServerName::try_from(host)
+    let domain = ServerName::try_from(host.to_owned())
         .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid dns name"))?;
     let socket = connector.connect(domain, socket).await?;
     Ok(Box::new(socket))

backend/telemetry_core/Cargo.toml

@@ -17,7 +17,7 @@ hex = "0.4.3"
 http = "0.2.4"
 hyper = "0.14.11"
 log = "0.4.14"
-maxminddb = "0.23.0"
+maxminddb = "0.27.0"
 num_cpus = "1.13.0"
 once_cell = "1.8.0"
 parking_lot = "0.12.1"
@@ -30,7 +30,7 @@ serde_json = "1.0.64"
 simple_logger = "4.0.0"
 smallvec = "1.6.1"
 soketto = "0.7.1"
-structopt = "0.3.21"
+clap = { version = "4.5", features = ["derive"] }
 thiserror = "1.0.25"
 tokio = { version = "1.10.1", features = ["full"] }
 tokio-util = { version = "0.7.4", features = ["compat"] }
@@ -41,7 +41,7 @@ jemallocator = "0.5.0"
 [dev-dependencies]
 shellwords = "1.1.0"
 test_utils = { path = "../test_utils" }
-criterion = { version = "0.4.0", features = ["async", "async_tokio"] }
+criterion = { version = "0.5", features = ["async", "async_tokio"] }
 
 [[bench]]
 name = "subscribe"

backend/telemetry_core/src/find_location.rs

@@ -111,16 +111,10 @@ impl Locator {
             return cached_loc;
         }
 
-        let City { city, location, .. } = self.city.lookup(ip.into()).ok()?;
-        let city = city
-            .as_ref()?
-            .names
-            .as_ref()?
-            .get("en")?
-            .to_string()
-            .into_boxed_str();
-        let latitude = location.as_ref()?.latitude? as f32;
-        let longitude = location?.longitude? as f32;
+        let City { city, location, .. } = self.city.lookup(ip.into()).ok()?.decode().ok()??;
+        let city = city.names.english?.to_string().into_boxed_str();
+        let latitude = location.latitude? as f32;
+        let longitude = location.longitude? as f32;
 
         let location = Arc::new(NodeLocation {
             city,

backend/telemetry_core/src/main.rs

@@ -26,13 +26,13 @@ use aggregator::{
     ToShardWebsocket,
 };
 use bincode::Options;
+use clap::Parser;
 use common::http_utils;
 use common::internal_messages;
 use common::ready_chunks_all::ReadyChunksAll;
 use futures::{SinkExt, StreamExt};
 use hyper::{Method, Response};
 use simple_logger::SimpleLogger;
-use structopt::StructOpt;
 
 #[cfg(not(target_env = "msvc"))]
 use jemallocator::Jemalloc;
@@ -48,49 +48,50 @@ const NAME: &str = "Substrate Telemetry Backend Core";
 const ABOUT: &str = "This is the Telemetry Backend Core that receives telemetry messages \
                      from Substrate/Polkadot nodes and provides the data to a subscribed feed";
 
-#[derive(StructOpt, Debug)]
-#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+#[derive(Parser, Debug)]
+#[command(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
 struct Opts {
     /// This is the socket address that Telemetry is listening to. This is restricted to
     /// localhost (127.0.0.1) by default and should be fine for most use cases. If
     /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
-    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8000")]
+    #[arg(short = 'l', long = "listen", default_value = "127.0.0.1:8000")]
     socket: std::net::SocketAddr,
     /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
     /// 'error' only logs errors and 'trace' logs everything.
-    #[structopt(long = "log", default_value = "info")]
+    #[arg(long = "log", default_value = "info")]
     log_level: log::LevelFilter,
     /// Space delimited list of the names of chains that are not allowed to connect to
     /// telemetry. Case sensitive.
-    #[structopt(long, required = false)]
+    // Takes every value up to the next flag (`--denylist a b c`), as structopt did.
+    #[arg(long, required = false, num_args = 1..)]
     denylist: Vec<String>,
     /// If it takes longer than this number of seconds to send the current batch of messages
     /// to a feed, the feed connection will be closed.
-    #[structopt(long, default_value = "10")]
+    #[arg(long, default_value = "10")]
     feed_timeout: u64,
     /// Number of worker threads to spawn. If "0" is given, use the number of CPUs available
     /// on the machine. If no value is given, use an internal default that we have deemed sane.
-    #[structopt(long)]
+    #[arg(long)]
     worker_threads: Option<usize>,
     /// Each aggregator keeps track of the entire node state. Feed subscriptions are split across
     /// aggregators.
-    #[structopt(long)]
+    #[arg(long)]
     num_aggregators: Option<usize>,
     /// How big can the message queue for each aggregator grow before we start dropping non-essential
     /// messages in an attempt to let it reduce?
-    #[structopt(long)]
+    #[arg(long)]
     aggregator_queue_len: Option<usize>,
     /// How many nodes from third party chains are allowed to connect before we prevent connections from them.
-    #[structopt(long, default_value = "1000")]
+    #[arg(long, default_value = "1000")]
     max_third_party_nodes: usize,
     /// Flag to expose the node's details (IP address, SysInfo, HwBench) of all connected
     /// nodes to the feed subscribers.
-    #[structopt(long)]
+    #[arg(long)]
     pub expose_node_details: bool,
 }
 
 fn main() {
-    let opts = Opts::from_args();
+    let opts = Opts::parse();
 
     SimpleLogger::new()
         .with_level(opts.log_level)

backend/telemetry_core/tests/soak_tests.rs

@@ -34,13 +34,13 @@ In general, if you run into issues, it may be better to run this on a linux
 box; MacOS seems to hit limits quicker in general.
 */
 
+use clap::Parser;
 use common::node_types::BlockHash;
 use common::ws_client::SentMessage;
 use futures::{future, StreamExt};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
-use structopt::StructOpt;
 use test_utils::workspace::{start_server, CoreOpts, ServerOpts, ShardOpts};
 
 /// A test runner which sends realistic(ish) messages from fake nodes to a telemetry server.
@@ -266,42 +266,42 @@ fn chain_names(total: usize) -> impl Iterator<Item = String> {
 /// General arguments that are used to start a soak test. Run `soak_test` as
 /// instructed by its documentation for full control over what is ran, or run
 /// preconfigured variants.
-#[derive(StructOpt)]
+#[derive(Parser)]
 struct SoakTestOpts {
     /// The number of shards to run this test with
-    #[structopt(long)]
+    #[arg(long)]
     shards: usize,
     /// The number of feeds to connect to the core
-    #[structopt(long)]
+    #[arg(long)]
     feeds: usize,
     /// The number of chains that nodes will pretend to belong to
-    #[structopt(long, default_value = "1")]
+    #[arg(long, default_value = "1")]
     chains: usize,
     /// The number of nodes to connect to each shard * chain combo.
     /// If we have 10 chains and 4 shards, setting this to 1 will connect `10 x 4 x 1 = 40` nodes.
-    #[structopt(long)]
+    #[arg(long)]
     nodes: usize,
     /// The number of different virtual nodes to connect per actual node socket connection
-    #[structopt(long, default_value = "1")]
+    #[arg(long, default_value = "1")]
     ids_per_node: usize,
     /// Number of aggregator loops to use in the core
-    #[structopt(long)]
+    #[arg(long)]
     core_num_aggregators: Option<usize>,
     /// Number of worker threads the core will use
-    #[structopt(long)]
+    #[arg(long)]
     core_worker_threads: Option<usize>,
     /// Number of worker threads each shard will use
-    #[structopt(long)]
+    #[arg(long)]
     shard_worker_threads: Option<usize>,
     /// Should we log output from the core/shards to stdout?
-    #[structopt(long)]
+    #[arg(long)]
     log_output: bool,
     /// How many worker threads should the soak test runner use?
-    #[structopt(long, default_value = "4")]
+    #[arg(long, default_value = "4")]
     test_worker_threads: usize,
 }
 
-/// Get soak test args from an envvar and parse them via structopt.
+/// Get soak test args from an envvar and parse them via clap.
 fn get_soak_test_opts() -> SoakTestOpts {
     let arg_string = std::env::var("SOAK_TEST_ARGS")
         .expect("Expecting args to be provided in the env var SOAK_TEST_ARGS");
@@ -311,5 +311,5 @@ fn get_soak_test_opts() -> SoakTestOpts {
     // The binary name is expected to be the first arg, so fake it:
     let all_args = std::iter::once("soak_test".to_owned()).chain(args.into_iter());
 
-    SoakTestOpts::from_iter(all_args)
+    SoakTestOpts::parse_from(all_args)
 }

backend/telemetry_shard/Cargo.toml

@@ -21,7 +21,7 @@ serde = { version = "1.0.126", features = ["derive"] }
 serde_json = "1.0.64"
 simple_logger = "4.0.0"
 soketto = "0.7.1"
-structopt = "0.3.21"
+clap = { version = "4.5", features = ["derive"] }
 thiserror = "1.0.25"
 tokio = { version = "1.10.1", features = ["full"] }
 tokio-util = { version = "0.7.4", features = ["compat"] }

backend/telemetry_shard/src/main.rs

@@ -29,6 +29,7 @@ use std::{
 
 use aggregator::{Aggregator, FromWebsocket};
 use blocked_addrs::BlockedAddrs;
+use clap::Parser;
 use common::byte_size::ByteSize;
 use common::http_utils;
 use common::node_message;
@@ -38,7 +39,6 @@ use futures::{SinkExt, StreamExt};
 use http::Uri;
 use hyper::{Method, Response};
 use simple_logger::SimpleLogger;
-use structopt::StructOpt;
 
 #[cfg(not(target_env = "msvc"))]
 use jemallocator::Jemalloc;
@@ -53,21 +53,21 @@ const NAME: &str = "Substrate Telemetry Backend Shard";
 const ABOUT: &str = "This is the Telemetry Backend Shard that forwards the \
                      data sent by Substrate/Polkadot nodes to the Backend Core";
 
-#[derive(StructOpt, Debug)]
-#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+#[derive(Parser, Debug)]
+#[command(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
 struct Opts {
     /// This is the socket address that this shard is listening to. This is restricted to
     /// localhost (127.0.0.1) by default and should be fine for most use cases. If
     /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
-    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8001")]
+    #[arg(short = 'l', long = "listen", default_value = "127.0.0.1:8001")]
     socket: std::net::SocketAddr,
     /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
     /// 'error' only logs errors and 'trace' logs everything.
-    #[structopt(long = "log", default_value = "info")]
+    #[arg(long = "log", default_value = "info")]
     log_level: log::LevelFilter,
     /// Url to the Backend Core endpoint accepting shard connections
-    #[structopt(
-        short = "c",
+    #[arg(
+        short = 'c',
         long = "core",
         default_value = "ws://127.0.0.1:8000/shard_submit/"
     )]
@@ -77,7 +77,7 @@ struct Opts {
     ///
     /// This is important because without a limit, a single connection could exhaust
     /// RAM by suggesting that it accounts for billions of nodes.
-    #[structopt(long, default_value = "20")]
+    #[arg(long, default_value = "20")]
     max_nodes_per_connection: usize,
     /// What is the maximum number of bytes per second, on average, that a connection from a
     /// node is allowed to send to a shard before it gets booted. This is averaged over a
@@ -86,26 +86,26 @@ struct Opts {
     ///
     /// As a reference point, syncing a new Polkadot node leads to a maximum of about 25k of
     /// traffic on average (at least initially).
-    #[structopt(long, default_value = "256k")]
+    #[arg(long, default_value = "256k")]
     max_node_data_per_second: ByteSize,
     /// How many seconds is a "/feed" connection that violates the '--max-node-data-per-second'
     /// value prevented from reconnecting to this shard for, in seconds.
-    #[structopt(long, default_value = "600")]
+    #[arg(long, default_value = "600")]
     node_block_seconds: u64,
     /// Number of worker threads to spawn. If "0" is given, use the number of CPUs available
     /// on the machine. If no value is given, use an internal default that we have deemed sane.
-    #[structopt(long)]
+    #[arg(long)]
     worker_threads: Option<usize>,
     /// Roughly how long to wait in seconds for new telemetry data to arrive from a node. If
     /// telemetry for a node does not arrive in this time frame, we remove the corresponding node
     /// state, and if no messages are received on the connection at all in this time, it will be
     /// dropped.
-    #[structopt(long, default_value = "60")]
+    #[arg(long, default_value = "60")]
     stale_node_timeout: u64,
 }
 
 fn main() {
-    let opts = Opts::from_args();
+    let opts = Opts::parse();
 
     SimpleLogger::new()
         .with_level(opts.log_level)

frontend/package.json

@@ -42,7 +42,7 @@
     "typescript": "^4.9.4",
     "webpack": "^5.76.0",
     "webpack-cli": "^5.0.1",
-    "webpack-dev-server": "^4.11.1",
+    "webpack-dev-server": "^5.2.6",
     "whatwg-fetch": "^3.6.2"
   },
   "lint-staged": {