add pyroscope (#4871)

* pyroscope

* fixup

* add pyroscope target

* fixins

* spellcheck

* rename 'pyroscoped' feature to 'pyroscope'

* build testnet binary with pyroscope feature

* Update Cargo.toml

Co-authored-by: Andronik <write@reusable.software>

* make args a string, resolve using std::net

* fixup

* remove --features pyroscope for testnet builds

Until the panics are fixed in upstream, this is sane.

* chore: bump pyroscope to 0.3.1

Fixes an underflow panic.

* Revert "remove --features pyroscope for testnet builds
"

This reverts commit 57dbdc7aa5f6427aeb1e3dfdfc97071c3a74851e.

* fix: Make sample rate a prime

Co-authored-by: Pierre Besson <pierre.besson@parity.io>
Co-authored-by: Andronik <write@reusable.software>

polkadot/.gitlab-ci.yml

@@ -195,7 +195,7 @@ test-build-linux-stable:
   script:
     - ./scripts/gitlab/test_linux_stable.sh
     # we're using the bin built here, instead of having a parallel `build-linux-release`
-    - time cargo build --profile testnet --verbose --bin polkadot
+    - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot
     - sccache -s
     # pack artifacts
     - mkdir -p ./artifacts

polkadot/Cargo.lock

@@ -1433,6 +1433,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "debugid"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba"
+dependencies = [
+ "uuid",
+]
+
 [[package]]
 name = "derivative"
 version = "2.2.0"
@@ -1834,6 +1843,18 @@ dependencies = [
  "scale-info",
 ]
 
+[[package]]
+name = "findshlibs"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64"
+dependencies = [
+ "cc",
+ "lazy_static",
+ "libc",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "fixed-hash"
 version = "0.7.0"
@@ -6116,6 +6137,7 @@ dependencies = [
  "polkadot-node-metrics",
  "polkadot-performance-test",
  "polkadot-service",
+ "pyroscope",
  "sc-cli",
  "sc-service",
  "sc-tracing",
@@ -7510,6 +7532,26 @@ dependencies = [
  "universal-hash",
 ]
 
+[[package]]
+name = "pprof"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55f35f865aa964be21fcde114cbd1cfbd9bf8a471460ed965b0f84f96c711401"
+dependencies = [
+ "backtrace",
+ "cfg-if 1.0.0",
+ "findshlibs",
+ "lazy_static",
+ "libc",
+ "log",
+ "nix",
+ "parking_lot 0.11.2",
+ "smallvec",
+ "symbolic-demangle",
+ "tempfile",
+ "thiserror",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.15"
@@ -7707,6 +7749,19 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "pyroscope"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e63f9bc346cdc6ad86ca90062248a1a7ce08fa44ad438637148359d1140733c"
+dependencies = [
+ "libc",
+ "log",
+ "pprof",
+ "reqwest",
+ "thiserror",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -10625,6 +10680,29 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
+[[package]]
+name = "symbolic-common"
+version = "8.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e92a52f07eed9afba3d6f883652cde7cd75fcf327dd44e84f210958379158737"
+dependencies = [
+ "debugid",
+ "memmap2 0.5.0",
+ "stable_deref_trait",
+ "uuid",
+]
+
+[[package]]
+name = "symbolic-demangle"
+version = "8.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9abc81544d9964975269165bfe5ad198d8b9e2e809c46527323f95588a57693"
+dependencies = [
+ "cpp_demangle",
+ "rustc-demangle",
+ "symbolic-common",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.86"
@@ -11410,6 +11488,12 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
 
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+
 [[package]]
 name = "valuable"
 version = "0.1.0"

polkadot/Cargo.toml

@@ -143,6 +143,7 @@ runtime-benchmarks= [ "polkadot-cli/runtime-benchmarks" ]
 try-runtime = [ "polkadot-cli/try-runtime" ]
 fast-runtime = [ "polkadot-cli/fast-runtime" ]
 runtime-metrics = [ "polkadot-cli/runtime-metrics" ]
+pyroscope = ["polkadot-cli/pyroscope"]
 
 # Configuration for building a .deb package - for use with `cargo-deb`
 [package.metadata.deb]

polkadot/cli/Cargo.toml

@@ -18,6 +18,7 @@ clap = { version = "3.0", features = ["derive"], optional = true }
 log = "0.4.13"
 thiserror = "1.0.30"
 futures = "0.3.21"
+pyro = { package = "pyroscope", version = "0.3.1", optional = true }
 
 service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true }
 polkadot-node-core-pvf = { path = "../node/core/pvf", optional = true }
@@ -57,6 +58,7 @@ trie-memory-tracker = ["sp-trie/memory-tracker"]
 full-node = ["service/full-node"]
 try-runtime = ["service/try-runtime"]
 fast-runtime = ["service/fast-runtime"]
+pyroscope = ["pyro"]
 
 # Configure the native runtimes to use. Polkadot is enabled by default.
 #

polkadot/cli/src/cli.rs

@@ -115,7 +115,14 @@ pub struct RunCmd {
 	/// Must be valid socket address, of format `IP:Port`
 	/// commonly `127.0.0.1:6831`.
 	#[clap(long)]
-	pub jaeger_agent: Option<std::net::SocketAddr>,
+	pub jaeger_agent: Option<String>,
+
+	/// Add the destination address to the `pyroscope` agent.
+	///
+	/// Must be valid socket address, of format `IP:Port`
+	/// commonly `127.0.0.1:4040`.
+	#[clap(long)]
+	pub pyroscope_server: Option<String>,
 }
 
 #[allow(missing_docs)]

polkadot/cli/src/command.rs

@@ -20,6 +20,7 @@ use log::info;
 use sc_cli::{Role, RuntimeVersion, SubstrateCli};
 use service::{self, IdentifyVariant};
 use sp_core::crypto::Ss58AddressFormatRegistry;
+use std::net::ToSocketAddrs;
 
 pub use crate::error::Error;
 pub use polkadot_performance_test::PerfCheckError;
@@ -266,7 +267,17 @@ where
 		info!("----------------------------");
 	}
 
-	let jaeger_agent = cli.run.jaeger_agent;
+	let jaeger_agent = if let Some(ref jaeger_agent) = cli.run.jaeger_agent {
+		Some(
+			jaeger_agent
+				.to_socket_addrs()
+				.map_err(Error::AddressResolutionFailure)?
+				.next()
+				.ok_or_else(|| Error::AddressResolutionMissing)?,
+		)
+	} else {
+		None
+	};
 
 	runner.run_node_until_exit(move |config| async move {
 		let role = config.role.clone();
@@ -293,6 +304,31 @@ where
 pub fn run() -> Result<()> {
 	let cli: Cli = Cli::from_args();
 
+	#[cfg(feature = "pyroscope")]
+	let mut pyroscope_agent_maybe = if let Some(ref agent_addr) = cli.run.pyroscope_server {
+		let address = agent_addr
+			.to_socket_addrs()
+			.map_err(Error::AddressResolutionFailure)?
+			.next()
+			.ok_or_else(|| Error::AddressResolutionMissing)?;
+		// The pyroscope agent requires a `http://` prefix, so we just do that.
+		let mut agent = pyro::PyroscopeAgent::builder(
+			"http://".to_owned() + address.to_string().as_str(),
+			"polkadot".to_owned(),
+		)
+		.sample_rate(113)
+		.build()?;
+		agent.start();
+		Some(agent)
+	} else {
+		None
+	};
+
+	#[cfg(not(feature = "pyroscope"))]
+	if cli.run.pyroscope_server.is_some() {
+		return Err(Error::PyroscopeNotCompiledIn)
+	}
+
 	match &cli.subcommand {
 		None => run_node_inner(cli, service::RealOverseerGen, polkadot_node_metrics::logger_hook()),
 		Some(Subcommand::BuildSpec(cmd)) => {
@@ -509,5 +545,10 @@ pub fn run() -> Result<()> {
 		)
 		.into()),
 	}?;
+
+	#[cfg(feature = "pyroscope")]
+	if let Some(mut pyroscope_agent) = pyroscope_agent_maybe.take() {
+		pyroscope_agent.stop();
+	}
 	Ok(())
 }

polkadot/cli/src/error.rs

@@ -31,6 +31,20 @@ pub enum Error {
 	#[error(transparent)]
 	PerfCheck(#[from] polkadot_performance_test::PerfCheckError),
 
+	#[cfg(not(feature = "pyroscope"))]
+	#[error("Binary was not compiled with `--feature=pyroscope`")]
+	PyroscopeNotCompiledIn,
+
+	#[cfg(feature = "pyroscope")]
+	#[error("Failed to connect to pyroscope agent")]
+	PyroscopeError(#[from] pyro::error::PyroscopeError),
+
+	#[error("Failed to resolve provided URL")]
+	AddressResolutionFailure(#[from] std::io::Error),
+
+	#[error("URL did not resolve to anything")]
+	AddressResolutionMissing,
+
 	#[error("Other: {0}")]
 	Other(String),
 }