diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c0232689..2e0b05b9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,3 +23,14 @@ jobs: keystore-file-name: develop_key.jks # develop variant is signed with signingConfigs.dev (develop_key.jks) build-debug-tests: false # TODO: Enable this, when debug build will be fixed for tests secrets: inherit + + # The Play build goes through R8; the develop build above does not. This builds the same rules into a + # debug-signed APK, so a class R8 renames or strips shows here and its mapping can be measured. + r8: + uses: pezkuwichain/pezkuwi-wallet-android/.github/workflows/android_build.yml@main + with: + branch: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || github.head_ref }} + gradlew-command: assembleReleaseTest + run-tests: false + upload-name: apk-r8 + secrets: inherit diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 195c1e79..28f87cbd 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -26,24 +26,51 @@ env: EHTERSCAN_API_KEY_ETHEREUM: ${{ secrets.EHTERSCAN_API_KEY_ETHEREUM }} INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }} DWELLIR_API_KEY: ${{ secrets.DWELLIR_API_KEY }} + TRONGRID_API_KEY: ${{ secrets.TRONGRID_API_KEY }} WALLET_CONNECT_PROJECT_ID: ${{ secrets.WALLET_CONNECT_PROJECT_ID }} DEBUG_GOOGLE_OAUTH_ID: ${{ secrets.DEBUG_GOOGLE_OAUTH_ID }} RELEASE_GOOGLE_OAUTH_ID: ${{ secrets.RELEASE_GOOGLE_OAUTH_ID }} jobs: codeql: - name: CodeQL Analysis - runs-on: [self-hosted, wallet] - if: github.event_name == 'push' # Only run on push, not PRs (self-hosted security) + # Advanced setup, because default setup analyses Java without a build and leaves out every + # Kotlin file, which is nearly all of this app. + name: CodeQL (${{ matrix.language }}) + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: javascript-typescript + build-mode: none + - language: rust + build-mode: none + - language: java-kotlin + build-mode: manual steps: + - name: Free disk space + if: matrix.language == 'java-kotlin' + run: | + # The CodeQL bundle in the tool cache is kept: the init step below uses it. + sudo rm -rf /usr/share/dotnet /usr/share/swift /opt/ghc /usr/local/.ghcup \ + /usr/local/share/boost /usr/local/share/powershell \ + /usr/local/lib/node_modules /usr/local/julia* /opt/microsoft + sudo docker image prune -af > /dev/null 2>&1 || true + df -h / + - name: Checkout uses: actions/checkout@v4 - name: Install dependencies + if: matrix.language == 'java-kotlin' uses: ./.github/workflows/install/ - name: Set up DEV Google Services + if: matrix.language == 'java-kotlin' uses: davidSchuppa/base64Secret-toFile-action@v3 with: secret: ${{ secrets.CI_DEVELOP_GOOGLE_SERVICES }} @@ -51,17 +78,25 @@ jobs: destination-path: ./app/ - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: - languages: java-kotlin + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} - name: Build for CodeQL - run: ./gradlew assembleDebug -x test -x lint + if: matrix.build-mode == 'manual' + # No Gradle or Kotlin daemon is left running: the analysis after the build needs that memory, and + # with them alive the runner ran out of it during query evaluation. + run: | + ./gradlew assembleDebug -x test -x lint --no-daemon -Pkotlin.compiler.execution.strategy=in-process + free -m env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 + with: + category: /language:${{ matrix.language }} dependency-review: name: Dependency Review diff --git a/app/build.gradle b/app/build.gradle index 92958d34..6af7cbaf 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -6,6 +6,10 @@ apply from: "../scripts/versions.gradle" apply from: "../scripts/secrets.gradle" android { + // The NDK that CI installs and the bindings build with. Without it AGP expects its default NDK, + // cannot strip the native libraries: they ship with their debug sections and no native-debug-symbols.zip is made. + ndkVersion "26.1.10909125" + defaultConfig { applicationId rootProject.applicationId versionCode computeVersionCode() @@ -61,8 +65,11 @@ android { matchingFallbacks = ['release'] signingConfig signingConfigs.debug + // No applicationIdSuffix: every release* build type gets the production Firebase config + // (applicationVariants below), whose only client is the production package. The debug + // signature keeps it from installing over a Play install, and the production API key, + // restricted to the release certificates, refuses its Firebase calls. versionNameSuffix '-releaseTest' - applicationIdSuffix '.releaseTest' buildConfigField "String", "BuildType", "\"releaseTest\"" } diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index cc79acc6..94ae04e7 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -67,6 +67,19 @@ -keep class * implements com.google.gson.TypeAdapterFactory -keep class * implements com.google.gson.JsonSerializer -keep class * implements com.google.gson.JsonDeserializer +# R8 full mode keeps generic signatures only on kept classes, so an anonymous `object : TypeToken() {}` +# lost its T: chains failed to load from the local DB ("TypeToken must be created with a type argument"). +# Gson 2.11 ships these rules; this app is on 2.10.1. +-keep,allowobfuscation,allowshrinking class com.google.gson.reflect.TypeToken +-keep,allowobfuscation,allowshrinking class * extends com.google.gson.reflect.TypeToken +# web3j's TypeReference reads its type argument the same way; without it the EVM token balance flow +# failed with "Missing type parameter". +-keep,allowobfuscation,allowshrinking class * extends org.web3j.abi.TypeReference +# Retrofit reads a suspend function's return type from Continuation's generic signature (rules from +# Retrofit 2.10; this app is on 2.9.0). +-keep,allowobfuscation,allowshrinking class kotlin.coroutines.Continuation +-keep,allowobfuscation,allowshrinking interface retrofit2.Call +-keep,allowobfuscation,allowshrinking class retrofit2.Response -keepclassmembers,allowobfuscation class * { @com.google.gson.annotations.SerializedName ; } @@ -92,7 +105,9 @@ # ============================================================ # SR25519 signing - keep the native methods and the class -keep class io.novafoundation.nova.sr25519.BizinikiwSr25519 { *; } --keep class io.novafoundation.nova.** { native ; } +# Names of classes with native methods and of those methods, which JNI binds by name. (A +# `-keep class io.novafoundation.nova.** { native ; }` here kept every class of the app, +# native methods or not.) -keepclasseswithmembernames class * { native ; } @@ -241,9 +256,20 @@ -keep class io.novafoundation.nova.**.navigation.** { *; } -keep class * extends androidx.navigation.NavDestination { *; } -# Keep all Nova foundation classes (prevent aggressive obfuscation) --keep class io.novafoundation.nova.** { *; } --keepnames class io.novafoundation.nova.** +# Field names of the app's own classes, so Gson reads and writes JSON by the declared names even where a +# class has no @SerializedName and sits outside the model/response/request/dto packages kept above. +# Class and method names are obfuscated. +-keepclassmembers class io.novafoundation.nova.** { ; } +# Constructors of the app's classes. R8 full mode makes a class it never sees instantiated abstract, and +# Gson instantiates its models reflectively: DappMetadataResponse, outside every name pattern above, +# failed with "Abstract classes can't be instantiated". Constructors are named either way, so +# this costs no obfuscation, and it applies only to classes that are otherwise kept. +-keepclassmembers class io.novafoundation.nova.** { (...); } + +# Methods a WebView page calls by name (DApp browser web3 bridge, integrity check, Transak events). +-keepclassmembers class * { + @android.webkit.JavascriptInterface ; +} # ============================================================ # Optimization settings @@ -269,6 +295,10 @@ -dontwarn org.apache.xalan.** -dontwarn org.ietf.jgss.** -dontwarn org.apache.http.** +# Referenced only by Jackson's Java 7 support class, which Jackson loads defensively; Android has no +# java.beans. R8 compatibility mode reports these as missing. +-dontwarn java.beans.ConstructorProperties +-dontwarn java.beans.Transient # ByteBuddy (test dependency) -dontwarn net.bytebuddy.** diff --git a/bindings/pezkuwi-light/rust/Cargo.toml b/bindings/pezkuwi-light/rust/Cargo.toml index 720553aa..64a0a93d 100644 --- a/bindings/pezkuwi-light/rust/Cargo.toml +++ b/bindings/pezkuwi-light/rust/Cargo.toml @@ -31,11 +31,11 @@ android_logger = { version = "0.14", default-features = false } env_logger = { version = "0.11", default-features = false, features = ["humantime"] } tungstenite = "0.24" -# Stripped here, not left to the Android Gradle plugin: it packages a library it cannot strip as -# it is, and with debug info this one is 147 MB per ABI instead of 8.6 MB. +# Line tables only: enough for the Play native-debug-symbols.zip to give file and line, at 59 MB +# per ABI before the app strips it to 8.6 MB. Full debug info (`debug = 2`) is 147 MB per ABI. [profile.release] -strip = true -debug = false +strip = false +debug = "line-tables-only" lto = true opt-level = "s" diff --git a/bindings/pezkuwi-light/rust/src/engine.rs b/bindings/pezkuwi-light/rust/src/engine.rs index 1a657430..25b33e48 100644 --- a/bindings/pezkuwi-light/rust/src/engine.rs +++ b/bindings/pezkuwi-light/rust/src/engine.rs @@ -25,7 +25,10 @@ use std::{ net::{Ipv4Addr, SocketAddr, TcpListener, TcpStream}, num::NonZero, path::PathBuf, - sync::{Arc, Mutex}, + sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, Mutex, + }, thread, time::Duration, }; @@ -49,6 +52,11 @@ pub struct Engine { struct Inner { client: Mutex>, chains: Mutex>, + /// Connection instances open on [`Inner::client`]; changed under both locks. + connections: AtomicUsize, + /// Source of [`Registered::registration`]. + registrations: AtomicU64, + platform: PezkuwiPlatform, executor: Arc>, token: String, db_dir: PathBuf, @@ -61,12 +69,16 @@ struct Registered { base: ChainId, /// Base instance of the relay chain, for a teyrchain. relay: Option, + /// Unique per registration, unlike `base`: smoldot reuses the id of a removed chain. + registration: u64, } #[derive(Debug)] pub enum Error { UnknownRelayChain(String), AlreadyRegistered(String), + NotRegistered(String), + RelayChainInUse(String), AddChain(String), Io(io::Error), } @@ -76,6 +88,8 @@ impl std::fmt::Display for Error { match self { Error::UnknownRelayChain(key) => write!(f, "relay chain {key} is not registered"), Error::AlreadyRegistered(key) => write!(f, "chain {key} is already registered"), + Error::NotRegistered(key) => write!(f, "chain {key} is not registered"), + Error::RelayChainInUse(key) => write!(f, "chain {key} is the relay chain of a registered chain"), Error::AddChain(err) => write!(f, "smoldot rejected the chain: {err}"), Error::Io(err) => write!(f, "{err}"), } @@ -103,6 +117,9 @@ impl Engine { let inner = Arc::new(Inner { client: Mutex::new(Client::new(platform.clone())), chains: Mutex::new(HashMap::new()), + connections: AtomicUsize::new(0), + registrations: AtomicU64::new(0), + platform: platform.clone(), executor: executor.clone(), token: hex::encode(token), db_dir, @@ -151,6 +168,7 @@ impl Engine { }; let database = fs::read_to_string(self.inner.database_path(key)).unwrap_or_default(); + let registration = self.inner.registrations.fetch_add(1, Ordering::SeqCst); let AddChainSuccess { chain_id, json_rpc_responses, @@ -177,6 +195,7 @@ impl Engine { spec: spec.into(), base: chain_id, relay, + registration, }, ); self.inner @@ -185,17 +204,44 @@ impl Engine { self.inner.clone(), key.to_owned(), chain_id, + registration, json_rpc_responses.expect("JSON-RPC is enabled"), )) .detach(); Ok(()) } + + /// Stops syncing a chain. Connections already open keep their own instance until they close; + /// a relay chain stays while a teyrchain registered on it does. The saved database is kept, + /// so registering the chain again resumes from it. + pub fn remove_chain(&self, key: &str) -> Result<(), Error> { + let mut chains = self.inner.chains.lock().unwrap(); + let base = chains.get(key).ok_or_else(|| Error::NotRegistered(key.into()))?.base; + if chains.values().any(|chain| chain.relay == Some(base)) { + return Err(Error::RelayChainInUse(key.into())); + } + + chains.remove(key); + let mut client = self.inner.client.lock().unwrap(); + let () = client.remove_chain(base); + self.inner.reset_when_idle(&chains, &mut client); + Ok(()) + } } impl Inner { fn database_path(&self, key: &str) -> PathBuf { database_path(&self.db_dir, key) } + + /// Replaces the client once no chain and no connection is left. smoldot keeps its + /// peer-to-peer network service, with the peer connections in it, for as long as the client + /// lives, so removing every chain alone leaves those connections open. + fn reset_when_idle(&self, chains: &HashMap, client: &mut Client) { + if chains.is_empty() && self.connections.load(Ordering::SeqCst) == 0 { + *client = Client::new(self.platform.clone()); + } + } } fn database_path(db_dir: &std::path::Path, key: &str) -> PathBuf { @@ -255,8 +301,12 @@ async fn serve(inner: Arc, stream: smol::Async) { let (chain_id, mut responses) = { let chains = inner.chains.lock().unwrap(); - let registered = &chains[&key]; - let added = inner.client.lock().unwrap().add_chain(AddChainConfig { + // Removed since the handshake routed it + let Some(registered) = chains.get(&key) else { + return; + }; + let mut client = inner.client.lock().unwrap(); + let added = client.add_chain(AddChainConfig { user_data: (), specification: ®istered.spec, // The base instance already holds the chain; smoldot shares it with this one. @@ -268,7 +318,10 @@ async fn serve(inner: Arc, stream: smol::Async) { }, }); match added { - Ok(success) => (success.chain_id, success.json_rpc_responses.expect("JSON-RPC is enabled")), + Ok(success) => { + inner.connections.fetch_add(1, Ordering::SeqCst); + (success.chain_id, success.json_rpc_responses.expect("JSON-RPC is enabled")) + } Err(err) => { log::warn!("{key}: cannot open a connection instance: {err}"); return; @@ -317,7 +370,11 @@ async fn serve(inner: Arc, stream: smol::Async) { }; futures_util::future::select(Box::pin(inbound), Box::pin(outbound)).await; - let () = inner.client.lock().unwrap().remove_chain(chain_id); + let chains = inner.chains.lock().unwrap(); + let mut client = inner.client.lock().unwrap(); + let () = client.remove_chain(chain_id); + inner.connections.fetch_sub(1, Ordering::SeqCst); + inner.reset_when_idle(&chains, &mut client); } /// JSON-RPC error for a request smoldot refused because its queue is full, so the caller gets an @@ -339,6 +396,7 @@ async fn save_database( inner: Arc, key: String, chain_id: ChainId, + registration: u64, mut responses: JsonRpcResponses, ) { let request = serde_json::json!({ @@ -353,14 +411,16 @@ async fn save_database( loop { smol::Timer::after(wait).await; wait = DATABASE_SAVE_INTERVAL; - if inner - .client - .lock() - .unwrap() - .json_rpc_request(request.clone(), chain_id) - .is_err() { - continue; + // Under the registry lock, so the chain cannot be removed between the check and the + // request: smoldot panics on a request to a removed chain. + let chains = inner.chains.lock().unwrap(); + if chains.get(&key).map(|chain| chain.registration) != Some(registration) { + return; + } + if inner.client.lock().unwrap().json_rpc_request(request.clone(), chain_id).is_err() { + continue; + } } while let Some(response) = responses.next().await { let response: serde_json::Value = match serde_json::from_str(&response) { diff --git a/bindings/pezkuwi-light/rust/src/lib.rs b/bindings/pezkuwi-light/rust/src/lib.rs index 3e6d5cd6..2934067b 100644 --- a/bindings/pezkuwi-light/rust/src/lib.rs +++ b/bindings/pezkuwi-light/rust/src/lib.rs @@ -156,6 +156,26 @@ pub extern "system" fn Java_io_novafoundation_nova_lightclient_PezkuwiLight_addC } } +/// Stops syncing the chain registered under `key`; a relay chain goes after its teyrchains. +#[no_mangle] +pub extern "system" fn Java_io_novafoundation_nova_lightclient_PezkuwiLight_removeChain<'local>( + mut env: JNIEnv<'local>, + _this: JObject<'local>, + key: JString<'local>, +) { + let Some(engine) = engine() else { + throw(&mut env, "light client is not started"); + return; + }; + let Some(key) = read_string(&mut env, &key) else { + throw(&mut env, "invalid chain key"); + return; + }; + if let Err(err) = engine.remove_chain(&key) { + throw(&mut env, &err.to_string()); + } +} + /// Restricts new peer connections to secure WebSockets. #[no_mangle] pub extern "system" fn Java_io_novafoundation_nova_lightclient_PezkuwiLight_setSecureOnly<'local>( diff --git a/bindings/pezkuwi-light/rust/tests/loopback.rs b/bindings/pezkuwi-light/rust/tests/loopback.rs index ba488b50..dc19bf26 100644 --- a/bindings/pezkuwi-light/rust/tests/loopback.rs +++ b/bindings/pezkuwi-light/rust/tests/loopback.rs @@ -68,6 +68,28 @@ fn scale_bytes(bytes: &[u8]) -> String { format!("0x{:02x}{}", (bytes.len() as u8) << 2, hex::encode(bytes)) } +/// Established TCP connections of this process to anything but the loopback interface (Linux). +fn peer_connections() -> usize { + let inodes: std::collections::HashSet = std::fs::read_dir("/proc/self/fd") + .unwrap() + .filter_map(|fd| std::fs::read_link(fd.ok()?.path()).ok()) + .filter_map(|target| { + let target = target.to_string_lossy().into_owned(); + Some(target.strip_prefix("socket:[")?.strip_suffix(']')?.to_owned()) + }) + .collect(); + ["/proc/net/tcp", "/proc/net/tcp6"] + .iter() + .flat_map(|table| std::fs::read_to_string(table).unwrap_or_default().lines().skip(1).map(str::to_owned).collect::>()) + .filter(|line| { + let fields: Vec<&str> = line.split_whitespace().collect(); + let remote = fields[2]; + let loopback = remote.starts_with("0100007F") || remote.starts_with("00000000000000000000000001000000"); + fields[3] == "01" && !loopback && inodes.contains(fields[9]) + }) + .count() +} + #[test] fn a_path_without_the_token_is_refused() { let engine = start(temporary_dir("refused")); @@ -231,3 +253,56 @@ fn resolves_after_a_start_from_the_saved_database() { } } } + +#[test] +#[ignore = "connects to the live Zagros testnet; needs PEZKUWI_LIGHT_RELAY_SPEC and PEZKUWI_LIGHT_PEOPLE_SPEC"] +fn a_removed_chain_stops_and_can_be_registered_again() { + use pezkuwi_light::engine::Error; + use std::sync::atomic::{AtomicBool, Ordering}; + + // A panic on an executor thread would not fail the test by itself. + static PANICKED: AtomicBool = AtomicBool::new(false); + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + PANICKED.store(true, Ordering::SeqCst); + default_hook(info); + })); + + let relay = std::fs::read_to_string(std::env::var("PEZKUWI_LIGHT_RELAY_SPEC").unwrap()).unwrap(); + let people = std::fs::read_to_string(std::env::var("PEZKUWI_LIGHT_PEOPLE_SPEC").unwrap()).unwrap(); + let engine = start(temporary_dir("remove")); + engine.add_chain("relay", &relay, None).unwrap(); + engine.add_chain("people", &people, Some("relay")).unwrap(); + let deadline = Instant::now() + Duration::from_secs(90); + while peer_connections() == 0 { + assert!(Instant::now() < deadline, "no peer connection was opened"); + std::thread::sleep(Duration::from_millis(500)); + } + + assert!(matches!(engine.remove_chain("relay"), Err(Error::RelayChainInUse(_)))); + engine.remove_chain("people").unwrap(); + engine.remove_chain("relay").unwrap(); + + // Removing every chain alone left smoldot's network service, and its peer connections, open. + // Connection attempts already under way finish first: about 20 s on Zagros. + let deadline = Instant::now() + Duration::from_secs(60); + while peer_connections() > 0 { + assert!(Instant::now() < deadline, "peer connections outlived every chain"); + std::thread::sleep(Duration::from_secs(1)); + } + assert!(matches!(engine.remove_chain("relay"), Err(Error::NotRegistered(_)))); + + match tungstenite::connect(format!("{}/relay", engine.base_url())) { + Err(tungstenite::Error::Http(response)) => assert_eq!(response.status(), 404), + other => panic!("a removed chain was served: {:?}", other.map(|_| ())), + } + + // Past the first database save of the removed chains, which must not reach smoldot. + std::thread::sleep(Duration::from_secs(25)); + assert!(!PANICKED.load(Ordering::SeqCst), "a removed chain's database save panicked"); + + engine.add_chain("relay", &relay, None).unwrap(); + let mut socket = connect(format!("{}/relay", engine.base_url())); + rpc(&mut socket, 1, "chainSpec_v1_chainName", serde_json::json!([])); + assert!(answer(&mut socket, 1)["result"].is_string()); +} diff --git a/bindings/pezkuwi-light/src/main/java/io/novafoundation/nova/lightclient/PezkuwiLight.kt b/bindings/pezkuwi-light/src/main/java/io/novafoundation/nova/lightclient/PezkuwiLight.kt index 434799f5..4aa52faa 100644 --- a/bindings/pezkuwi-light/src/main/java/io/novafoundation/nova/lightclient/PezkuwiLight.kt +++ b/bindings/pezkuwi-light/src/main/java/io/novafoundation/nova/lightclient/PezkuwiLight.kt @@ -31,6 +31,15 @@ object PezkuwiLight { */ external fun addChain(key: String, chainSpec: String, relayChainKey: String?) + /** + * Stops syncing the chain under [key]. Connections already open keep working until they close. + * Its database is kept, so a later [addChain] resumes from it. + * + * @throws IllegalStateException if the key is not registered, or is the relay chain of a + * registered teyrchain + */ + external fun removeChain(key: String) + /** * Restricts new peer connections to secure WebSockets (`wss`, normally on port 443), which * look like any HTTPS WebSocket on the wire. Open connections are kept. diff --git a/feature-wallet-impl/src/main/java/io/novafoundation/nova/feature_wallet_impl/data/network/blockchain/assets/history/statemine/StatemineAssetHistory.kt b/feature-wallet-impl/src/main/java/io/novafoundation/nova/feature_wallet_impl/data/network/blockchain/assets/history/statemine/StatemineAssetHistory.kt index 44c613a7..d329149e 100644 --- a/feature-wallet-impl/src/main/java/io/novafoundation/nova/feature_wallet_impl/data/network/blockchain/assets/history/statemine/StatemineAssetHistory.kt +++ b/feature-wallet-impl/src/main/java/io/novafoundation/nova/feature_wallet_impl/data/network/blockchain/assets/history/statemine/StatemineAssetHistory.kt @@ -2,7 +2,9 @@ package io.novafoundation.nova.feature_wallet_impl.data.network.blockchain.asset import io.novafoundation.nova.common.data.network.runtime.binding.bindAccountIdentifier import io.novafoundation.nova.common.data.network.runtime.binding.bindNumber -import io.novafoundation.nova.common.utils.oneOf +import io.novafoundation.nova.common.utils.instanceOf +import io.novasama.substrate_sdk_android.runtime.AccountId +import io.novasama.substrate_sdk_android.runtime.definitions.types.generics.GenericCall import io.novafoundation.nova.feature_wallet_api.data.network.blockhain.assets.history.realtime.RealtimeHistoryUpdate import io.novafoundation.nova.feature_wallet_api.data.network.blockhain.assets.history.realtime.substrate.SubstrateRealtimeOperationFetcher import io.novafoundation.nova.feature_wallet_api.data.network.blockhain.assets.history.realtime.substrate.SubstrateRealtimeOperationFetcher.Factory.Source @@ -22,10 +24,6 @@ import io.novafoundation.nova.runtime.multiNetwork.ChainRegistry import io.novafoundation.nova.runtime.multiNetwork.chain.model.Chain import io.novafoundation.nova.runtime.multiNetwork.chain.model.hasSameId import io.novafoundation.nova.runtime.multiNetwork.getRuntime -import io.novasama.substrate_sdk_android.runtime.RuntimeSnapshot -import io.novasama.substrate_sdk_android.runtime.definitions.types.generics.GenericCall -import io.novasama.substrate_sdk_android.runtime.metadata.call -import io.novasama.substrate_sdk_android.runtime.metadata.module class StatemineAssetHistory( private val chainRegistry: ChainRegistry, @@ -63,29 +61,31 @@ class StatemineAssetHistory( val runtime = chainRegistry.getRuntime(chain.id) val call = extrinsicVisit.call - if (!call.isTransfer(runtime, chainAsset)) return null + val statemineType = chainAsset.requireStatemine() + val moduleName = statemineType.palletNameOrDefault() - val amount = bindNumber(call.arguments["amount"]) + val (senderId, recipientId) = call.assetTransferParties(moduleName, extrinsicVisit.origin) ?: return null + + if (!statemineType.hasSameId(runtime, dynamicInstanceId = call.arguments["id"])) return null return RealtimeHistoryUpdate.Type.Transfer( - senderId = extrinsicVisit.origin, - recipientId = bindAccountIdentifier(call.arguments["target"]), - amountInPlanks = amount, + senderId = senderId, + recipientId = recipientId, + amountInPlanks = bindNumber(call.arguments["amount"]), chainAsset = chainAsset, ) } - - private fun GenericCall.Instance.isTransfer(runtime: RuntimeSnapshot, chainAsset: Chain.Asset): Boolean { - val statemineType = chainAsset.requireStatemine() - val moduleName = statemineType.palletNameOrDefault() - val module = runtime.metadata.module(moduleName) - - val matchingCall = oneOf( - module.call("transfer"), - module.call("transfer_keep_alive"), - ) - - return matchingCall && statemineType.hasSameId(runtime, dynamicInstanceId = arguments["id"]) - } } } + +/** Sender and recipient of an assets pallet transfer, or null when the call is not one. */ +internal fun GenericCall.Instance.assetTransferParties(moduleName: String, origin: AccountId): Pair? = when { + instanceOf(moduleName, "transfer", "transfer_keep_alive") -> + origin to bindAccountIdentifier(arguments["target"]) + + // Spends an allowance: the funds leave `owner`, not the delegate that signed (the USDT bridge credits this way) + instanceOf(moduleName, "transfer_approved") -> + bindAccountIdentifier(arguments["owner"]) to bindAccountIdentifier(arguments["destination"]) + + else -> null +} diff --git a/feature-wallet-impl/src/test/java/io/novafoundation/nova/feature_wallet_impl/data/network/blockchain/assets/history/statemine/AssetTransferPartiesTest.kt b/feature-wallet-impl/src/test/java/io/novafoundation/nova/feature_wallet_impl/data/network/blockchain/assets/history/statemine/AssetTransferPartiesTest.kt new file mode 100644 index 00000000..8a152b15 --- /dev/null +++ b/feature-wallet-impl/src/test/java/io/novafoundation/nova/feature_wallet_impl/data/network/blockchain/assets/history/statemine/AssetTransferPartiesTest.kt @@ -0,0 +1,45 @@ +package io.novafoundation.nova.feature_wallet_impl.data.network.blockchain.assets.history.statemine + +import io.novasama.substrate_sdk_android.runtime.definitions.types.generics.GenericCall +import io.novasama.substrate_sdk_android.runtime.metadata.module.MetadataFunction +import io.novasama.substrate_sdk_android.runtime.metadata.module.Module +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertNull +import org.junit.Test +import java.math.BigInteger + +class AssetTransferPartiesTest { + + private val origin = ByteArray(32) { 1 } + private val owner = ByteArray(32) { 2 } + private val destination = ByteArray(32) { 3 } + + @Test + fun `transfer goes from the signer to the target`() { + val (sender, recipient) = call("transfer", "target" to destination).assetTransferParties("Assets", origin)!! + + assertArrayEquals(origin, sender) + assertArrayEquals(destination, recipient) + } + + @Test + fun `transfer approved goes from the owner to the destination, not from the signing delegate`() { + val (sender, recipient) = call("transfer_approved", "owner" to owner, "destination" to destination) + .assetTransferParties("Assets", origin)!! + + assertArrayEquals(owner, sender) + assertArrayEquals(destination, recipient) + } + + @Test + fun `other calls are not transfers`() { + assertNull(call("approve_transfer", "delegate" to destination).assetTransferParties("Assets", origin)) + assertNull(call("transfer", "target" to destination, module = "ForeignAssets").assetTransferParties("Assets", origin)) + } + + private fun call(name: String, vararg arguments: Pair, module: String = "Assets") = GenericCall.Instance( + module = Module(module, null, null, null, emptyMap(), emptyMap(), BigInteger.ZERO), + function = MetadataFunction(name, emptyList(), emptyList(), 0 to 0), + arguments = mapOf("id" to BigInteger.ONE, "amount" to BigInteger.TEN, *arguments) + ) +} diff --git a/feature-xcm/api/src/main/java/io/novafoundation/nova/feature_xcm_api/multiLocation/MultiLocationEncoding.kt b/feature-xcm/api/src/main/java/io/novafoundation/nova/feature_xcm_api/multiLocation/MultiLocationEncoding.kt index 4d32399e..e0887705 100644 --- a/feature-xcm/api/src/main/java/io/novafoundation/nova/feature_xcm_api/multiLocation/MultiLocationEncoding.kt +++ b/feature-xcm/api/src/main/java/io/novafoundation/nova/feature_xcm_api/multiLocation/MultiLocationEncoding.kt @@ -112,7 +112,6 @@ private fun bindGlobalConsensusJunction(instance: Any?): Junction { "Polkadot" -> Junction.GlobalConsensus(NetworkId.Substrate(Chain.Geneses.POLKADOT)) "Kusama" -> Junction.GlobalConsensus(NetworkId.Substrate(Chain.Geneses.KUSAMA)) - "Westend" -> Junction.GlobalConsensus(NetworkId.Substrate(Chain.Geneses.WESTEND)) "Ethereum" -> { val chainId = bindInt(asDictEnum.value.castToStruct()["chain_id"]) Junction.GlobalConsensus(NetworkId.Ethereum(chainId)) @@ -189,7 +188,6 @@ private fun NetworkId.Substrate.toEncodableInstance(): Any { return when (genesisHash) { Chain.Geneses.POLKADOT -> DictEnum.Entry("Polkadot", null) Chain.Geneses.KUSAMA -> DictEnum.Entry("Kusama", null) - Chain.Geneses.WESTEND -> DictEnum.Entry("Westend", null) Chain.Ids.ETHEREUM -> DictEnum.Entry("Ethereum", null) else -> DictEnum.Entry("ByGenesis", genesisHash.fromHex()) } diff --git a/feature-xcm/api/src/test/java/io/novafoundation/nova/feature_xcm_api/multiLocation/MultiLocationEncodingTest.kt b/feature-xcm/api/src/test/java/io/novafoundation/nova/feature_xcm_api/multiLocation/MultiLocationEncodingTest.kt new file mode 100644 index 00000000..67ac556e --- /dev/null +++ b/feature-xcm/api/src/test/java/io/novafoundation/nova/feature_xcm_api/multiLocation/MultiLocationEncodingTest.kt @@ -0,0 +1,42 @@ +package io.novafoundation.nova.feature_xcm_api.multiLocation + +import io.novafoundation.nova.common.utils.HexString +import io.novafoundation.nova.feature_xcm_api.multiLocation.MultiLocation.Junction.GlobalConsensus +import io.novafoundation.nova.feature_xcm_api.multiLocation.MultiLocation.NetworkId +import io.novafoundation.nova.feature_xcm_api.versions.XcmVersion +import io.novasama.substrate_sdk_android.extensions.toHexString +import io.novasama.substrate_sdk_android.runtime.definitions.types.composite.DictEnum +import org.junit.Assert.assertEquals +import org.junit.Test + +class MultiLocationEncodingTest { + + // PezkuwiChain runtimes declare `ThisNetwork = ByGenesis(..)`, and XCM v5 has no named variant for them + @Test + fun `zagros is encoded by its genesis`() { + assertEncodedByGenesis(ZAGROS) + } + + @Test + fun `pezkuwi is encoded by its genesis`() { + assertEncodedByGenesis(PEZKUWI) + } + + private fun assertEncodedByGenesis(genesis: HexString) { + val location = RelativeMultiLocation(parents = 1, interior = Junctions(GlobalConsensus(NetworkId.Substrate(genesis)))) + + val encoded = location.toEncodableInstanceExt(XcmVersion.V5) + val interior = requireNotNull(encoded.get>("interior")) + val junction = (interior.value as List<*>).single() as DictEnum.Entry<*> + val network = junction.value as DictEnum.Entry<*> + + assertEquals("GlobalConsensus", junction.name) + assertEquals("ByGenesis", network.name) + assertEquals(genesis, (network.value as ByteArray).toHexString(withPrefix = false)) + } + + private companion object { + const val ZAGROS = "1b0b4727bbb5e44ed587d1056b4ad537ffb330034509a58866a088d77e792cd5" + const val PEZKUWI = "1aa94987791a5544e9667ec249d2cef1b8fdd6083c85b93fc37892d54a1156ca" + } +} diff --git a/gradle.properties b/gradle.properties index c63fc8a3..85845d96 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,3 +19,9 @@ android.defaults.buildfeatures.buildconfig=true android.nonTransitiveRClass=false android.nonFinalResIds=false android.experiments.cacheability.pageAlignSharedLibs=true + +# R8 compatibility mode. Full mode keeps generic signatures only on explicitly kept classes and makes a +# class it never sees instantiated abstract; this app's Gson and Retrofit models rely on both through +# reflection (SubQueryResponse decoded to a LinkedTreeMap in the full-mode build). Obfuscation is the +# same in both modes. +android.enableR8.fullMode=false diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt index ceec116a..2ab09168 100644 --- a/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt @@ -26,6 +26,7 @@ import io.novafoundation.nova.runtime.multiNetwork.connection.ConnectionPool import io.novafoundation.nova.runtime.multiNetwork.connection.ConnectionSecrets import io.novafoundation.nova.runtime.multiNetwork.connection.Web3ApiPool import io.novafoundation.nova.runtime.multiNetwork.connection.autobalance.NodeAutobalancer +import io.novafoundation.nova.runtime.multiNetwork.connection.light.LightClientSpecApi import io.novafoundation.nova.runtime.multiNetwork.connection.light.LightClients import io.novafoundation.nova.runtime.multiNetwork.connection.autobalance.strategy.NodeSelectionStrategyProvider import io.novafoundation.nova.runtime.multiNetwork.connection.node.healthState.NodeHealthStateTesterFactory @@ -186,7 +187,8 @@ class ChainRegistryModule { fun provideLightClients( context: Context, encryptedFirstDns: EncryptedFirstDns, - ) = LightClients(context, encryptedFirstDns) + networkApiCreator: NetworkApiCreator, + ) = LightClients(context, encryptedFirstDns, networkApiCreator.create(LightClientSpecApi::class.java)) @Provides @ApplicationScope diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/ext/ChainExt.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/ext/ChainExt.kt index 97a1d7e7..527366cf 100644 --- a/runtime/src/main/java/io/novafoundation/nova/runtime/ext/ChainExt.kt +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/ext/ChainExt.kt @@ -461,9 +461,6 @@ object ChainGeneses { const val KUSAMA = "b0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe" const val POLKADOT = "91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3" - // Westend constant now points to Zagros Testnet - const val WESTEND = "96eb58af1bb7288115b5e4ff1590422533e749293f231974536dc6672417d06f" - const val KUSAMA_ASSET_HUB = "48239ef607d7928874027a43a67689209727dfb3d3dc5e5b03a39bdc2eda771a" const val ACALA = "fc41b9bd8ef8fe53d58c7ea67c794c7ec9a73daf05e6d54b14ff6342c99ba64c" diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/ChainConnection.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/ChainConnection.kt index 2696724a..58ac4d08 100644 --- a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/ChainConnection.kt +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/ChainConnection.kt @@ -197,7 +197,10 @@ class ChainConnection internal constructor( is FallbackEvent.Socket -> { val onLight = light != null && event.state.urlOrNull() == light?.saturatedUrl - if (event.state is State.Connected && !onLight) policy.onRpcConnected() + if (event.state is State.Connected && !onLight) { + policy.onRpcConnected(System.currentTimeMillis()) + lightClients.refreshSpecsInBackground() + } if (event.state.needsAutobalance()) { Log.d(LIGHT_FALLBACK_TAG, "${chain.value.name}: ${event.state} failed, source ${policy.source}, on light $onLight") @@ -217,9 +220,17 @@ class ChainConnection internal constructor( } } - FallbackEvent.Tick -> if (policy.onTick(rpcNodeCount(), System.currentTimeMillis())) { - Log.d(LIGHT_FALLBACK_TAG, "${chain.value.name}: retrying RPC") - emit(latestRpc) + FallbackEvent.Tick -> { + val now = System.currentTimeMillis() + if (policy.onTick(rpcNodeCount(), now)) { + Log.d(LIGHT_FALLBACK_TAG, "${chain.value.name}: retrying RPC") + emit(latestRpc) + } + if (policy.shouldReleaseLight(now)) { + Log.d(LIGHT_FALLBACK_TAG, "${chain.value.name}: RPC is stable, releasing the light client") + light = null + lightClients.release(chain.value.id) + } } } } diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightClientSpecApi.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightClientSpecApi.kt new file mode 100644 index 00000000..65405a54 --- /dev/null +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightClientSpecApi.kt @@ -0,0 +1,11 @@ +package io.novafoundation.nova.runtime.multiNetwork.connection.light + +import retrofit2.http.GET +import retrofit2.http.Url + +interface LightClientSpecApi { + + /** A file of the config repository, checked against its signed targets by the app's HTTP client. */ + @GET + suspend fun spec(@Url url: String): String +} diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightClients.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightClients.kt index 262250bf..b2e45a47 100644 --- a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightClients.kt +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightClients.kt @@ -6,36 +6,45 @@ import io.novafoundation.nova.common.utils.LOG_TAG import io.novafoundation.nova.lightclient.HostResolver import io.novafoundation.nova.lightclient.PezkuwiLight import io.novafoundation.nova.runtime.multiNetwork.chain.model.ChainId +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import okhttp3.Dns import java.io.File +import java.util.concurrent.atomic.AtomicBoolean /** * The embedded light client (`bindings/pezkuwi-light`) for the chains it can verify. * - * Started on first use, so a wallet whose RPC nodes answer never runs it. Chain specifications - * come from the APK (`light-client/` assets, built from pezkuwi-wallet-utils): the genesis state - * root, boot nodes and, for a relay chain, a checkpoint. Peer host names resolve through [dns], - * the app's DNS-over-HTTPS-first resolver. + * Started on first use, so a wallet whose RPC nodes answer never runs it, and a chain stops + * syncing once [release]d. Chain specifications come from the APK (`light-client/` assets, built + * from pezkuwi-wallet-utils): the genesis state root, boot nodes and, for a relay chain, a + * checkpoint. Peer host names resolve through [dns], the app's DNS-over-HTTPS-first resolver. */ class LightClients( private val context: Context, private val dns: Dns, + private val specApi: LightClientSpecApi, ) { - private class LightChain(val asset: String, val relay: ChainId?) + private class LightChain(val file: String, val relay: ChainId?) private val chains = mapOf( - ZAGROS to LightChain("light-client/zagros.json", relay = null), - ZAGROS_PEOPLE to LightChain("light-client/people-zagros.json", relay = ZAGROS), + ZAGROS to LightChain("zagros.json", relay = null), + ZAGROS_PEOPLE to LightChain("people-zagros.json", relay = ZAGROS), ) + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val specRefreshStarted = AtomicBoolean(false) + private val mutex = Mutex() private var baseUrl: String? = null private val registered = mutableSetOf() + private val inUse = mutableSetOf() fun supports(chainId: ChainId): Boolean = chainId in chains @@ -46,6 +55,7 @@ class LightClients( if (!supports(chainId)) return@runCatching null val base = baseUrl ?: PezkuwiLight.start(File(context.filesDir, DATABASE_DIR), resolver()).also { baseUrl = it } register(chainId) + inUse += chainId PezkuwiLight.chainUrl(base, chainId) }.onFailure { Log.w(LOG_TAG, "Light client unavailable for $chainId", it) @@ -53,16 +63,72 @@ class LightClients( } } + /** + * [chainId] no longer needs the light client, so it stops syncing: a synced chain costs data + * and CPU while nothing reads it. A relay chain goes once none of its teyrchains is in use. + */ + suspend fun release(chainId: ChainId) = mutex.withLock { + withContext(Dispatchers.IO) { + inUse -= chainId + while (true) { + val unused = registered.firstOrNull { key -> key !in inUse && registered.none { chains.getValue(it).relay == key } } ?: break + // Still registered on failure, so a later url() does not register it twice + runCatching { PezkuwiLight.removeChain(unused) } + .onSuccess { registered -= unused } + .onFailure { Log.w(LOG_TAG, "Cannot stop the light client for $unused", it) } + .getOrNull() ?: break + } + } + } + + /** + * Fetches each chain's specification from the config repository, once per process, and keeps it + * when it is the same chain with a later checkpoint ([LightSpecs.isUpdate]). Called while RPC + * works: the light client is wanted when it does not, and then the repository may not answer + * either. Without it a first fallback starts from the checkpoint the APK was built with. + */ + fun refreshSpecsInBackground() { + if (!specRefreshStarted.compareAndSet(false, true)) return + + scope.launch { + chains.values.forEach { chain -> + runCatching { + val candidate = specApi.spec(SPEC_SOURCE + chain.file) + if (LightSpecs.isUpdate(current = spec(chain), candidate = candidate)) { + writeAtomically(storedSpec(chain), candidate) + Log.d(LOG_TAG, "Light client specification ${chain.file} updated") + } + }.onFailure { Log.w(LOG_TAG, "Cannot refresh light client specification ${chain.file}", it) } + } + } + } + private fun register(chainId: ChainId) { if (chainId in registered) return val chain = chains.getValue(chainId) chain.relay?.let(::register) - val spec = context.assets.open(chain.asset).bufferedReader().use { it.readText() } - PezkuwiLight.addChain(chainId, spec, chain.relay) + PezkuwiLight.addChain(chainId, spec(chain), chain.relay) registered += chainId } + /** The stored specification when it is later than the APK's; an app update can bring a later one. */ + private fun spec(chain: LightChain): String { + val bundled = context.assets.open(BUNDLED_DIR + chain.file).bufferedReader().use { it.readText() } + val stored = storedSpec(chain).takeIf { it.exists() }?.readText() ?: return bundled + + return if (LightSpecs.isUpdate(current = bundled, candidate = stored)) stored else bundled + } + + private fun storedSpec(chain: LightChain) = File(context.filesDir, "$SPECS_DIR/${chain.file}") + + private fun writeAtomically(file: File, text: String) { + file.parentFile?.mkdirs() + val temporary = File(file.parentFile, "${file.name}.tmp") + temporary.writeText(text) + if (!temporary.renameTo(file)) error("Cannot replace $file") + } + private fun resolver() = HostResolver { host -> dns.lookup(host).mapNotNull { it.hostAddress }.toTypedArray() } @@ -72,5 +138,8 @@ class LightClients( const val ZAGROS_PEOPLE: ChainId = "c72b38f36affaeb2661018b201c2be02e8c09fc699640525d39200a10fd77878" const val DATABASE_DIR = "light-client" + const val SPECS_DIR = "light-client-specs" + const val BUNDLED_DIR = "light-client/" + const val SPEC_SOURCE = "https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/light-client/" } } diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicy.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicy.kt index f146a2ae..3b4471fc 100644 --- a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicy.kt +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicy.kt @@ -7,12 +7,15 @@ package io.novafoundation.nova.runtime.multiNetwork.connection.light * those nodes have a fixed number of light client slots. Once every RPC node has failed in a row, * the light client takes over. RPC is retried after [initialBackoffMillis], doubling up to * [maxBackoffMillis] while it keeps failing, and takes back the connection once it connects. + * The light client is kept for [releaseLightAfterMillis] after that, in case RPC fails again + * soon, and then released: a synced light client costs data and CPU while nothing reads it. * * Not thread-safe: one connection feeds it its events in order. */ class LightFallbackPolicy( private val initialBackoffMillis: Long = 60_000, private val maxBackoffMillis: Long = 15 * 60_000, + private val releaseLightAfterMillis: Long = 5 * 60_000, ) { enum class Source { RPC, LIGHT } @@ -23,21 +26,26 @@ class LightFallbackPolicy( private var consecutiveRpcFailures = 0 private var backoffMillis = initialBackoffMillis private var rpcRetryAt: Long? = null + private var lightHeld = false + private var rpcConnectedSince: Long? = null /** An RPC node failed. True when the light client should take over now. */ fun onRpcFailure(rpcNodeCount: Int, nowMillis: Long): Boolean { if (source != Source.RPC) return false + rpcConnectedSince = null consecutiveRpcFailures++ if (consecutiveRpcFailures < rpcNodeCount.coerceAtLeast(1)) return false source = Source.LIGHT + lightHeld = true rpcRetryAt = nowMillis + backoffMillis backoffMillis = (backoffMillis * 2).coerceAtMost(maxBackoffMillis) return true } - fun onRpcConnected() { + fun onRpcConnected(nowMillis: Long) { + if (rpcConnectedSince == null) rpcConnectedSince = nowMillis source = Source.RPC consecutiveRpcFailures = 0 backoffMillis = initialBackoffMillis @@ -56,6 +64,18 @@ class LightFallbackPolicy( return true } + /** + * True, once, when the light client should be released now: RPC has stayed connected for + * [releaseLightAfterMillis] since the light client was last used. + */ + fun shouldReleaseLight(nowMillis: Long): Boolean { + val since = rpcConnectedSince ?: return false + if (!lightHeld || nowMillis - since < releaseLightAfterMillis) return false + + lightHeld = false + return true + } + /** The light client could not start or its connection failed: RPC is all there is. */ fun onLightUnavailable(rpcNodeCount: Int) { if (source == Source.LIGHT) returnToRpc(rpcNodeCount) diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightSpecs.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightSpecs.kt new file mode 100644 index 00000000..5657b6cf --- /dev/null +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightSpecs.kt @@ -0,0 +1,57 @@ +package io.novafoundation.nova.runtime.multiNetwork.connection.light + +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import java.math.BigInteger + +/** Which of two light client chain specifications to keep. */ +object LightSpecs { + + /** + * True when [candidate] is the same chain as [current] and should replace it: a relay chain only + * for a later checkpoint, so an older file never rolls the checkpoint back; a chain without a + * checkpoint whenever it differs (boot nodes change). + */ + fun isUpdate(current: String, candidate: String): Boolean { + val currentSpec = parse(current) ?: return false + val candidateSpec = parse(candidate) ?: return false + + if (currentSpec.chainId() != candidateSpec.chainId() || currentSpec.stateRootHash() != candidateSpec.stateRootHash()) return false + + val currentCheckpoint = currentSpec.checkpointNumber() + val candidateCheckpoint = candidateSpec.checkpointNumber() + + return when { + currentCheckpoint == null && candidateCheckpoint == null -> currentSpec != candidateSpec + currentCheckpoint == null -> true + candidateCheckpoint == null -> false + else -> candidateCheckpoint > currentCheckpoint + } + } + + private fun parse(spec: String): JsonObject? = runCatching { JsonParser.parseString(spec).asJsonObject }.getOrNull() + + private fun JsonObject.chainId(): String? = get("id")?.asString + + private fun JsonObject.stateRootHash(): String? = getAsJsonObject("genesis")?.get("stateRootHash")?.asString + + /** Block number of the checkpoint header: SCALE, a 32-byte parent hash then a compact number. */ + private fun JsonObject.checkpointNumber(): BigInteger? = runCatching { + val header = getAsJsonObject("lightSyncState").get("finalizedBlockHeader").asString.removePrefix("0x") + decodeCompact(hexToBytes(header.substring(64, minOf(header.length, 64 + 18)))) + }.getOrNull() + + private fun decodeCompact(bytes: ByteArray): BigInteger { + val first = bytes[0].toInt() and 0xff + val littleEndian = { from: Int, count: Int -> BigInteger(1, bytes.copyOfRange(from, from + count).reversedArray()) } + + return when (first and 0b11) { + 0b00 -> BigInteger.valueOf((first shr 2).toLong()) + 0b01 -> littleEndian(0, 2).shiftRight(2) + 0b10 -> littleEndian(0, 4).shiftRight(2) + else -> littleEndian(1, (first shr 2) + 4) + } + } + + private fun hexToBytes(hex: String) = ByteArray(hex.length / 2) { hex.substring(it * 2, it * 2 + 2).toInt(16).toByte() } +} diff --git a/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicyTest.kt b/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicyTest.kt index 1444397b..af2ded7b 100644 --- a/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicyTest.kt +++ b/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightFallbackPolicyTest.kt @@ -9,7 +9,7 @@ import org.junit.Test class LightFallbackPolicyTest { private val minute = 60_000L - private val policy = LightFallbackPolicy(initialBackoffMillis = minute, maxBackoffMillis = 4 * minute) + private val policy = LightFallbackPolicy(initialBackoffMillis = minute, maxBackoffMillis = 4 * minute, releaseLightAfterMillis = 5 * minute) @Test fun `light client takes over only after every RPC node failed in a row`() { @@ -24,7 +24,7 @@ class LightFallbackPolicyTest { @Test fun `a connected RPC node resets the count`() { policy.onRpcFailure(rpcNodeCount = 2, nowMillis = 0) - policy.onRpcConnected() + policy.onRpcConnected(nowMillis = 0) assertFalse(policy.onRpcFailure(rpcNodeCount = 2, nowMillis = 0)) assertTrue(policy.onRpcFailure(rpcNodeCount = 2, nowMillis = 0)) @@ -68,7 +68,7 @@ class LightFallbackPolicyTest { policy.onTick(rpcNodeCount = 1, nowMillis = minute) policy.onRpcFailure(rpcNodeCount = 1, nowMillis = minute) policy.onTick(rpcNodeCount = 1, nowMillis = 3 * minute) - policy.onRpcConnected() + policy.onRpcConnected(nowMillis = 3 * minute) assertTrue(policy.onRpcFailure(rpcNodeCount = 1, nowMillis = 10 * minute)) assertFalse(policy.onTick(rpcNodeCount = 1, nowMillis = 11 * minute - 1)) @@ -91,4 +91,53 @@ class LightFallbackPolicyTest { assertFalse(policy.onRpcFailure(rpcNodeCount = 1, nowMillis = 0)) assertEquals(Source.LIGHT, policy.source) } + + @Test + fun `the light client is released once RPC has stayed connected for the release delay`() { + useLightThenReturnToRpcAt(minute) + + assertFalse(policy.shouldReleaseLight(nowMillis = 6 * minute - 1)) + assertTrue(policy.shouldReleaseLight(nowMillis = 6 * minute)) + assertFalse(policy.shouldReleaseLight(nowMillis = 7 * minute)) + } + + @Test + fun `an RPC failure restarts the release delay`() { + useLightThenReturnToRpcAt(minute) + + policy.onRpcFailure(rpcNodeCount = 2, nowMillis = 3 * minute) + policy.onRpcConnected(nowMillis = 4 * minute) + + assertFalse(policy.shouldReleaseLight(nowMillis = 9 * minute - 1)) + assertTrue(policy.shouldReleaseLight(nowMillis = 9 * minute)) + } + + @Test + fun `a repeated connected state does not restart the release delay`() { + useLightThenReturnToRpcAt(minute) + policy.onRpcConnected(nowMillis = 3 * minute) + + assertTrue(policy.shouldReleaseLight(nowMillis = 6 * minute)) + } + + @Test + fun `nothing is released while the light client serves the connection`() { + policy.onRpcFailure(rpcNodeCount = 1, nowMillis = 0) + + assertFalse(policy.shouldReleaseLight(nowMillis = 100 * minute)) + } + + @Test + fun `a light client that was never used is not released`() { + policy.onRpcConnected(nowMillis = 0) + + assertFalse(policy.shouldReleaseLight(nowMillis = 100 * minute)) + } + + private fun useLightThenReturnToRpcAt(nowMillis: Long) { + policy.onRpcFailure(rpcNodeCount = 2, nowMillis = 0) + policy.onRpcFailure(rpcNodeCount = 2, nowMillis = 0) + policy.onTick(rpcNodeCount = 2, nowMillis = nowMillis) + policy.onRpcConnected(nowMillis = nowMillis) + } } diff --git a/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightSpecsTest.kt b/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightSpecsTest.kt new file mode 100644 index 00000000..c30d2c47 --- /dev/null +++ b/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/connection/light/LightSpecsTest.kt @@ -0,0 +1,69 @@ +package io.novafoundation.nova.runtime.multiNetwork.connection.light + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class LightSpecsTest { + + @Test + fun `a relay chain specification is replaced only by a later checkpoint`() { + assertTrue(LightSpecs.isUpdate(relay(checkpoint = 1_000), relay(checkpoint = 1_020_017))) + assertFalse(LightSpecs.isUpdate(relay(checkpoint = 1_020_017), relay(checkpoint = 1_000))) + assertFalse(LightSpecs.isUpdate(relay(checkpoint = 1_000), relay(checkpoint = 1_000))) + } + + @Test + fun `a checkpoint does not move to another chain`() { + assertFalse(LightSpecs.isUpdate(relay(checkpoint = 1), relay(checkpoint = 2, stateRoot = "0x02"))) + assertFalse(LightSpecs.isUpdate(relay(checkpoint = 1), relay(checkpoint = 2, id = "pezkuwi"))) + } + + @Test + fun `a chain without a checkpoint takes any change of the same chain`() { + assertTrue(LightSpecs.isUpdate(people(bootNode = "a"), people(bootNode = "b"))) + assertFalse(LightSpecs.isUpdate(people(bootNode = "a"), people(bootNode = "a"))) + } + + @Test + fun `a specification that dropped its checkpoint is not taken`() { + assertFalse(LightSpecs.isUpdate(relay(checkpoint = 5), relay(checkpoint = null))) + } + + @Test + fun `an unreadable candidate is not taken`() { + assertFalse(LightSpecs.isUpdate(relay(checkpoint = 5), "")) + assertFalse(LightSpecs.isUpdate(relay(checkpoint = 5), "{}")) + } + + @Test + fun `compact numbers of every width are read`() { + for (number in listOf(0, 63, 64, 16_383, 16_384, 1_073_741_823, 1_073_741_824)) { + assertTrue("$number", LightSpecs.isUpdate(relay(checkpoint = number.toLong()), relay(checkpoint = number.toLong() + 1))) + assertFalse("$number", LightSpecs.isUpdate(relay(checkpoint = number.toLong() + 1), relay(checkpoint = number.toLong()))) + } + } + + private fun relay(checkpoint: Long?, stateRoot: String = "0x01", id: String = "zagros"): String { + val sync = checkpoint?.let { """, "lightSyncState": {"finalizedBlockHeader": "0x${"00".repeat(32)}${compact(it)}${"00".repeat(64)}"}""" } ?: "" + return """{"id": "$id", "genesis": {"stateRootHash": "$stateRoot"}, "bootNodes": []$sync}""" + } + + private fun people(bootNode: String) = """{"id": "people-zagros", "genesis": {"stateRootHash": "0x03"}, "bootNodes": ["$bootNode"]}""" + + /** SCALE compact encoding, little endian, as hex. */ + private fun compact(value: Long): String { + val bytes = when { + value < 1 shl 6 -> byteArrayOf((value shl 2).toByte()) + value < 1 shl 14 -> le((value shl 2) or 0b01, 2) + value < 1 shl 30 -> le((value shl 2) or 0b10, 4) + else -> { + val body = le(value, 8).dropLastWhile { it == 0.toByte() }.toByteArray() + byteArrayOf((((body.size - 4) shl 2) or 0b11).toByte()) + body + } + } + return bytes.joinToString("") { "%02x".format(it) } + } + + private fun le(value: Long, size: Int) = ByteArray(size) { (value shr (8 * it)).toByte() } +}