diff --git a/build.gradle b/build.gradle index 94a580fc..6d7d59cf 100644 --- a/build.gradle +++ b/build.gradle @@ -153,6 +153,7 @@ buildscript { retrofitDep = "com.squareup.retrofit2:retrofit:$retrofitVersion" interceptorVersion = "com.squareup.okhttp3:logging-interceptor:$okhttpVersion" + okhttpDnsOverHttpsDep = "com.squareup.okhttp3:okhttp-dnsoverhttps:$okhttpVersion" gsonConvertedDep = "com.squareup.retrofit2:converter-gson:$retrofitVersion" scalarsConverterDep = "com.squareup.retrofit2:converter-scalars:$retrofitVersion" diff --git a/common/build.gradle b/common/build.gradle index 903043b4..fdae3b00 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -115,6 +115,7 @@ dependencies { api gsonConvertedDep implementation scalarsConverterDep implementation interceptorVersion + implementation okhttpDnsOverHttpsDep implementation zXingCoreDep implementation zXingEmbeddedDep diff --git a/common/src/main/java/io/novafoundation/nova/common/data/network/dns/EncryptedFirstDns.kt b/common/src/main/java/io/novafoundation/nova/common/data/network/dns/EncryptedFirstDns.kt new file mode 100644 index 00000000..df4b1722 --- /dev/null +++ b/common/src/main/java/io/novafoundation/nova/common/data/network/dns/EncryptedFirstDns.kt @@ -0,0 +1,79 @@ +package io.novafoundation.nova.common.data.network.dns + +import okhttp3.Dns +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.OkHttpClient +import okhttp3.dnsoverhttps.DnsOverHttps +import java.net.Inet6Address +import java.net.InetAddress +import java.net.UnknownHostException +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +/** + * Resolves names over DNS-over-HTTPS first, so a network that answers plain DNS with a wrong address does not + * decide where the app connects; if every encrypted resolver fails, the system resolver answers, so no network + * is worse off than before. A resolver that failed is skipped for [resolverDownMillis], so a network that blocks + * DoH pays its timeout once, not on every lookup. + */ +class EncryptedFirstDns( + private val resolvers: List, + private val fallback: Dns = Dns.SYSTEM, + private val resolverDownMillis: Long = TimeUnit.MINUTES.toMillis(10), + private val clock: () -> Long = System::currentTimeMillis, +) : Dns { + + companion object { + + /** Google, Cloudflare and Quad9, each reached by IP so resolving the resolver needs no plain DNS. */ + fun publicResolvers(bootstrapClient: OkHttpClient): List = listOf( + doh(bootstrapClient, "https://dns.google/dns-query", "8.8.8.8", "8.8.4.4", "2001:4860:4860::8888"), + doh(bootstrapClient, "https://cloudflare-dns.com/dns-query", "1.1.1.1", "1.0.0.1", "2606:4700:4700::1111"), + doh(bootstrapClient, "https://dns.quad9.net/dns-query", "9.9.9.9", "149.112.112.112", "2620:fe::fe"), + ) + + private fun doh(client: OkHttpClient, url: String, vararg bootstrap: String): Dns = DnsOverHttps.Builder() + .client(client) + .url(url.toHttpUrl()) + .bootstrapDnsHosts(bootstrap.map { InetAddress.getByName(it) }) + .includeIPv6(true) + .build() + + private val IP_LITERAL = Regex("^[0-9.]+$|:") + } + + private val downUntil = ConcurrentHashMap() + + override fun lookup(hostname: String): List { + // Addresses and local names never leave the device. + if (IP_LITERAL.containsMatchIn(hostname) || hostname == "localhost") return fallback.lookup(hostname) + + resolvers.forEachIndexed { index, resolver -> + if (isDown(index)) return@forEachIndexed + + try { + val addresses = resolver.lookup(hostname) + // DoH answers IPv6 whether or not this network has it; the system resolver would not. OkHttp tries + // addresses in order, so IPv4 goes first and a broken IPv6 route cannot stall every connection. + if (addresses.isNotEmpty()) return addresses.sortedBy { it is Inet6Address } + } catch (e: UnknownHostException) { + // DnsOverHttps reports an unreachable resolver with the I/O failure as the cause; NXDOMAIN and + // SERVFAIL come without one and say nothing about the resolver. + if (e.cause != null) markDown(index) + } + } + + return fallback.lookup(hostname) + } + + private fun isDown(index: Int): Boolean { + val until = downUntil[index] ?: return false + if (clock() < until) return true + downUntil.remove(index, until) + return false + } + + private fun markDown(index: Int) { + downUntil[index] = clock() + resolverDownMillis + } +} diff --git a/common/src/main/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptor.kt b/common/src/main/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptor.kt new file mode 100644 index 00000000..e1080539 --- /dev/null +++ b/common/src/main/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptor.kt @@ -0,0 +1,134 @@ +package io.novafoundation.nova.common.data.network.mirror + +import okhttp3.CacheControl +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +/** + * Every config file the app reads (chains, assets, DApps, XCM, banners, icons) lives on + * raw.githubusercontent.com. When that host cannot be reached -- blocked, down, rate limited -- a fresh + * install has nothing to show. This interceptor retries the same file on mirrors that serve the same + * repository content, in order, and remembers for [hostDownMillis] that the primary failed so the next + * requests do not each wait for its timeout first. + * + * A 404 is an answer, not an outage: it is returned as is and no mirror is asked. + */ +class ConfigMirrorInterceptor( + private val mirrors: List = DEFAULT_MIRRORS, + private val hostDownMillis: Long = TimeUnit.MINUTES.toMillis(10), + private val clock: () -> Long = System::currentTimeMillis, +) : Interceptor { + + companion object { + const val GITHUB_RAW_HOST = "raw.githubusercontent.com" + + val DEFAULT_MIRRORS = listOf( + GithubMirror.JsDelivr("cdn.jsdelivr.net"), + GithubMirror.JsDelivr("fastly.jsdelivr.net"), + ) + + private val OUTAGE_CODES = setOf(403, 429, 451) + } + + private val downUntil = ConcurrentHashMap() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val file = GithubFile.parse(request.url) + + if (file == null || request.method != "GET") return chain.proceed(request) + + val candidates = listOf(request) + mirrors.map { mirror -> + request.newBuilder() + .url(mirror.urlOf(file).newBuilder().encodedQuery(request.url.encodedQuery).build()) + // A mirror's max-age would pin its copy in the HTTP cache long after the primary is back. + .cacheControl(CacheControl.Builder().noCache().build()) + .build() + } + + val available = candidates.filterNot { isDown(it.url.host) }.ifEmpty { candidates } + + var lastFailure: IOException? = null + var lastOutage: Response? = null + + for ((index, candidate) in available.withIndex()) { + val isLast = index == available.lastIndex + + val response = try { + chain.proceed(candidate) + } catch (e: IOException) { + markDown(candidate.url.host) + lastFailure = e + continue + } + + if (!isOutage(response) || isLast) { + if (!isOutage(response)) downUntil.remove(candidate.url.host) + lastOutage?.close() + return response + } + + markDown(candidate.url.host) + lastOutage?.close() + lastOutage = response + } + + lastOutage?.let { return it } + throw lastFailure ?: IOException("No source answered for ${request.url}") + } + + private fun isOutage(response: Response) = response.code in OUTAGE_CODES || response.code >= 500 + + private fun isDown(host: String): Boolean { + val until = downUntil[host] ?: return false + if (clock() < until) return true + downUntil.remove(host, until) + return false + } + + private fun markDown(host: String) { + downUntil[host] = clock() + hostDownMillis + } +} + +/** A file in a GitHub repository at a ref, as raw.githubusercontent.com addresses it. */ +data class GithubFile(val owner: String, val repo: String, val ref: String, val path: String) { + + companion object { + + fun parse(url: HttpUrl): GithubFile? { + if (url.host != ConfigMirrorInterceptor.GITHUB_RAW_HOST) return null + + val segments = url.pathSegments + if (segments.size < 4) return null + + val (owner, repo) = segments + // raw accepts both `master` and `refs/heads/master`; mirrors only know the short form. + val refLength = if (segments[2] == "refs" && segments.size >= 6) 3 else 1 + val ref = segments.subList(2, 2 + refLength).last() + val path = segments.drop(2 + refLength).joinToString("/") + + if (path.isEmpty()) return null + + return GithubFile(owner, repo, ref, path) + } + } +} + +sealed interface GithubMirror { + + fun urlOf(file: GithubFile): HttpUrl + + /** jsDelivr serves any public GitHub repository at `/gh/{owner}/{repo}@{ref}/{path}`. */ + class JsDelivr(private val host: String) : GithubMirror { + + override fun urlOf(file: GithubFile): HttpUrl { + return "https://$host/gh/${file.owner}/${file.repo}@${file.ref}/${file.path}".toHttpUrl() + } + } +} diff --git a/common/src/main/java/io/novafoundation/nova/common/data/network/signed/SignedConfig.kt b/common/src/main/java/io/novafoundation/nova/common/data/network/signed/SignedConfig.kt new file mode 100644 index 00000000..b1828a9c --- /dev/null +++ b/common/src/main/java/io/novafoundation/nova/common/data/network/signed/SignedConfig.kt @@ -0,0 +1,172 @@ +package io.novafoundation.nova.common.data.network.signed + +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters +import org.bouncycastle.crypto.signers.Ed25519Signer +import org.bouncycastle.util.encoders.Base64 +import java.security.MessageDigest + +/** + * Signed config metadata, after TUF: a root, signed by a threshold of root keys, names the keys that may sign the + * targets list and how many must; the targets list gives the sha256 of every config file. The APK carries a first + * root. A new root is accepted only one version at a time and only when a threshold of the old root's keys and of + * its own keys signed it, so keys can be replaced without an app release and a stolen targets key can be revoked. + * + * Every metadata file is an envelope: `{"signed": base64(payload), "signatures": [{"keyid", "sig"}]}`, where the + * signature is Ed25519 over the payload bytes exactly as published and keyid is hex sha256 of the 32-byte key. + */ +class SignedConfigException(message: String) : Exception(message) + +class Envelope(val payload: ByteArray, val signatures: List>) { + + companion object { + + fun parse(bytes: ByteArray): Envelope = guard { + val json = parseJson(String(bytes, Charsets.UTF_8)).asJsonObject + val signatures = json.getAsJsonArray("signatures").map { + val signature = it.asJsonObject + signature["keyid"].asString to Base64.decode(signature["sig"].asString) + } + + Envelope(Base64.decode(json["signed"].asString), signatures) + } + } + + /** How many of [keyIds] signed the payload; a key signing twice counts once, a key not in [keys] not at all. */ + fun validSignatures(keys: Map, keyIds: Set): Int { + return signatures + .filter { (keyId, signature) -> keyId in keyIds && keys[keyId]?.let { verifyEd25519(it, payload, signature) } == true } + .map { (keyId, _) -> keyId } + .toSet() + .size + } + + fun payloadJson(expectedType: String): JsonObject = guard { + val json = parseJson(String(payload, Charsets.UTF_8)).asJsonObject + if (json["type"].asString != expectedType) throw SignedConfigException("expected $expectedType, got ${json["type"]}") + json + } +} + +class Role(val keyIds: Set, val threshold: Int) + +class RootMetadata( + val version: Long, + val expiresAt: Long, + val keys: Map, + val root: Role, + val targets: Role, +) { + + companion object { + + fun parse(envelope: Envelope): RootMetadata = guard { + val json = envelope.payloadJson("root") + val keys = json.getAsJsonObject("keys").entrySet().associate { (keyId, key) -> + val bytes = Base64.decode(key.asString) + if (bytes.size != 32 || keyIdOf(bytes) != keyId) throw SignedConfigException("key $keyId does not match its id") + keyId to bytes + } + + RootMetadata( + version = json["version"].asLong, + expiresAt = json["expires"].asLong, + keys = keys, + root = parseRole(json.getAsJsonObject("root"), keys), + targets = parseRole(json.getAsJsonObject("targets"), keys), + ) + } + + private fun parseRole(json: JsonObject, keys: Map): Role { + val keyIds = json.getAsJsonArray("keyids").map { it.asString }.toSet() + val threshold = json["threshold"].asInt + if (threshold < 1 || threshold > keyIds.size || !keys.keys.containsAll(keyIds)) throw SignedConfigException("unusable role") + return Role(keyIds, threshold) + } + } +} + +class TargetsMetadata(val version: Long, val expiresAt: Long, val files: Map) { + + companion object { + + fun parse(envelope: Envelope): TargetsMetadata = guard { + val json = envelope.payloadJson("targets") + TargetsMetadata( + version = json["version"].asLong, + expiresAt = json["expires"].asLong, + files = json.getAsJsonObject("files").entrySet().associate { (path, hash) -> path to hash.asString.lowercase() }, + ) + } + } + + fun matches(path: String, content: ByteArray): Boolean = files[path] == sha256Hex(content) +} + +object SignedConfigRules { + + /** A targets version may move forward by at most this much at once, so a stolen key cannot freeze updates. */ + const val MAX_TARGETS_VERSION_JUMP = 10_000L + + /** The APK's own root: it must be signed by a threshold of its own root keys. */ + fun trustBundledRoot(envelope: Envelope): RootMetadata { + val root = RootMetadata.parse(envelope) + requireThreshold(envelope, root.keys, root.root, "bundled root") + return root + } + + /** The root that follows [trusted]; expiry is not checked here, so an expired root can still be rotated out. */ + fun nextRoot(trusted: RootMetadata, envelope: Envelope): RootMetadata { + val next = RootMetadata.parse(envelope) + if (next.version != trusted.version + 1) throw SignedConfigException("root ${next.version} does not follow ${trusted.version}") + requireThreshold(envelope, trusted.keys, trusted.root, "root ${next.version} by the previous root") + requireThreshold(envelope, next.keys, next.root, "root ${next.version} by itself") + return next + } + + /** + * [now] is seconds since the epoch; callers pass the latest of the device clock and anything already trusted, + * so turning the clock back does not revive an expired list. + */ + fun verifyTargets(root: RootMetadata, envelope: Envelope, lastTrustedVersion: Long?, now: Long): TargetsMetadata { + if (root.expiresAt <= now) throw SignedConfigException("root ${root.version} expired") + requireThreshold(envelope, root.keys, root.targets, "targets") + + val targets = TargetsMetadata.parse(envelope) + if (targets.expiresAt <= now) throw SignedConfigException("targets ${targets.version} expired") + if (lastTrustedVersion != null) { + if (targets.version < lastTrustedVersion) throw SignedConfigException("targets ${targets.version} is older than $lastTrustedVersion") + if (targets.version - lastTrustedVersion > MAX_TARGETS_VERSION_JUMP) throw SignedConfigException("targets ${targets.version} jumps too far") + } + return targets + } + + private fun requireThreshold(envelope: Envelope, keys: Map, role: Role, what: String) { + val valid = envelope.validSignatures(keys, role.keyIds) + if (valid < role.threshold) throw SignedConfigException("$what: $valid of ${role.threshold} required signatures") + } +} + +fun keyIdOf(publicKey: ByteArray): String = sha256Hex(publicKey) + +fun sha256Hex(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + +private fun verifyEd25519(publicKey: ByteArray, message: ByteArray, signature: ByteArray): Boolean = runCatching { + Ed25519Signer().apply { + init(false, Ed25519PublicKeyParameters(publicKey, 0)) + update(message, 0, message.size) + }.verifySignature(signature) +}.getOrDefault(false) + +private inline fun guard(block: () -> T): T = try { + block() +} catch (e: SignedConfigException) { + throw e +} catch (e: Exception) { + throw SignedConfigException("malformed metadata: $e") +} + +// The Gson the app resolves predates JsonParser.parseString. +@Suppress("DEPRECATION") +private fun parseJson(text: String) = JsonParser().parse(text) diff --git a/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt b/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt index 796543ea..077aed93 100644 --- a/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt +++ b/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt @@ -6,6 +6,7 @@ import android.content.SharedPreferences import android.graphics.Color import coil.ImageLoader import coil.decode.SvgDecoder +import coil.util.CoilUtils import com.google.android.play.core.integrity.IntegrityManagerFactory import dagger.Module import dagger.Provides @@ -24,6 +25,8 @@ import io.novafoundation.nova.common.data.memory.ComputationalCache import io.novafoundation.nova.common.data.memory.RealComputationalCache import io.novafoundation.nova.common.data.network.NetworkApiCreator import io.novafoundation.nova.common.data.network.coingecko.CoinGeckoLinkParser +import io.novafoundation.nova.common.data.network.dns.EncryptedFirstDns +import io.novafoundation.nova.common.data.network.mirror.ConfigMirrorInterceptor import io.novafoundation.nova.common.data.providers.deviceid.AndroidDeviceIdProvider import io.novafoundation.nova.common.data.providers.deviceid.DeviceIdProvider import io.novafoundation.nova.common.data.repository.AssetsIconModeRepository @@ -116,6 +119,7 @@ import io.novafoundation.nova.common.view.input.selector.ListSelectorMixin import io.novafoundation.nova.common.view.input.selector.RealListSelectorMixinFactory import io.novasama.substrate_sdk_android.encrypt.Signer import io.novasama.substrate_sdk_android.icon.IconGenerator +import okhttp3.OkHttpClient import java.security.SecureRandom import java.util.Random import javax.inject.Qualifier @@ -143,7 +147,19 @@ class CommonModule { @Provides @ApplicationScope - fun imageLoader(context: Context) = ImageLoader.Builder(context) + fun imageLoader( + context: Context, + configMirrorInterceptor: ConfigMirrorInterceptor, + encryptedFirstDns: EncryptedFirstDns + ) = ImageLoader.Builder(context) + // Token icons live on raw.githubusercontent.com too; Coil keeps its own disk cache. + .okHttpClient { + OkHttpClient.Builder() + .cache(CoilUtils.createDefaultCache(context)) + .addInterceptor(configMirrorInterceptor) + .dns(encryptedFirstDns) + .build() + } .componentRegistry { add(SvgDecoder(context)) } diff --git a/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt b/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt index 4896f028..312f303f 100644 --- a/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt +++ b/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt @@ -11,6 +11,8 @@ import io.novafoundation.nova.common.data.network.AppLinksProvider import io.novafoundation.nova.common.data.network.HttpExceptionHandler import io.novafoundation.nova.common.data.network.NetworkApiCreator import io.novafoundation.nova.common.data.network.TimeHeaderInterceptor +import io.novafoundation.nova.common.data.network.dns.EncryptedFirstDns +import io.novafoundation.nova.common.data.network.mirror.ConfigMirrorInterceptor import io.novafoundation.nova.common.data.network.rpc.SocketSingleRequestExecutor import io.novafoundation.nova.common.di.scope.ApplicationScope import io.novafoundation.nova.common.mixin.api.NetworkStateMixin @@ -35,6 +37,9 @@ import java.util.concurrent.TimeUnit private const val HTTP_CACHE = "http_cache" private const val CACHE_SIZE = 50L * 1024L * 1024L // 50 MiB private const val TIMEOUT_SECONDS = 20L +private const val DOH_TIMEOUT_SECONDS = 4L +private const val DOH_CACHE = "doh_cache" +private const val DOH_CACHE_SIZE = 1L * 1024L * 1024L // 1 MiB private const val SOCKET_CONNECTION_TIMEOUT = 5_000 @@ -74,10 +79,30 @@ class NetworkModule { ) } + @Provides + @ApplicationScope + fun provideConfigMirrorInterceptor() = ConfigMirrorInterceptor() + + @Provides + @ApplicationScope + fun provideEncryptedFirstDns(context: Context): EncryptedFirstDns { + // Only carries DNS queries to the resolvers, which it reaches by IP; short timeouts so a network that + // blocks DoH falls back to the system resolver quickly. + val resolverClient = OkHttpClient.Builder() + .connectTimeout(DOH_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .readTimeout(DOH_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .cache(Cache(File(context.cacheDir, DOH_CACHE), DOH_CACHE_SIZE)) + .build() + + return EncryptedFirstDns(EncryptedFirstDns.publicResolvers(resolverClient)) + } + @Provides @ApplicationScope fun provideOkHttpClient( - context: Context + context: Context, + configMirrorInterceptor: ConfigMirrorInterceptor, + encryptedFirstDns: EncryptedFirstDns ): OkHttpClient { val builder = OkHttpClient.Builder() .connectTimeout(TIMEOUT_SECONDS, TimeUnit.SECONDS) @@ -85,7 +110,9 @@ class NetworkModule { .readTimeout(TIMEOUT_SECONDS, TimeUnit.SECONDS) .cache(Cache(File(context.cacheDir, HTTP_CACHE), CACHE_SIZE)) .retryOnConnectionFailure(true) + .dns(encryptedFirstDns) .addInterceptor(TimeHeaderInterceptor()) + .addInterceptor(configMirrorInterceptor) if (BuildConfig.DEBUG) { builder.addInterceptor(HttpLoggingInterceptor().setLevel(HttpLoggingInterceptor.Level.BODY)) diff --git a/common/src/test/java/io/novafoundation/nova/common/data/network/dns/EncryptedFirstDnsTest.kt b/common/src/test/java/io/novafoundation/nova/common/data/network/dns/EncryptedFirstDnsTest.kt new file mode 100644 index 00000000..4c2ab7a7 --- /dev/null +++ b/common/src/test/java/io/novafoundation/nova/common/data/network/dns/EncryptedFirstDnsTest.kt @@ -0,0 +1,92 @@ +package io.novafoundation.nova.common.data.network.dns + +import okhttp3.Dns +import org.junit.Assert.assertEquals +import org.junit.Test +import java.io.IOException +import java.net.InetAddress +import java.net.UnknownHostException + +class EncryptedFirstDnsTest { + + private val honest = InetAddress.getByName("203.0.113.7") + private val poisoned = InetAddress.getByName("192.0.2.66") + + private var now = 0L + private val calls = mutableListOf() + + private fun resolver(name: String, answer: () -> List) = object : Dns { + override fun lookup(hostname: String): List { + calls += "$name:$hostname" + return answer() + } + } + + private val system = resolver("system") { listOf(poisoned) } + + private fun unreachable(): Nothing = throw UnknownHostException("h").apply { initCause(IOException("connect timed out")) } + + @Test + fun `the encrypted answer wins over the system resolver`() { + val dns = EncryptedFirstDns(listOf(resolver("doh1") { listOf(honest) }), system) + + assertEquals(listOf(honest), dns.lookup("raw.githubusercontent.com")) + assertEquals(listOf("doh1:raw.githubusercontent.com"), calls) + } + + @Test + fun `IPv4 addresses come first`() { + val v6 = InetAddress.getByName("2001:db8::1") + val dns = EncryptedFirstDns(listOf(resolver("doh1") { listOf(v6, honest) }), system) + + assertEquals(listOf(honest, v6), dns.lookup("a.example")) + } + + @Test + fun `an unreachable resolver hands over to the next`() { + val dns = EncryptedFirstDns(listOf(resolver("doh1") { unreachable() }, resolver("doh2") { listOf(honest) }), system) + + assertEquals(listOf(honest), dns.lookup("a.example")) + } + + @Test + fun `when every encrypted resolver fails the system resolver answers`() { + val dns = EncryptedFirstDns(listOf(resolver("doh1") { unreachable() }), system) + + assertEquals(listOf(poisoned), dns.lookup("a.example")) + } + + @Test + fun `an unreachable resolver is not waited on again inside its window`() { + val dns = EncryptedFirstDns(listOf(resolver("doh1") { unreachable() }), system, resolverDownMillis = 1_000, clock = { now }) + dns.lookup("a.example") + calls.clear() + + dns.lookup("b.example") + assertEquals(listOf("system:b.example"), calls) + + now += 1_001 + calls.clear() + dns.lookup("c.example") + assertEquals(listOf("doh1:c.example", "system:c.example"), calls) + } + + @Test + fun `a name that does not exist does not take the resolver out`() { + val dns = EncryptedFirstDns(listOf(resolver("doh1") { throw UnknownHostException("x: NXDOMAIN") }), system, clock = { now }) + dns.lookup("missing.example") + calls.clear() + + dns.lookup("b.example") + assertEquals(listOf("doh1:b.example", "system:b.example"), calls) + } + + @Test + fun `addresses and localhost never leave the device`() { + val dns = EncryptedFirstDns(listOf(resolver("doh1") { listOf(honest) }), resolver("system") { emptyList() }) + + listOf("10.0.2.2", "::1", "localhost").forEach { dns.lookup(it) } + + assertEquals(listOf("system:10.0.2.2", "system:::1", "system:localhost"), calls) + } +} diff --git a/common/src/test/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptorTest.kt b/common/src/test/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptorTest.kt new file mode 100644 index 00000000..030bccd6 --- /dev/null +++ b/common/src/test/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptorTest.kt @@ -0,0 +1,124 @@ +package io.novafoundation.nova.common.data.network.mirror + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test +import java.io.IOException + +private const val RAW = "https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/android/chains.json" +private const val CDN = "https://cdn.jsdelivr.net/gh/pezkuwichain/pezkuwi-wallet-utils@master/chains/v22/android/chains.json" +private const val FASTLY = "https://fastly.jsdelivr.net/gh/pezkuwichain/pezkuwi-wallet-utils@master/chains/v22/android/chains.json" + +class ConfigMirrorInterceptorTest { + + private var now = 0L + private val hits = mutableListOf() + + /** What each host does, standing in for the network: a status code, or null for a connection failure. */ + private val hosts = mutableMapOf() + + private val client = OkHttpClient.Builder() + .addInterceptor(ConfigMirrorInterceptor(hostDownMillis = 1_000, clock = { now })) + .addInterceptor(Interceptor { chain -> fakeNetwork(chain.request()) }) + .build() + + @Test + fun `a blocked primary is served from the first mirror`() { + hosts["raw.githubusercontent.com"] = null + hosts["cdn.jsdelivr.net"] = 200 + + assertEquals("cdn.jsdelivr.net", fetch(RAW)) + assertEquals(listOf(RAW, CDN), hits) + } + + @Test + fun `mirrors are tried in order until one answers`() { + hosts["raw.githubusercontent.com"] = 451 + hosts["cdn.jsdelivr.net"] = null + hosts["fastly.jsdelivr.net"] = 200 + + assertEquals("fastly.jsdelivr.net", fetch(RAW)) + assertEquals(listOf(RAW, CDN, FASTLY), hits) + } + + @Test + fun `a server error on the primary goes to a mirror`() { + hosts["raw.githubusercontent.com"] = 503 + hosts["cdn.jsdelivr.net"] = 200 + + assertEquals("cdn.jsdelivr.net", fetch(RAW)) + } + + @Test + fun `a missing file is an answer and no mirror is asked`() { + hosts["raw.githubusercontent.com"] = 404 + + assertNull(fetch(RAW)) + assertEquals(listOf(RAW), hits) + } + + @Test + fun `a failed primary is skipped until its window passes`() { + hosts["raw.githubusercontent.com"] = null + hosts["cdn.jsdelivr.net"] = 200 + fetch(RAW) + hits.clear() + + fetch(RAW) + assertEquals("the primary is not waited on again inside the window", listOf(CDN), hits) + + now += 1_001 + hits.clear() + hosts["raw.githubusercontent.com"] = 200 + assertEquals("raw.githubusercontent.com", fetch(RAW)) + assertEquals(listOf(RAW), hits) + } + + @Test(expected = IOException::class) + fun `when every source fails the failure reaches the caller`() { + fetch(RAW) + } + + @Test + fun `other hosts are left alone`() { + hosts["example.org"] = null + + runCatching { fetch("https://example.org/a.json") } + assertEquals(listOf("https://example.org/a.json"), hits) + } + + @Test + fun `the long ref form maps to the mirror's short form`() { + val file = GithubFile.parse("https://raw.githubusercontent.com/o/r/refs/heads/master/a/b.json".toHttpUrl()) + + assertEquals(GithubFile("o", "r", "master", "a/b.json"), file) + } + + private fun fetch(url: String): String? { + client.newCall(Request.Builder().url(url).build()).execute().use { response -> + return if (response.isSuccessful) response.body!!.string() else null + } + } + + private fun fakeNetwork(request: Request): Response { + hits += request.url.toString() + + val code = hosts[request.url.host] ?: throw IOException("unreachable: ${request.url.host}") + + return Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message("") + .body(request.url.host.toResponseBody("text/plain".toMediaType())) + .build() + } +} diff --git a/common/src/test/java/io/novafoundation/nova/common/data/network/signed/SignedConfigTest.kt b/common/src/test/java/io/novafoundation/nova/common/data/network/signed/SignedConfigTest.kt new file mode 100644 index 00000000..2a6853ce --- /dev/null +++ b/common/src/test/java/io/novafoundation/nova/common/data/network/signed/SignedConfigTest.kt @@ -0,0 +1,157 @@ +package io.novafoundation.nova.common.data.network.signed + +import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters +import org.bouncycastle.crypto.signers.Ed25519Signer +import org.bouncycastle.util.encoders.Base64 +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.security.SecureRandom + +class SignedConfigTest { + + private class Key { + val private = Ed25519PrivateKeyParameters(SecureRandom()) + val public: ByteArray = private.generatePublicKey().encoded + val id = keyIdOf(public) + + fun sign(payload: ByteArray): ByteArray = Ed25519Signer().apply { + init(true, private) + update(payload, 0, payload.size) + }.generateSignature() + } + + private val rootKeys = List(3) { Key() } + private val targetsKey = Key() + private val now = 1_800_000_000L + + private fun envelope(payload: String, vararg signers: Key): Envelope { + val bytes = payload.toByteArray() + val signatures = signers.joinToString(",") { """{"keyid":"${it.id}","sig":"${Base64.toBase64String(it.sign(bytes))}"}""" } + return Envelope.parse("""{"signed":"${Base64.toBase64String(bytes)}","signatures":[$signatures]}""".toByteArray()) + } + + private fun rootPayload(version: Long, root: List, threshold: Int = 2, targets: List = listOf(targetsKey), expires: Long = now + 1000): String { + val keys = (root + targets).distinct().joinToString(",") { """"${it.id}":"${Base64.toBase64String(it.public)}"""" } + val ids = { list: List -> list.joinToString(",") { "\"${it.id}\"" } } + return """{"type":"root","version":$version,"expires":$expires,"keys":{$keys},""" + + """"root":{"keyids":[${ids(root)}],"threshold":$threshold},"targets":{"keyids":[${ids(targets)}],"threshold":1}}""" + } + + private fun targetsPayload(version: Long, files: Map, expires: Long = now + 1000): String { + val entries = files.entries.joinToString(",") { (path, content) -> "\"$path\":\"${sha256Hex(content)}\"" } + return """{"type":"targets","version":$version,"expires":$expires,"files":{$entries}}""" + } + + private val bundledRoot = SignedConfigRules.trustBundledRoot(envelope(rootPayload(1, rootKeys), rootKeys[0], rootKeys[1])) + + @Test + fun `a root needs two of its three keys`() { + val one = envelope(rootPayload(1, rootKeys), rootKeys[0]) + val sameKeyTwice = envelope(rootPayload(1, rootKeys), rootKeys[0], rootKeys[0]) + + assertRejected { SignedConfigRules.trustBundledRoot(one) } + assertRejected { SignedConfigRules.trustBundledRoot(sameKeyTwice) } + } + + @Test + fun `targets signed by the targets key are trusted and checked file by file`() { + val chains = "[1,2,3]".toByteArray() + val targets = SignedConfigRules.verifyTargets(bundledRoot, envelope(targetsPayload(5, mapOf("chains.json" to chains)), targetsKey), null, now) + + assertTrue(targets.matches("chains.json", chains)) + assertFalse(targets.matches("chains.json", "[1,2,666]".toByteArray())) + assertFalse(targets.matches("other.json", chains)) + } + + @Test + fun `targets signed by anyone else are rejected`() { + val stranger = Key() + + assertRejected { SignedConfigRules.verifyTargets(bundledRoot, envelope(targetsPayload(5, emptyMap()), stranger), null, now) } + } + + @Test + fun `a payload changed after signing is rejected`() { + val signed = envelope(targetsPayload(5, emptyMap()), targetsKey) + val tampered = Envelope(targetsPayload(6, emptyMap()).toByteArray(), signed.signatures) + + assertRejected { SignedConfigRules.verifyTargets(bundledRoot, tampered, null, now) } + } + + @Test + fun `targets cannot go back, jump too far, or be used after they expire`() { + assertRejected { SignedConfigRules.verifyTargets(bundledRoot, envelope(targetsPayload(4, emptyMap()), targetsKey), 5, now) } + val tooFar = 5 + SignedConfigRules.MAX_TARGETS_VERSION_JUMP + 1 + assertRejected { SignedConfigRules.verifyTargets(bundledRoot, envelope(targetsPayload(tooFar, emptyMap()), targetsKey), 5, now) } + assertRejected { SignedConfigRules.verifyTargets(bundledRoot, envelope(targetsPayload(6, emptyMap(), expires = now), targetsKey), 5, now) } + + assertEquals(5L, SignedConfigRules.verifyTargets(bundledRoot, envelope(targetsPayload(5, emptyMap()), targetsKey), 5, now).version) + } + + @Test + fun `a metadata file of the wrong type is rejected`() { + // Well formed as targets in every field but its type: only the type check stands in the way. + val mislabelled = targetsPayload(5, emptyMap()).replace("\"type\":\"targets\"", "\"type\":\"root\"") + + assertRejected { SignedConfigRules.verifyTargets(bundledRoot, envelope(mislabelled, targetsKey), null, now) } + } + + @Test + fun `a new root needs the old keys and its own`() { + val newKeys = List(3) { Key() } + + val byOldOnly = envelope(rootPayload(2, newKeys), rootKeys[0], rootKeys[1]) + val byNewOnly = envelope(rootPayload(2, newKeys), newKeys[0], newKeys[1]) + val byBoth = envelope(rootPayload(2, newKeys), rootKeys[0], rootKeys[2], newKeys[1], newKeys[2]) + + assertRejected { SignedConfigRules.nextRoot(bundledRoot, byOldOnly) } + assertRejected { SignedConfigRules.nextRoot(bundledRoot, byNewOnly) } + assertEquals(2L, SignedConfigRules.nextRoot(bundledRoot, byBoth).version) + } + + @Test + fun `roots move one version at a time`() { + val skipping = envelope(rootPayload(3, rootKeys), rootKeys[0], rootKeys[1]) + + assertRejected { SignedConfigRules.nextRoot(bundledRoot, skipping) } + } + + @Test + fun `a rotated root revokes the old targets key`() { + val newTargetsKey = Key() + val root2 = SignedConfigRules.nextRoot(bundledRoot, envelope(rootPayload(2, rootKeys, targets = listOf(newTargetsKey)), rootKeys[0], rootKeys[1])) + + assertRejected { SignedConfigRules.verifyTargets(root2, envelope(targetsPayload(6, emptyMap()), targetsKey), 5, now) } + assertEquals(6L, SignedConfigRules.verifyTargets(root2, envelope(targetsPayload(6, emptyMap()), newTargetsKey), 5, now).version) + } + + @Test + fun `an expired root signs nothing`() { + val expired = SignedConfigRules.trustBundledRoot(envelope(rootPayload(1, rootKeys, expires = now), rootKeys[0], rootKeys[1])) + + assertRejected { SignedConfigRules.verifyTargets(expired, envelope(targetsPayload(5, emptyMap()), targetsKey), null, now) } + } + + @Test + fun `a key that does not match its id is rejected`() { + val forged = rootPayload(1, rootKeys).replace(Base64.toBase64String(rootKeys[2].public), Base64.toBase64String(Key().public)) + + assertRejected { SignedConfigRules.trustBundledRoot(envelope(forged, rootKeys[0], rootKeys[1])) } + } + + @Test + fun `garbage is rejected, not thrown as something else`() { + assertRejected { Envelope.parse("not json".toByteArray()) } + } + + private fun assertRejected(block: () -> Unit) { + try { + block() + } catch (e: SignedConfigException) { + return + } + throw AssertionError("expected SignedConfigException") + } +} diff --git a/feature-dapp-impl/build.gradle b/feature-dapp-impl/build.gradle index 030dda39..1eeb7e1e 100644 --- a/feature-dapp-impl/build.gradle +++ b/feature-dapp-impl/build.gradle @@ -1,6 +1,7 @@ apply plugin: 'kotlin-parcelize' apply from: '../tests.gradle' apply from: "../scripts/secrets.gradle" +apply from: "../scripts/bundled-config.gradle" android { @@ -21,6 +22,14 @@ android { } namespace 'io.novafoundation.nova.feature_dapp_impl' + testOptions { + unitTests { + // android.util.Log.* throws "Method ... not mocked" in plain JVM unit tests; the first-install + // fallback logs the failed fetch it recovers from. + returnDefaultValues = true + } + } + packagingOptions { resources.excludes.add("META-INF/NOTICE.md") } @@ -96,3 +105,10 @@ dependencies { testImplementation jUnitDep testImplementation mockitoDep } + +// The DApps catalog a first install shows when none can be fetched (AssetBundledDappCatalog). +bundleConfigAsset( + "bundleDappsConfig", + "https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/dapps/dapps_full.json", + "bundled-config/dapps.json" +) { it instanceof Map && it.dapps instanceof List && !it.dapps.isEmpty() } diff --git a/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/data/repository/BundledDappCatalog.kt b/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/data/repository/BundledDappCatalog.kt new file mode 100644 index 00000000..5fa785a0 --- /dev/null +++ b/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/data/repository/BundledDappCatalog.kt @@ -0,0 +1,34 @@ +package io.novafoundation.nova.feature_dapp_impl.data.repository + +import android.content.Context +import android.util.Log +import com.google.gson.Gson +import io.novafoundation.nova.common.utils.LOG_TAG +import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataResponse + +/** The DApps catalog the APK was built with: what is shown until a catalog can be fetched. */ +fun interface BundledDappCatalog { + + fun load(): DappMetadataResponse? + + companion object { + val None = BundledDappCatalog { null } + } +} + +class AssetBundledDappCatalog( + private val context: Context, + private val gson: Gson, +) : BundledDappCatalog { + + companion object { + // Written at build time by the `bundleDappsConfig` task in feature-dapp-impl/build.gradle. + const val ASSET_PATH = "bundled-config/dapps.json" + } + + override fun load(): DappMetadataResponse? = runCatching { + context.assets.open(ASSET_PATH).bufferedReader().use { gson.fromJson(it, DappMetadataResponse::class.java) } + }.onFailure { + Log.e(LOG_TAG, "No usable bundled DApps catalog", it) + }.getOrNull() +} diff --git a/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/data/repository/RealDAppMetadataRepository.kt b/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/data/repository/RealDAppMetadataRepository.kt index 8c999fc9..5ca08c1c 100644 --- a/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/data/repository/RealDAppMetadataRepository.kt +++ b/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/data/repository/RealDAppMetadataRepository.kt @@ -1,6 +1,8 @@ package io.novafoundation.nova.feature_dapp_impl.data.repository +import android.util.Log import io.novafoundation.nova.common.utils.CollectionDiffer +import io.novafoundation.nova.common.utils.LOG_TAG import io.novafoundation.nova.common.utils.Urls import io.novafoundation.nova.common.utils.retryUntilDone import io.novafoundation.nova.core_db.dao.BrowserHostSettingsDao @@ -11,6 +13,8 @@ import io.novafoundation.nova.feature_dapp_api.data.repository.DAppMetadataRepos import io.novafoundation.nova.feature_dapp_impl.data.mappers.mapDAppMetadataResponseToDAppMetadatas import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataApi import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataRemote +import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataResponse +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow @@ -19,7 +23,8 @@ import kotlinx.coroutines.flow.first class RealDAppMetadataRepository( private val dappMetadataApi: DappMetadataApi, private val remoteApiUrl: String, - private val browserHostSettingsDao: BrowserHostSettingsDao + private val browserHostSettingsDao: BrowserHostSettingsDao, + private val bundledCatalog: BundledDappCatalog = BundledDappCatalog.None, ) : DAppMetadataRepository { private val dappMetadatasFlow = MutableSharedFlow(replay = 1, onBufferOverflow = BufferOverflow.DROP_OLDEST) @@ -29,7 +34,30 @@ class RealDAppMetadataRepository( } override suspend fun syncDAppMetadatas() { - val response = retryUntilDone { dappMetadataApi.getParachainMetadata(remoteApiUrl) } + if (dappMetadatasFlow.replayCache.isEmpty()) { + // Every read below waits for a first catalog. Try the network once, and if that fails show the + // catalog the APK was built with, so the DApps screen opens; the fetched one replaces it below. + val firstTry = try { + dappMetadataApi.getParachainMetadata(remoteApiUrl) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Log.e(LOG_TAG, "First DApps catalog fetch failed, showing the bundled catalog", e) + null + } + + if (firstTry != null) { + applyCatalog(firstTry) + return + } + + bundledCatalog.load()?.let { dappMetadatasFlow.emit(mapDAppMetadataResponseToDAppMetadatas(it)) } + } + + applyCatalog(retryUntilDone { dappMetadataApi.getParachainMetadata(remoteApiUrl) }) + } + + private suspend fun applyCatalog(response: DappMetadataResponse) { val dappMetadatas = mapDAppMetadataResponseToDAppMetadatas(response) syncHostSettings(response.dapps) dappMetadatasFlow.emit(dappMetadatas) diff --git a/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/di/modules/DappMetadataModule.kt b/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/di/modules/DappMetadataModule.kt index 941da9c0..78e34f59 100644 --- a/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/di/modules/DappMetadataModule.kt +++ b/feature-dapp-impl/src/main/java/io/novafoundation/nova/feature_dapp_impl/di/modules/DappMetadataModule.kt @@ -1,5 +1,7 @@ package io.novafoundation.nova.feature_dapp_impl.di.modules +import android.content.Context +import com.google.gson.Gson import dagger.Module import dagger.Provides import io.novafoundation.nova.common.data.network.NetworkApiCreator @@ -8,6 +10,7 @@ import io.novafoundation.nova.core_db.dao.BrowserHostSettingsDao import io.novafoundation.nova.feature_dapp_api.data.repository.DAppMetadataRepository import io.novafoundation.nova.feature_dapp_impl.BuildConfig import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataApi +import io.novafoundation.nova.feature_dapp_impl.data.repository.AssetBundledDappCatalog import io.novafoundation.nova.feature_dapp_impl.data.repository.RealDAppMetadataRepository @Module @@ -23,10 +26,13 @@ class DappMetadataModule { @FeatureScope fun provideDRepository( api: DappMetadataApi, - dappHostSettingsDao: BrowserHostSettingsDao + dappHostSettingsDao: BrowserHostSettingsDao, + context: Context, + gson: Gson ): DAppMetadataRepository = RealDAppMetadataRepository( dappMetadataApi = api, remoteApiUrl = BuildConfig.DAPP_METADATAS_URL, - browserHostSettingsDao = dappHostSettingsDao + browserHostSettingsDao = dappHostSettingsDao, + bundledCatalog = AssetBundledDappCatalog(context, gson) ) } diff --git a/feature-dapp-impl/src/test/java/io/novafoundation/nova/feature_dapp_impl/data/repository/RealDAppMetadataRepositoryTest.kt b/feature-dapp-impl/src/test/java/io/novafoundation/nova/feature_dapp_impl/data/repository/RealDAppMetadataRepositoryTest.kt new file mode 100644 index 00000000..16cb4edb --- /dev/null +++ b/feature-dapp-impl/src/test/java/io/novafoundation/nova/feature_dapp_impl/data/repository/RealDAppMetadataRepositoryTest.kt @@ -0,0 +1,73 @@ +package io.novafoundation.nova.feature_dapp_impl.data.repository + +import io.novafoundation.nova.core_db.dao.BrowserHostSettingsDao +import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataApi +import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataRemote +import io.novafoundation.nova.feature_dapp_impl.data.network.metadata.DappMetadataResponse +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mock +import org.mockito.Mockito.`when` +import org.mockito.junit.MockitoJUnitRunner +import java.io.IOException + +@RunWith(MockitoJUnitRunner::class) +class RealDAppMetadataRepositoryTest { + + @Mock + lateinit var hostSettingsDao: BrowserHostSettingsDao + + private val bundled = catalogOf("https://bundled.example") + private val remote = catalogOf("https://remote.example") + + @Test + fun `a first install shows the bundled catalog until one is fetched`() = runBlocking { + `when`(hostSettingsDao.getBrowserAllHostSettings()).thenReturn(emptyList()) + val network = CompletableDeferred() + var calls = 0 + val api = object : DappMetadataApi { + override suspend fun getParachainMetadata(url: String): DappMetadataResponse { + calls++ + if (calls == 1) throw IOException("unreachable") + return network.await() + } + } + val repository = RealDAppMetadataRepository(api, "url", hostSettingsDao) { bundled } + + val sync = launch { repository.syncDAppMetadatas() } + while (!repository.isDAppsSynced()) yield() + + assertEquals("https://bundled.example", repository.getDAppCatalog().dApps.single().url) + + network.complete(remote) + sync.join() + + assertEquals("https://remote.example", repository.getDAppCatalog().dApps.single().url) + } + + @Test + fun `the bundled catalog is not read when the fetch works`() = runBlocking { + `when`(hostSettingsDao.getBrowserAllHostSettings()).thenReturn(emptyList()) + var bundledReads = 0 + val api = object : DappMetadataApi { + override suspend fun getParachainMetadata(url: String) = remote + } + val repository = RealDAppMetadataRepository(api, "url", hostSettingsDao) { bundledReads++; bundled } + + repository.syncDAppMetadatas() + + assertEquals(0, bundledReads) + assertEquals("https://remote.example", repository.getDAppCatalog().dApps.single().url) + } + + private fun catalogOf(url: String) = DappMetadataResponse( + popular = emptyList(), + categories = emptyList(), + dapps = listOf(DappMetadataRemote(name = "d", icon = "i", url = url, categories = emptyList(), desktopOnly = null)) + ) +} diff --git a/runtime/build.gradle b/runtime/build.gradle index 436183ea..ca53ccd1 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -1,11 +1,14 @@ apply from: '../scripts/secrets.gradle' apply plugin: 'kotlin-parcelize' +apply from: '../scripts/bundled-config.gradle' + +def chainsUrl = "https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/android/chains.json" android { defaultConfig { - buildConfigField "String", "CHAINS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/android/chains.json\"" + buildConfigField "String", "CHAINS_URL", "\"$chainsUrl\"" buildConfigField "String", "EVM_ASSETS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/assets/evm/v3/assets.json\"" buildConfigField "String", "PRE_CONFIGURED_CHAINS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/preConfigured/chains.json\"" buildConfigField "String", "PRE_CONFIGURED_CHAIN_DETAILS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/preConfigured/details\"" @@ -27,13 +30,21 @@ android { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' - buildConfigField "String", "CHAINS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/android/chains.json\"" + buildConfigField "String", "CHAINS_URL", "\"$chainsUrl\"" buildConfigField "String", "EVM_ASSETS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/assets/evm/v3/assets.json\"" buildConfigField "String", "PRE_CONFIGURED_CHAINS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/preConfigured/chains.json\"" buildConfigField "String", "PRE_CONFIGURED_CHAIN_DETAILS_URL", "\"https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/preConfigured/details\"" } } namespace 'io.novafoundation.nova.runtime' + + testOptions { + unitTests { + // android.util.Log.* throws "Method ... not mocked" in plain JVM unit tests; the first-install + // fallback logs the failed fetch it recovers from. + returnDefaultValues = true + } + } } dependencies { @@ -66,4 +77,7 @@ dependencies { androidTestImplementation androidTestRunnerDep androidTestImplementation androidTestRulesDep androidTestImplementation androidJunitDep -} \ No newline at end of file +} + +// The chain list a first install starts from when none can be fetched (AssetBundledChains). +bundleConfigAsset("bundleChainsConfig", chainsUrl, "bundled-config/chains.json") { it instanceof List && !it.isEmpty() } diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt index eff94b38..29ff4187 100644 --- a/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/di/ChainRegistryModule.kt @@ -1,5 +1,6 @@ package io.novafoundation.nova.runtime.di +import android.content.Context import com.google.gson.Gson import dagger.Module import dagger.Provides @@ -15,6 +16,7 @@ import io.novafoundation.nova.runtime.ethereum.Web3ApiFactory import io.novafoundation.nova.runtime.multiNetwork.ChainRegistry import io.novafoundation.nova.runtime.multiNetwork.asset.EvmAssetsSyncService import io.novafoundation.nova.runtime.multiNetwork.asset.remote.AssetFetcher +import io.novafoundation.nova.runtime.multiNetwork.chain.AssetBundledChains import io.novafoundation.nova.runtime.multiNetwork.chain.ChainSyncService import io.novafoundation.nova.runtime.multiNetwork.chain.remote.ChainFetcher import io.novafoundation.nova.runtime.multiNetwork.connection.ChainConnection @@ -56,8 +58,9 @@ class ChainRegistryModule { dao: ChainDao, chainAssetDao: ChainAssetDao, chainFetcher: ChainFetcher, - gson: Gson - ) = ChainSyncService(dao, chainFetcher, gson) + gson: Gson, + context: Context + ) = ChainSyncService(dao, chainFetcher, gson, AssetBundledChains(context, gson)) @Provides @ApplicationScope diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/chain/BundledChains.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/chain/BundledChains.kt new file mode 100644 index 00000000..476ed64c --- /dev/null +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/chain/BundledChains.kt @@ -0,0 +1,37 @@ +package io.novafoundation.nova.runtime.multiNetwork.chain + +import android.content.Context +import android.util.Log +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import io.novafoundation.nova.common.utils.LOG_TAG +import io.novafoundation.nova.runtime.multiNetwork.chain.remote.model.ChainRemote + +/** The chain list the APK was built with: what a first install starts from when no chain list can be fetched. */ +fun interface BundledChains { + + fun load(): List? + + companion object { + val None = BundledChains { null } + } +} + +class AssetBundledChains( + private val context: Context, + private val gson: Gson, +) : BundledChains { + + companion object { + // Written at build time by the `bundleChainsConfig` task in runtime/build.gradle. + const val ASSET_PATH = "bundled-config/chains.json" + } + + override fun load(): List? = runCatching { + context.assets.open(ASSET_PATH).bufferedReader().use { reader -> + gson.fromJson>(reader, object : TypeToken>() {}.type) + } + }.onFailure { + Log.e(LOG_TAG, "No usable bundled chain list", it) + }.getOrNull() +} diff --git a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncService.kt b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncService.kt index 5e571e0b..bab41c46 100644 --- a/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncService.kt +++ b/runtime/src/main/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncService.kt @@ -12,6 +12,7 @@ import io.novafoundation.nova.core_db.model.chain.AssetSourceLocal import io.novafoundation.nova.core_db.model.chain.ChainAssetLocal.Companion.ENABLED_DEFAULT_BOOL import io.novafoundation.nova.core_db.model.chain.ChainLocal import io.novafoundation.nova.core_db.model.chain.ChainNodeLocal +import io.novafoundation.nova.core_db.model.chain.JoinedChainInfo import io.novafoundation.nova.core_db.model.chain.NodeSelectionPreferencesLocal import io.novafoundation.nova.runtime.multiNetwork.chain.mappers.mapExternalApisToLocal import io.novafoundation.nova.runtime.multiNetwork.chain.mappers.mapRemoteAssetToLocal @@ -19,17 +20,48 @@ import io.novafoundation.nova.runtime.multiNetwork.chain.mappers.mapRemoteChainT import io.novafoundation.nova.runtime.multiNetwork.chain.mappers.mapRemoteExplorersToLocal import io.novafoundation.nova.runtime.multiNetwork.chain.mappers.mapRemoteNodesToLocal import io.novafoundation.nova.runtime.multiNetwork.chain.remote.ChainFetcher +import io.novafoundation.nova.runtime.multiNetwork.chain.remote.model.ChainRemote +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext class ChainSyncService( private val chainDao: ChainDao, private val chainFetcher: ChainFetcher, - private val gson: Gson + private val gson: Gson, + private val bundledChains: BundledChains = BundledChains.None, ) { suspend fun syncUp() = withContext(Dispatchers.Default) { - val localChainsJoinedInfo = chainDao.getJoinChainInfo().filter { it.chain.source != ChainLocal.Source.CUSTOM } + if (loadLocal().isEmpty()) { + // A first install has nothing to show until a chain list arrives. Try the network once (the + // config mirrors are behind it), and if that fails, start from the list the APK was built with + // so the wallet opens; the remote list replaces it below as soon as it can be fetched. + val firstTry = try { + chainFetcher.getChains() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Log.e(LOG_TAG, "First chain fetch failed, starting from the bundled chain list", e) + null + } + + if (firstTry != null) { + applyRemoteChains(firstTry, loadLocal()) + return@withContext + } + + bundledChains.load()?.let { applyRemoteChains(it, loadLocal()) } + } + + val remoteChains = retryUntilDone { chainFetcher.getChains() } + + applyRemoteChains(remoteChains, loadLocal()) + } + + private suspend fun loadLocal() = chainDao.getJoinChainInfo().filter { it.chain.source != ChainLocal.Source.CUSTOM } + + private suspend fun applyRemoteChains(remoteChains: List, localChainsJoinedInfo: List) { val oldChains = localChainsJoinedInfo.map { it.chain } val oldAssets = localChainsJoinedInfo.flatMap { it.assets }.filter { it.source == AssetSourceLocal.DEFAULT } val oldNodes = localChainsJoinedInfo.flatMap { it.nodes }.filter { it.source != ChainNodeLocal.Source.CUSTOM } @@ -40,8 +72,6 @@ class ChainSyncService( val oldChainsById = oldChains.associateBy { it.id } val associatedOldAssets = oldAssets.associateBy { it.fullId() } - val remoteChains = retryUntilDone { chainFetcher.getChains() } - // A transient upstream issue (CDN hiccup, regional network filtering, a bad publish) can make // chainFetcher.getChains() return successfully with a suspiciously small/empty list instead of // throwing. Applying that as a diff against a populated local DB would delete most or all of the @@ -54,7 +84,7 @@ class ChainSyncService( "Refusing to apply chain sync: remote returned ${remoteChains.size} chains vs ${oldChains.size} currently stored " + "(would remove more than half). Likely a transient fetch issue - skipping this sync cycle." ) - return@withContext + return } // One malformed/incompatible chain (a new field the app's mapper doesn't understand yet, a bad diff --git a/runtime/src/test/java/android/util/Log.java b/runtime/src/test/java/android/util/Log.java index dd394e3f..12627f4d 100644 --- a/runtime/src/test/java/android/util/Log.java +++ b/runtime/src/test/java/android/util/Log.java @@ -22,5 +22,10 @@ public class Log { return 0; } + public static int e(String tag, String msg, Throwable tr) { + System.out.println("ERROR: " + tag + ": " + msg + ": " + tr); + return 0; + } + // add other methods if required... } \ No newline at end of file diff --git a/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncServiceTest.kt b/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncServiceTest.kt index dc880dd6..bd4451cc 100644 --- a/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncServiceTest.kt +++ b/runtime/src/test/java/io/novafoundation/nova/runtime/multiNetwork/chain/ChainSyncServiceTest.kt @@ -25,6 +25,7 @@ import io.novafoundation.nova.test_shared.emptyDiff import io.novafoundation.nova.test_shared.insertsElement import io.novafoundation.nova.test_shared.removesElement import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals import org.junit.Before import org.junit.Test import org.junit.runner.RunWith @@ -131,6 +132,50 @@ class ChainSyncServiceTest { } } + @Test + fun `a first install starts from the bundled list when the fetch fails`() { + runBlocking { + val bundled = REMOTE_CHAIN.copy(chainId = "0xbb") + chainSyncService = ChainSyncService(dao, chainFetcher, gson) { listOf(bundled) } + localReturns(emptyList()) + `when`(chainFetcher.getChains()).thenThrow(RuntimeException("unreachable")).thenReturn(listOf(REMOTE_CHAIN)) + + chainSyncService.syncUp() + + verify(dao).applyDiff( + chainDiff = insertsChainWithId(bundled.chainId), + assetsDiff = insertsAssetWithId(assetId), + nodesDiff = insertsNodeWithUrl(nodeUrl), + explorersDiff = insertsExplorerByName(explorerName), + externalApisDiff = insertsTransferApiByUrl(transferApiUrl), + nodeSelectionPreferencesDiff = insertsNodeSelectionPreferences(bundled.chainId) + ) + // ... and the fetched list still follows it. + verify(dao).applyDiff( + chainDiff = insertsChainWithId(REMOTE_CHAIN.chainId), + assetsDiff = insertsAssetWithId(assetId), + nodesDiff = insertsNodeWithUrl(nodeUrl), + explorersDiff = insertsExplorerByName(explorerName), + externalApisDiff = insertsTransferApiByUrl(transferApiUrl), + nodeSelectionPreferencesDiff = insertsNodeSelectionPreferences(REMOTE_CHAIN.chainId) + ) + } + } + + @Test + fun `the bundled list is not read when the fetch works`() { + runBlocking { + var bundledReads = 0 + chainSyncService = ChainSyncService(dao, chainFetcher, gson) { bundledReads++; emptyList() } + localReturns(emptyList()) + remoteReturns(listOf(REMOTE_CHAIN)) + + chainSyncService.syncUp() + + assertEquals(0, bundledReads) + } + } + @Test fun `should not insert the same chain`() { runBlocking { diff --git a/scripts/bundled-config.gradle b/scripts/bundled-config.gradle new file mode 100644 index 00000000..0a158c2f --- /dev/null +++ b/scripts/bundled-config.gradle @@ -0,0 +1,44 @@ +// Config files the APK is built with, so a first install can open when none can be fetched +// (GitHub blocked, offline). Each is the live file at build time, fetched from GitHub or its jsDelivr +// mirror, checked by the caller's `isUsable`, and placed in the module's assets under `assetPath`. +// +// apply from: '../scripts/bundled-config.gradle' +// bundleConfigAsset("bundleChainsConfig", chainsUrl, "bundled-config/chains.json") { it instanceof List && !it.isEmpty() } + +ext.bundleConfigAsset = { String taskName, String url, String assetPath, Closure isUsable -> + def outputDir = "$buildDir/generated/bundledConfig" + def target = file("$outputDir/$assetPath") + + def task = tasks.register(taskName) { + outputs.file(target) + outputs.upToDateWhen { false } + + doLast { + def sources = [ + url, + url.replaceFirst('^https://raw\\.githubusercontent\\.com/([^/]+)/([^/]+)/([^/]+)/', 'https://cdn.jsdelivr.net/gh/$1/$2@$3/') + ] + def bytes = null + for (source in sources) { + try { + bytes = new URL(source).getBytes(connectTimeout: 20_000, readTimeout: 60_000) + break + } catch (IOException e) { + logger.warn("$taskName: $source failed: $e") + } + } + if (bytes == null) throw new GradleException("$taskName: no source answered for $url") + + def parsed = new groovy.json.JsonSlurper().parse(bytes) + if (!isUsable(parsed)) throw new GradleException("$taskName: $url is not a usable config file") + + target.parentFile.mkdirs() + target.bytes = bytes + def sha256 = java.security.MessageDigest.getInstance("SHA-256").digest(bytes).encodeHex().toString() + logger.lifecycle("$taskName: $assetPath ${bytes.length} bytes, sha256 $sha256") + } + } + + android.sourceSets.main.assets.srcDir(outputDir) + preBuild.dependsOn(task) +}