From 0b1319c86631fb09a45f290ff9668a81fbb0727c Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Sat, 31 Oct 2026 00:36:48 +0400 Subject: [PATCH] feat(network): fetch config from jsDelivr when raw.githubusercontent.com fails Every config file the app reads -- chains, assets, DApps, XCM, banners, token icons -- comes from raw.githubusercontent.com with no other source, so when that host cannot be reached a fresh install has no chains at all. ConfigMirrorInterceptor retries the same file on cdn.jsdelivr.net and then fastly.jsdelivr.net when the primary fails to connect or answers 403, 429, 451 or 5xx. A 404 is an answer and is returned as is. A host that failed is skipped for ten minutes so later requests do not each wait for its timeout. Mirror requests revalidate, so a mirror's long max-age cannot pin a copy in the HTTP cache after the primary is back. The shared OkHttp client and Coil's image client both carry it. --- .../network/mirror/ConfigMirrorInterceptor.kt | 134 ++++++++++++++++++ .../nova/common/di/modules/CommonModule.kt | 12 +- .../nova/common/di/modules/NetworkModule.kt | 9 +- .../mirror/ConfigMirrorInterceptorTest.kt | 124 ++++++++++++++++ 4 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 common/src/main/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptor.kt create mode 100644 common/src/test/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptorTest.kt diff --git a/common/src/main/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptor.kt b/common/src/main/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptor.kt new file mode 100644 index 00000000..e1080539 --- /dev/null +++ b/common/src/main/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptor.kt @@ -0,0 +1,134 @@ +package io.novafoundation.nova.common.data.network.mirror + +import okhttp3.CacheControl +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +/** + * Every config file the app reads (chains, assets, DApps, XCM, banners, icons) lives on + * raw.githubusercontent.com. When that host cannot be reached -- blocked, down, rate limited -- a fresh + * install has nothing to show. This interceptor retries the same file on mirrors that serve the same + * repository content, in order, and remembers for [hostDownMillis] that the primary failed so the next + * requests do not each wait for its timeout first. + * + * A 404 is an answer, not an outage: it is returned as is and no mirror is asked. + */ +class ConfigMirrorInterceptor( + private val mirrors: List = DEFAULT_MIRRORS, + private val hostDownMillis: Long = TimeUnit.MINUTES.toMillis(10), + private val clock: () -> Long = System::currentTimeMillis, +) : Interceptor { + + companion object { + const val GITHUB_RAW_HOST = "raw.githubusercontent.com" + + val DEFAULT_MIRRORS = listOf( + GithubMirror.JsDelivr("cdn.jsdelivr.net"), + GithubMirror.JsDelivr("fastly.jsdelivr.net"), + ) + + private val OUTAGE_CODES = setOf(403, 429, 451) + } + + private val downUntil = ConcurrentHashMap() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val file = GithubFile.parse(request.url) + + if (file == null || request.method != "GET") return chain.proceed(request) + + val candidates = listOf(request) + mirrors.map { mirror -> + request.newBuilder() + .url(mirror.urlOf(file).newBuilder().encodedQuery(request.url.encodedQuery).build()) + // A mirror's max-age would pin its copy in the HTTP cache long after the primary is back. + .cacheControl(CacheControl.Builder().noCache().build()) + .build() + } + + val available = candidates.filterNot { isDown(it.url.host) }.ifEmpty { candidates } + + var lastFailure: IOException? = null + var lastOutage: Response? = null + + for ((index, candidate) in available.withIndex()) { + val isLast = index == available.lastIndex + + val response = try { + chain.proceed(candidate) + } catch (e: IOException) { + markDown(candidate.url.host) + lastFailure = e + continue + } + + if (!isOutage(response) || isLast) { + if (!isOutage(response)) downUntil.remove(candidate.url.host) + lastOutage?.close() + return response + } + + markDown(candidate.url.host) + lastOutage?.close() + lastOutage = response + } + + lastOutage?.let { return it } + throw lastFailure ?: IOException("No source answered for ${request.url}") + } + + private fun isOutage(response: Response) = response.code in OUTAGE_CODES || response.code >= 500 + + private fun isDown(host: String): Boolean { + val until = downUntil[host] ?: return false + if (clock() < until) return true + downUntil.remove(host, until) + return false + } + + private fun markDown(host: String) { + downUntil[host] = clock() + hostDownMillis + } +} + +/** A file in a GitHub repository at a ref, as raw.githubusercontent.com addresses it. */ +data class GithubFile(val owner: String, val repo: String, val ref: String, val path: String) { + + companion object { + + fun parse(url: HttpUrl): GithubFile? { + if (url.host != ConfigMirrorInterceptor.GITHUB_RAW_HOST) return null + + val segments = url.pathSegments + if (segments.size < 4) return null + + val (owner, repo) = segments + // raw accepts both `master` and `refs/heads/master`; mirrors only know the short form. + val refLength = if (segments[2] == "refs" && segments.size >= 6) 3 else 1 + val ref = segments.subList(2, 2 + refLength).last() + val path = segments.drop(2 + refLength).joinToString("/") + + if (path.isEmpty()) return null + + return GithubFile(owner, repo, ref, path) + } + } +} + +sealed interface GithubMirror { + + fun urlOf(file: GithubFile): HttpUrl + + /** jsDelivr serves any public GitHub repository at `/gh/{owner}/{repo}@{ref}/{path}`. */ + class JsDelivr(private val host: String) : GithubMirror { + + override fun urlOf(file: GithubFile): HttpUrl { + return "https://$host/gh/${file.owner}/${file.repo}@${file.ref}/${file.path}".toHttpUrl() + } + } +} diff --git a/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt b/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt index 796543ea..ba192d63 100644 --- a/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt +++ b/common/src/main/java/io/novafoundation/nova/common/di/modules/CommonModule.kt @@ -6,6 +6,7 @@ import android.content.SharedPreferences import android.graphics.Color import coil.ImageLoader import coil.decode.SvgDecoder +import coil.util.CoilUtils import com.google.android.play.core.integrity.IntegrityManagerFactory import dagger.Module import dagger.Provides @@ -24,6 +25,7 @@ import io.novafoundation.nova.common.data.memory.ComputationalCache import io.novafoundation.nova.common.data.memory.RealComputationalCache import io.novafoundation.nova.common.data.network.NetworkApiCreator import io.novafoundation.nova.common.data.network.coingecko.CoinGeckoLinkParser +import io.novafoundation.nova.common.data.network.mirror.ConfigMirrorInterceptor import io.novafoundation.nova.common.data.providers.deviceid.AndroidDeviceIdProvider import io.novafoundation.nova.common.data.providers.deviceid.DeviceIdProvider import io.novafoundation.nova.common.data.repository.AssetsIconModeRepository @@ -116,6 +118,7 @@ import io.novafoundation.nova.common.view.input.selector.ListSelectorMixin import io.novafoundation.nova.common.view.input.selector.RealListSelectorMixinFactory import io.novasama.substrate_sdk_android.encrypt.Signer import io.novasama.substrate_sdk_android.icon.IconGenerator +import okhttp3.OkHttpClient import java.security.SecureRandom import java.util.Random import javax.inject.Qualifier @@ -143,7 +146,14 @@ class CommonModule { @Provides @ApplicationScope - fun imageLoader(context: Context) = ImageLoader.Builder(context) + fun imageLoader(context: Context, configMirrorInterceptor: ConfigMirrorInterceptor) = ImageLoader.Builder(context) + // Token icons live on raw.githubusercontent.com too; Coil keeps its own disk cache. + .okHttpClient { + OkHttpClient.Builder() + .cache(CoilUtils.createDefaultCache(context)) + .addInterceptor(configMirrorInterceptor) + .build() + } .componentRegistry { add(SvgDecoder(context)) } diff --git a/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt b/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt index 4896f028..9f0e6f8f 100644 --- a/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt +++ b/common/src/main/java/io/novafoundation/nova/common/di/modules/NetworkModule.kt @@ -11,6 +11,7 @@ import io.novafoundation.nova.common.data.network.AppLinksProvider import io.novafoundation.nova.common.data.network.HttpExceptionHandler import io.novafoundation.nova.common.data.network.NetworkApiCreator import io.novafoundation.nova.common.data.network.TimeHeaderInterceptor +import io.novafoundation.nova.common.data.network.mirror.ConfigMirrorInterceptor import io.novafoundation.nova.common.data.network.rpc.SocketSingleRequestExecutor import io.novafoundation.nova.common.di.scope.ApplicationScope import io.novafoundation.nova.common.mixin.api.NetworkStateMixin @@ -74,10 +75,15 @@ class NetworkModule { ) } + @Provides + @ApplicationScope + fun provideConfigMirrorInterceptor() = ConfigMirrorInterceptor() + @Provides @ApplicationScope fun provideOkHttpClient( - context: Context + context: Context, + configMirrorInterceptor: ConfigMirrorInterceptor ): OkHttpClient { val builder = OkHttpClient.Builder() .connectTimeout(TIMEOUT_SECONDS, TimeUnit.SECONDS) @@ -86,6 +92,7 @@ class NetworkModule { .cache(Cache(File(context.cacheDir, HTTP_CACHE), CACHE_SIZE)) .retryOnConnectionFailure(true) .addInterceptor(TimeHeaderInterceptor()) + .addInterceptor(configMirrorInterceptor) if (BuildConfig.DEBUG) { builder.addInterceptor(HttpLoggingInterceptor().setLevel(HttpLoggingInterceptor.Level.BODY)) diff --git a/common/src/test/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptorTest.kt b/common/src/test/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptorTest.kt new file mode 100644 index 00000000..030bccd6 --- /dev/null +++ b/common/src/test/java/io/novafoundation/nova/common/data/network/mirror/ConfigMirrorInterceptorTest.kt @@ -0,0 +1,124 @@ +package io.novafoundation.nova.common.data.network.mirror + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test +import java.io.IOException + +private const val RAW = "https://raw.githubusercontent.com/pezkuwichain/pezkuwi-wallet-utils/master/chains/v22/android/chains.json" +private const val CDN = "https://cdn.jsdelivr.net/gh/pezkuwichain/pezkuwi-wallet-utils@master/chains/v22/android/chains.json" +private const val FASTLY = "https://fastly.jsdelivr.net/gh/pezkuwichain/pezkuwi-wallet-utils@master/chains/v22/android/chains.json" + +class ConfigMirrorInterceptorTest { + + private var now = 0L + private val hits = mutableListOf() + + /** What each host does, standing in for the network: a status code, or null for a connection failure. */ + private val hosts = mutableMapOf() + + private val client = OkHttpClient.Builder() + .addInterceptor(ConfigMirrorInterceptor(hostDownMillis = 1_000, clock = { now })) + .addInterceptor(Interceptor { chain -> fakeNetwork(chain.request()) }) + .build() + + @Test + fun `a blocked primary is served from the first mirror`() { + hosts["raw.githubusercontent.com"] = null + hosts["cdn.jsdelivr.net"] = 200 + + assertEquals("cdn.jsdelivr.net", fetch(RAW)) + assertEquals(listOf(RAW, CDN), hits) + } + + @Test + fun `mirrors are tried in order until one answers`() { + hosts["raw.githubusercontent.com"] = 451 + hosts["cdn.jsdelivr.net"] = null + hosts["fastly.jsdelivr.net"] = 200 + + assertEquals("fastly.jsdelivr.net", fetch(RAW)) + assertEquals(listOf(RAW, CDN, FASTLY), hits) + } + + @Test + fun `a server error on the primary goes to a mirror`() { + hosts["raw.githubusercontent.com"] = 503 + hosts["cdn.jsdelivr.net"] = 200 + + assertEquals("cdn.jsdelivr.net", fetch(RAW)) + } + + @Test + fun `a missing file is an answer and no mirror is asked`() { + hosts["raw.githubusercontent.com"] = 404 + + assertNull(fetch(RAW)) + assertEquals(listOf(RAW), hits) + } + + @Test + fun `a failed primary is skipped until its window passes`() { + hosts["raw.githubusercontent.com"] = null + hosts["cdn.jsdelivr.net"] = 200 + fetch(RAW) + hits.clear() + + fetch(RAW) + assertEquals("the primary is not waited on again inside the window", listOf(CDN), hits) + + now += 1_001 + hits.clear() + hosts["raw.githubusercontent.com"] = 200 + assertEquals("raw.githubusercontent.com", fetch(RAW)) + assertEquals(listOf(RAW), hits) + } + + @Test(expected = IOException::class) + fun `when every source fails the failure reaches the caller`() { + fetch(RAW) + } + + @Test + fun `other hosts are left alone`() { + hosts["example.org"] = null + + runCatching { fetch("https://example.org/a.json") } + assertEquals(listOf("https://example.org/a.json"), hits) + } + + @Test + fun `the long ref form maps to the mirror's short form`() { + val file = GithubFile.parse("https://raw.githubusercontent.com/o/r/refs/heads/master/a/b.json".toHttpUrl()) + + assertEquals(GithubFile("o", "r", "master", "a/b.json"), file) + } + + private fun fetch(url: String): String? { + client.newCall(Request.Builder().url(url).build()).execute().use { response -> + return if (response.isSuccessful) response.body!!.string() else null + } + } + + private fun fakeNetwork(request: Request): Response { + hits += request.url.toString() + + val code = hosts[request.url.host] ?: throw IOException("unreachable: ${request.url.host}") + + return Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message("") + .body(request.url.host.toResponseBody("text/plain".toMediaType())) + .build() + } +}