From b5f5af2cce0bd919e1d3ad769443adfb82caec2a Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Sat, 17 Oct 2026 19:58:49 +0400 Subject: [PATCH] fix(chains): strip every keyed provider URL, and make the gate fail closed The first pass matched only `infura.io/.../v3/`, so it missed `https://...infura.io/v3/`, upper-case keys and hosts, and Nova's OnFinality `?apikey=` URLs (Moonriver, and the Hades testnet in v20/v21 preConfigured). preConfigured was copied from Nova without the filter, and the integrity check passed if grep itself failed. - one predicate, is_keyed(), in sync_from_nova.py: Infura /v3/ ids and apikey/api_key/key/token query values, case-insensitive; {PLACEHOLDER} values are allowed - preConfigured details are filtered after the copy; a chain left with no node loses its details file and its index entry (Hades, v20/v21 only) - rewritten files keep their own indent, so the diff is the data change only - check 5 imports is_keyed() and checks every string in every JSON file under chains/; an unreadable file or an empty tree is an error - check 3 reports a grep error instead of passing on it --- chains/v10/chains_dev.json | 4 - chains/v11/chains_dev.json | 4 - chains/v12/chains_dev.json | 4 - chains/v13/chains_dev.json | 4 - chains/v20/preConfigured/chains.json | 4 - chains/v20/preConfigured/chains_dev.json | 4 - ...02e331195227da62567d7bf9bbd1abaf77522.json | 19 ----- ...02e331195227da62567d7bf9bbd1abaf77522.json | 19 ----- chains/v21/preConfigured/chains.json | 4 - chains/v21/preConfigured/chains_dev.json | 4 - ...02e331195227da62567d7bf9bbd1abaf77522.json | 19 ----- ...02e331195227da62567d7bf9bbd1abaf77522.json | 19 ----- chains/v4/chains_dev.json | 4 - chains/v5/chains_dev.json | 4 - chains/v6/chains_dev.json | 4 - chains/v7/chains_dev.json | 4 - chains/v8/chains_dev.json | 4 - chains/v9/chains_dev.json | 4 - scripts/check_pezkuwi_integrity.py | 48 ++++++++---- scripts/sync_from_nova.py | 73 +++++++++++++++---- 20 files changed, 92 insertions(+), 161 deletions(-) delete mode 100644 chains/v20/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json delete mode 100644 chains/v20/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json delete mode 100644 chains/v21/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json delete mode 100644 chains/v21/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json diff --git a/chains/v10/chains_dev.json b/chains/v10/chains_dev.json index f01f4be..f6cec4b 100644 --- a/chains/v10/chains_dev.json +++ b/chains/v10/chains_dev.json @@ -1355,10 +1355,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v11/chains_dev.json b/chains/v11/chains_dev.json index 8f6c4e0..df4b6a2 100644 --- a/chains/v11/chains_dev.json +++ b/chains/v11/chains_dev.json @@ -1369,10 +1369,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v12/chains_dev.json b/chains/v12/chains_dev.json index 16c2ef0..7a61c19 100644 --- a/chains/v12/chains_dev.json +++ b/chains/v12/chains_dev.json @@ -1380,10 +1380,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v13/chains_dev.json b/chains/v13/chains_dev.json index 7943767..b4aea2b 100644 --- a/chains/v13/chains_dev.json +++ b/chains/v13/chains_dev.json @@ -1441,10 +1441,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v20/preConfigured/chains.json b/chains/v20/preConfigured/chains.json index d6acaa2..fbf873a 100644 --- a/chains/v20/preConfigured/chains.json +++ b/chains/v20/preConfigured/chains.json @@ -183,10 +183,6 @@ "chainId": "65c489fb7bfaf4a27f283faf694a64a8d7ae8fc0ed970fc4ebbf25d03095e4d0", "name": "Niskala (TESTNET)" }, - { - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)" - }, { "chainId": "466edf864b4314b97f36e45ec21ddb39e0bdc52789377b91be0957d5afad2eb2", "name": "Energy Web X Paseo (TESTNET)" diff --git a/chains/v20/preConfigured/chains_dev.json b/chains/v20/preConfigured/chains_dev.json index a38ccbb..7e29609 100644 --- a/chains/v20/preConfigured/chains_dev.json +++ b/chains/v20/preConfigured/chains_dev.json @@ -139,10 +139,6 @@ "chainId": "65c489fb7bfaf4a27f283faf694a64a8d7ae8fc0ed970fc4ebbf25d03095e4d0", "name": "Niskala (TESTNET)" }, - { - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)" - }, { "chainId": "466edf864b4314b97f36e45ec21ddb39e0bdc52789377b91be0957d5afad2eb2", "name": "Energy Web X Paseo (TESTNET)" diff --git a/chains/v20/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json b/chains/v20/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json deleted file mode 100644 index 3a0d692..0000000 --- a/chains/v20/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)", - "assets": [ - { - "assetId": 0, - "symbol": "notNodl", - "precision": 11, - "icon": "https://raw.githubusercontent.com/novasamatech/nova-utils/master/icons/tokens/white/Default.svg" - } - ], - "nodes": [ - { - "url": "wss://node-6957502816543653888.lh.onfinality.io/ws?apikey=09b04494-3139-4b57-a5d1-e1c4c18748ce", - "name": "OnFinality" - } - ], - "addressPrefix": 37 -} diff --git a/chains/v20/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json b/chains/v20/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json deleted file mode 100644 index 3a0d692..0000000 --- a/chains/v20/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)", - "assets": [ - { - "assetId": 0, - "symbol": "notNodl", - "precision": 11, - "icon": "https://raw.githubusercontent.com/novasamatech/nova-utils/master/icons/tokens/white/Default.svg" - } - ], - "nodes": [ - { - "url": "wss://node-6957502816543653888.lh.onfinality.io/ws?apikey=09b04494-3139-4b57-a5d1-e1c4c18748ce", - "name": "OnFinality" - } - ], - "addressPrefix": 37 -} diff --git a/chains/v21/preConfigured/chains.json b/chains/v21/preConfigured/chains.json index e4488a2..1b184dd 100644 --- a/chains/v21/preConfigured/chains.json +++ b/chains/v21/preConfigured/chains.json @@ -187,10 +187,6 @@ "chainId": "65c489fb7bfaf4a27f283faf694a64a8d7ae8fc0ed970fc4ebbf25d03095e4d0", "name": "Niskala (TESTNET)" }, - { - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)" - }, { "chainId": "466edf864b4314b97f36e45ec21ddb39e0bdc52789377b91be0957d5afad2eb2", "name": "Energy Web X Paseo (TESTNET)" diff --git a/chains/v21/preConfigured/chains_dev.json b/chains/v21/preConfigured/chains_dev.json index 34dc485..bcaa8e9 100644 --- a/chains/v21/preConfigured/chains_dev.json +++ b/chains/v21/preConfigured/chains_dev.json @@ -135,10 +135,6 @@ "chainId": "65c489fb7bfaf4a27f283faf694a64a8d7ae8fc0ed970fc4ebbf25d03095e4d0", "name": "Niskala (TESTNET)" }, - { - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)" - }, { "chainId": "466edf864b4314b97f36e45ec21ddb39e0bdc52789377b91be0957d5afad2eb2", "name": "Energy Web X Paseo (TESTNET)" diff --git a/chains/v21/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json b/chains/v21/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json deleted file mode 100644 index 8bd1492..0000000 --- a/chains/v21/preConfigured/details/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)", - "assets": [ - { - "assetId": 0, - "symbol": "notNodl", - "precision": 11, - "icon": "Default.svg" - } - ], - "nodes": [ - { - "url": "wss://node-7273232234617282560.nv.onfinality.io/ws?apikey=b937a7d7-7395-49b9-b745-60a0342fa365", - "name": "OnFinality" - } - ], - "addressPrefix": 37 -} diff --git a/chains/v21/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json b/chains/v21/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json deleted file mode 100644 index 8bd1492..0000000 --- a/chains/v21/preConfigured/detailsDev/d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chainId": "d9d12d9139d020f658d4469fa5c02e331195227da62567d7bf9bbd1abaf77522", - "name": "Parachain Hades testnet (TESTNET)", - "assets": [ - { - "assetId": 0, - "symbol": "notNodl", - "precision": 11, - "icon": "Default.svg" - } - ], - "nodes": [ - { - "url": "wss://node-7273232234617282560.nv.onfinality.io/ws?apikey=b937a7d7-7395-49b9-b745-60a0342fa365", - "name": "OnFinality" - } - ], - "addressPrefix": 37 -} diff --git a/chains/v4/chains_dev.json b/chains/v4/chains_dev.json index ed07293..ba289bf 100644 --- a/chains/v4/chains_dev.json +++ b/chains/v4/chains_dev.json @@ -1225,10 +1225,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v5/chains_dev.json b/chains/v5/chains_dev.json index ae0983a..bc980ce 100644 --- a/chains/v5/chains_dev.json +++ b/chains/v5/chains_dev.json @@ -1283,10 +1283,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v6/chains_dev.json b/chains/v6/chains_dev.json index 6cbfdea..be821e4 100644 --- a/chains/v6/chains_dev.json +++ b/chains/v6/chains_dev.json @@ -1315,10 +1315,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v7/chains_dev.json b/chains/v7/chains_dev.json index 7c6f8bb..b10b75d 100644 --- a/chains/v7/chains_dev.json +++ b/chains/v7/chains_dev.json @@ -1329,10 +1329,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v8/chains_dev.json b/chains/v8/chains_dev.json index 7e48d54..2eadcfa 100644 --- a/chains/v8/chains_dev.json +++ b/chains/v8/chains_dev.json @@ -1355,10 +1355,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/chains/v9/chains_dev.json b/chains/v9/chains_dev.json index 4acc21b..deb789b 100644 --- a/chains/v9/chains_dev.json +++ b/chains/v9/chains_dev.json @@ -1355,10 +1355,6 @@ } ], "nodes": [ - { - "url": "wss://moonriver.api.onfinality.io/ws?apikey=41935d3d-3d6c-4a90-a0e1-33c065401d05", - "name": "OnFinality node" - }, { "url": "wss://moonriver.api.onfinality.io/public-ws", "name": "OnFinality node" diff --git a/scripts/check_pezkuwi_integrity.py b/scripts/check_pezkuwi_integrity.py index f0aaa35..05d881a 100755 --- a/scripts/check_pezkuwi_integrity.py +++ b/scripts/check_pezkuwi_integrity.py @@ -88,6 +88,8 @@ grep = subprocess.run( capture_output=True, text=True, ) +if grep.returncode not in (0, 1): + errors.append(f"grep for working-branch asset URLs failed: {grep.stderr.strip()}") if grep.stdout.strip(): files = grep.stdout.strip().splitlines() errors.append( @@ -107,21 +109,41 @@ for fixture in ("pezkuwi_assets_for_testBalance.json",): f"tests/{fixture} has drifted from pezkuwi-overlay/tests/{fixture}" ) -# ── 5. No third-party API key in a node URL ────────────────────────────────── +# ── 5. No third-party API key in a URL ─────────────────────────────────────── # -# Nova's configs shipped Nova's own Infura project keys inside node URLs, and -# every sync copied them into ours. Our builds fill {INFURA_API_KEY} from our -# own secret; a keyed URL is someone else's quota and dies when they rotate it. -grep = subprocess.run( - ["grep", "-rlE", r"infura\.io/.*/v3/[0-9a-f]{32}", str(ROOT / "chains")], - capture_output=True, - text=True, -) -if grep.stdout.strip(): - files = grep.stdout.strip().splitlines() +# Nova's configs shipped Nova's own provider keys (Infura project ids, OnFinality +# apikeys) inside node URLs, and every sync copied them into ours. Our builds fill +# {INFURA_API_KEY}-style placeholders from our own secrets; a keyed URL is someone +# else's quota and dies when they rotate it. The predicate is the one the sync +# uses, and every string in every JSON file is checked, so a file that does not +# parse is a failure, not a pass. +sys.path.insert(0, str(ROOT / "scripts")) +from sync_from_nova import is_keyed # noqa: E402 + +keyed_files = [] +json_files = sorted(p for p in (ROOT / "chains").rglob("*.json")) +if not json_files: + errors.append("chains/ has no JSON files; the key check would have checked nothing") +for path in json_files: + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + errors.append(f"{path.relative_to(ROOT)} could not be read for the key check: {exc}") + continue + stack = [doc] + while stack: + node = stack.pop() + if isinstance(node, dict): + stack.extend(node.values()) + elif isinstance(node, list): + stack.extend(node) + elif isinstance(node, str) and is_keyed(node): + keyed_files.append(path) + break +if keyed_files: errors.append( - f"{len(files)} file(s) under chains/ embed an Infura key in a node URL; " - f"use {{INFURA_API_KEY}}, e.g. {Path(files[0]).relative_to(ROOT)}" + f"{len(keyed_files)} file(s) under chains/ embed a provider key in a URL; " + f"use a {{PLACEHOLDER}}, e.g. {keyed_files[0].relative_to(ROOT)}" ) if errors: diff --git a/scripts/sync_from_nova.py b/scripts/sync_from_nova.py index d3299f8..abbcdc9 100644 --- a/scripts/sync_from_nova.py +++ b/scripts/sync_from_nova.py @@ -52,25 +52,65 @@ def load_blacklist() -> set: return set() -# Nova's configs carry Nova's own Infura project keys written into node URLs. -# They are not ours to use: our builds fill `{INFURA_API_KEY}` from our own -# secret, and a keyed URL breaks for every user the day Nova rotates it. -KEYED_ENDPOINT = re.compile(r"infura\.io/.*/v3/[0-9a-f]{32}") +# Nova's configs carry Nova's own provider keys written into node URLs (Infura +# project ids, OnFinality apikeys). They are not ours to use: our builds fill +# placeholders like `{INFURA_API_KEY}` from our own secrets, and a keyed URL +# breaks for every user the day Nova rotates it. check_pezkuwi_integrity.py +# imports this same predicate, so the filter and the gate cannot disagree. +KEYED_ENDPOINT = re.compile( + r"infura\.io/(?:[^?#\s]*/)?v3/[0-9a-z]{32}" + r"|[?&](?:api[_-]?key|apikey|access[_-]?token|token|key)=(?!\{)[^&#\s]+", + re.IGNORECASE, +) + + +def is_keyed(url: str) -> bool: + """True if the URL embeds a literal credential rather than a {PLACEHOLDER}.""" + return bool(KEYED_ENDPOINT.search(url)) + + +def strip_chain(chain: dict) -> dict | None: + """The chain without keyed nodes; None if no node is left.""" + nodes = chain.get('nodes') + if not isinstance(nodes, list): + return chain + kept = [n for n in nodes if not (isinstance(n, dict) and is_keyed(str(n.get('url', ''))))] + if not kept: + return None + return chain if len(kept) == len(nodes) else {**chain, 'nodes': kept} def strip_keyed_endpoints(chains: list) -> list: - """Drop nodes whose URL embeds a third-party API key, and any chain left with no node.""" - out = [] - for chain in chains: - nodes = chain.get('nodes') - if nodes is not None: - kept = [n for n in nodes if not KEYED_ENDPOINT.search(n.get('url', ''))] - if not kept: - continue - if len(kept) != len(nodes): - chain = {**chain, 'nodes': kept} - out.append(chain) - return out + """Drop nodes whose URL embeds a third-party key, and any chain left with no node.""" + return [c for c in (strip_chain(c) for c in chains) if c is not None] + + +def rewrite_in_place(path: Path, data): + """Rewrite a JSON file keeping its own indent and trailing newline, so a + filtered copy of Nova's file differs from it only where the data does.""" + raw = path.read_text(encoding='utf-8') + second = raw.split('\n', 2)[1] if raw.count('\n') >= 1 else '' + indent = (len(second) - len(second.lstrip(' '))) or 2 + out = json.dumps(data, indent=indent, ensure_ascii=False) + path.write_text(out + ('\n' if raw.endswith('\n') else ''), encoding='utf-8') + + +def strip_preconfigured(preconfig: Path): + """Apply strip_chain to every details file; a chain left with no node is + removed together with its entry in the matching index.""" + for details, index in (("details", "chains.json"), ("detailsDev", "chains_dev.json")): + dropped = set() + for f in sorted((preconfig / details).glob("*.json")): + chain = load_json(f) + stripped = strip_chain(chain) + if stripped is None: + dropped.add(chain.get('chainId')) + f.unlink() + elif stripped is not chain: + rewrite_in_place(f, stripped) + index_file = preconfig / index + if dropped and index_file.exists(): + rewrite_in_place(index_file, [c for c in load_json(index_file) if c.get('chainId') not in dropped]) def merge_chains(nova_chains: list, pezkuwi_chains: list, blocked_ids: set) -> list: @@ -165,6 +205,7 @@ def sync_chains(): if output_preconfig.exists(): shutil.rmtree(output_preconfig) shutil.copytree(nova_preconfig, output_preconfig) + strip_preconfigured(output_preconfig) def sync_xcm():