From a8105d2610a825e9f62c0914c2eedc464c1a02a3 Mon Sep 17 00:00:00 2001 From: Jaco Date: Wed, 26 Jan 2022 08:07:49 +0100 Subject: [PATCH] Adjust/DRY-up JS retrieval process (#1006) * Adjust/DRY-up JS retrieval process * Adjust fetch in tests * Remove unneeded async specifiers --- packages/phishing/src/addrcheck.spec.ts | 13 ++--- packages/phishing/src/bundle.ts | 70 +++++++++++++------------ packages/phishing/src/fetch.ts | 10 +++- 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/packages/phishing/src/addrcheck.spec.ts b/packages/phishing/src/addrcheck.spec.ts index 875186098..c7d071907 100644 --- a/packages/phishing/src/addrcheck.spec.ts +++ b/packages/phishing/src/addrcheck.spec.ts @@ -5,16 +5,13 @@ import fs from 'fs'; import { decodeAddress } from '@polkadot/util-crypto'; -import { fetchWithTimeout } from './fetch'; +import { fetchJson, fetchText } from './fetch'; const TICKS = '```'; +const TIMEOUT = 5000; const ourAddrList = JSON.parse(fs.readFileSync('address.json', 'utf-8')) as Record; -function fetch (url: string): Promise { - return fetchWithTimeout(url, 5000); -} - // loop through each site for a number of times, applying the transform async function loopSome (site: string, matcher: () => Promise): Promise<[string, string[]]> { const found: string[] = []; @@ -46,7 +43,7 @@ async function loopSome (site: string, matcher: () => Promise): // shared between polkadot.center & polkadot-event.com (addresses are also the same on first run) function checkGetWallet (site: string): Promise<[string, string[]]> { return loopSome(site, async (): Promise => { - const result = await (await fetch(`https://${site}/get_wallet.php`)).json() as Record; + const result = await fetchJson>(`https://${site}/get_wallet.php`, TIMEOUT); return (result && result.wallet) ? [result.wallet.replace('\r', '').trim()] @@ -59,7 +56,7 @@ function checkTag (url: string, tag: string, attr?: string): Promise<[string, st const site = url.split('/')[2]; return loopSome(site, async (): Promise => { - const result = await (await fetch(url)).text(); + const result = await fetchText(url, TIMEOUT); // /

(.*?)<\/p>/g const match = new RegExp(`<${tag}${attr ? ` ${attr}` : ''}>(.*?)`, 'g').exec(result); @@ -82,7 +79,7 @@ function checkAttr (url: string, attr: string): Promise<[string, string[]]> { const site = url.split('/')[2]; return loopSome(site, async (): Promise => { - const result = await (await fetch(url)).text(); + const result = await fetchText(url, TIMEOUT); const match = new RegExp(`${attr}"[a-zA-Z0-9]+"`, 'g').exec(result); return match && match.length diff --git a/packages/phishing/src/bundle.ts b/packages/phishing/src/bundle.ts index 18d2cca0b..bb7b02f2a 100644 --- a/packages/phishing/src/bundle.ts +++ b/packages/phishing/src/bundle.ts @@ -6,7 +6,7 @@ import type { AddressList, HostList } from './types'; import { u8aEq } from '@polkadot/util'; import { decodeAddress } from '@polkadot/util-crypto'; -import { fetchWithTimeout } from './fetch'; +import { fetchJson } from './fetch'; export { packageInfo } from './packageInfo'; @@ -29,39 +29,44 @@ function extractHost (path: string): string { .split('/')[0]; } +// logs an error in a consistent format +function log (error: unknown, check: string): void { + console.warn(`Error checking ${check}, assuming non-phishing`, (error as Error).message); +} + /** * Retrieve a list of known phishing addresses */ export async function retrieveAddrList (allowCached = true): Promise { const now = Date.now(); - if (allowCached && cacheAddrList && (now < cacheAddrEnd)) { - return cacheAddrList; - } + return (allowCached && cacheAddrList && (now < cacheAddrEnd)) + ? cacheAddrList + : fetchJson(ADDRESS_JSON).then((list) => { + cacheAddrEnd = now + CACHE_TIMEOUT; + cacheAddrList = list; - const response = await fetchWithTimeout(ADDRESS_JSON); - const list = (await response.json()) as AddressList; - - cacheAddrEnd = now + CACHE_TIMEOUT; - cacheAddrList = list; - - return list; + return list; + }); } +/** + * Retrieve a list of known phishing addresses in raw Uint8Array format + */ async function retrieveAddrU8a (allowCached = true): Promise<[string, Uint8Array[]][]> { const now = Date.now(); - if (allowCached && cacheAddrU8a && (now < cacheAddrEnd)) { - return cacheAddrU8a; - } + return (allowCached && cacheAddrU8a && (now < cacheAddrEnd)) + ? cacheAddrU8a + : retrieveAddrList(allowCached).then((all) => { + cacheAddrU8a = Object + .entries(all) + .map(([key, addresses]): [string, Uint8Array[]] => + [key, addresses.map((a) => decodeAddress(a))] + ); - const all = await retrieveAddrList(allowCached); - - cacheAddrU8a = Object - .entries(all) - .map(([key, addresses]): [string, Uint8Array[]] => [key, addresses.map((a) => decodeAddress(a))]); - - return cacheAddrU8a; + return cacheAddrU8a; + }); } /** @@ -70,17 +75,14 @@ async function retrieveAddrU8a (allowCached = true): Promise<[string, Uint8Array export async function retrieveHostList (allowCached = true): Promise { const now = Date.now(); - if (allowCached && cacheHostList && (now < cacheHostEnd)) { - return cacheHostList; - } + return (allowCached && cacheHostList && (now < cacheHostEnd)) + ? cacheHostList + : fetchJson(ALL_JSON).then((list) => { + cacheHostEnd = now + CACHE_TIMEOUT; + cacheHostList = list; - const response = await fetchWithTimeout(ALL_JSON); - const list = (await response.json()) as HostList; - - cacheHostEnd = now + CACHE_TIMEOUT; - cacheHostList = list; - - return list; + return list; + }); } /** @@ -108,13 +110,13 @@ export function checkHost (items: string[], host: string): boolean { */ export async function checkAddress (address: string | Uint8Array, allowCached = true): Promise { try { - const all = await retrieveAddrU8a(allowCached); const u8a = decodeAddress(address); + const all = await retrieveAddrU8a(allowCached); const entry = all.find(([, all]) => all.some((a) => u8aEq(a, u8a))) || [null]; return entry[0]; } catch (error) { - console.error('Exception while checking address, assuming non-phishing', (error as Error).message); + log(error, 'address'); return null; } @@ -130,7 +132,7 @@ export async function checkIfDenied (host: string, allowCached = true): Promise< return checkHost(deny, host); } catch (error) { - console.error(`Exception while checking ${host}, assuming non-phishing`, (error as Error).message); + log(error, host); return false; } diff --git a/packages/phishing/src/fetch.ts b/packages/phishing/src/fetch.ts index f3cb025ad..b91912c9c 100644 --- a/packages/phishing/src/fetch.ts +++ b/packages/phishing/src/fetch.ts @@ -4,7 +4,7 @@ import { fetch } from '@polkadot/x-fetch'; // a fetch with a 2s timeout -export async function fetchWithTimeout (url: string, timeout = 2000): Promise { +async function fetchWithTimeout (url: string, timeout = 2000): Promise { const controller = new AbortController(); let isAborted = false; const id = setTimeout((): void => { @@ -28,3 +28,11 @@ export async function fetchWithTimeout (url: string, timeout = 2000): Promise (url: string, timeout = 2000): Promise { + return fetchWithTimeout(url, timeout).then((r) => r.json()); +} + +export function fetchText (url: string, timeout = 2000): Promise { + return fetchWithTimeout(url, timeout).then((r) => r.text()); +}