Adjust/DRY-up JS retrieval process (#1006)

* Adjust/DRY-up JS retrieval process

* Adjust fetch in tests

* Remove unneeded async specifiers

packages/phishing/src/addrcheck.spec.ts

@@ -5,16 +5,13 @@ import fs from 'fs';
 
 import { decodeAddress } from '@polkadot/util-crypto';
 
-import { fetchWithTimeout } from './fetch';
+import { fetchJson, fetchText } from './fetch';
 
 const TICKS = '```';
+const TIMEOUT = 5000;
 
 const ourAddrList = JSON.parse(fs.readFileSync('address.json', 'utf-8')) as Record<string, string[]>;
 
-function fetch (url: string): Promise<Response> {
-  return fetchWithTimeout(url, 5000);
-}
-
 // loop through each site for a number of times, applying the transform
 async function loopSome (site: string, matcher: () => Promise<string[] | null>): Promise<[string, string[]]> {
   const found: string[] = [];
@@ -46,7 +43,7 @@ async function loopSome (site: string, matcher: () => Promise<string[] | null>):
 // shared between polkadot.center & polkadot-event.com (addresses are also the same on first run)
 function checkGetWallet (site: string): Promise<[string, string[]]> {
   return loopSome(site, async (): Promise<string[] | null> => {
-    const result = await (await fetch(`https://${site}/get_wallet.php`)).json() as Record<string, string>;
+    const result = await fetchJson<Record<string, string>>(`https://${site}/get_wallet.php`, TIMEOUT);
 
     return (result && result.wallet)
       ? [result.wallet.replace('\r', '').trim()]
@@ -59,7 +56,7 @@ function checkTag (url: string, tag: string, attr?: string): Promise<[string, st
   const site = url.split('/')[2];
 
   return loopSome(site, async (): Promise<string[] | null> => {
-    const result = await (await fetch(url)).text();
+    const result = await fetchText(url, TIMEOUT);
 
     // /<p id="trnsctin">(.*?)<\/p>/g
     const match = new RegExp(`<${tag}${attr ? ` ${attr}` : ''}>(.*?)</${tag}>`, 'g').exec(result);
@@ -82,7 +79,7 @@ function checkAttr (url: string, attr: string): Promise<[string, string[]]> {
   const site = url.split('/')[2];
 
   return loopSome(site, async (): Promise<string[] | null> => {
-    const result = await (await fetch(url)).text();
+    const result = await fetchText(url, TIMEOUT);
     const match = new RegExp(`${attr}"[a-zA-Z0-9]+"`, 'g').exec(result);
 
     return match && match.length

packages/phishing/src/bundle.ts

@@ -6,7 +6,7 @@ import type { AddressList, HostList } from './types';
 import { u8aEq } from '@polkadot/util';
 import { decodeAddress } from '@polkadot/util-crypto';
 
-import { fetchWithTimeout } from './fetch';
+import { fetchJson } from './fetch';
 
 export { packageInfo } from './packageInfo';
 
@@ -29,39 +29,44 @@ function extractHost (path: string): string {
     .split('/')[0];
 }
 
+// logs an error in a consistent format
+function log (error: unknown, check: string): void {
+  console.warn(`Error checking ${check}, assuming non-phishing`, (error as Error).message);
+}
+
 /**
  * Retrieve a list of known phishing addresses
  */
 export async function retrieveAddrList (allowCached = true): Promise<AddressList> {
   const now = Date.now();
 
-  if (allowCached && cacheAddrList && (now < cacheAddrEnd)) {
-    return cacheAddrList;
-  }
+  return (allowCached && cacheAddrList && (now < cacheAddrEnd))
+    ? cacheAddrList
+    : fetchJson<AddressList>(ADDRESS_JSON).then((list) => {
+      cacheAddrEnd = now + CACHE_TIMEOUT;
+      cacheAddrList = list;
 
-  const response = await fetchWithTimeout(ADDRESS_JSON);
-  const list = (await response.json()) as AddressList;
-
-  cacheAddrEnd = now + CACHE_TIMEOUT;
-  cacheAddrList = list;
-
-  return list;
+      return list;
+    });
 }
 
+/**
+ * Retrieve a list of known phishing addresses in raw Uint8Array format
+ */
 async function retrieveAddrU8a (allowCached = true): Promise<[string, Uint8Array[]][]> {
   const now = Date.now();
 
-  if (allowCached && cacheAddrU8a && (now < cacheAddrEnd)) {
-    return cacheAddrU8a;
-  }
+  return (allowCached && cacheAddrU8a && (now < cacheAddrEnd))
+    ? cacheAddrU8a
+    : retrieveAddrList(allowCached).then((all) => {
+      cacheAddrU8a = Object
+        .entries(all)
+        .map(([key, addresses]): [string, Uint8Array[]] =>
+          [key, addresses.map((a) => decodeAddress(a))]
+        );
 
-  const all = await retrieveAddrList(allowCached);
-
-  cacheAddrU8a = Object
-    .entries(all)
-    .map(([key, addresses]): [string, Uint8Array[]] => [key, addresses.map((a) => decodeAddress(a))]);
-
-  return cacheAddrU8a;
+      return cacheAddrU8a;
+    });
 }
 
 /**
@@ -70,17 +75,14 @@ async function retrieveAddrU8a (allowCached = true): Promise<[string, Uint8Array
 export async function retrieveHostList (allowCached = true): Promise<HostList> {
   const now = Date.now();
 
-  if (allowCached && cacheHostList && (now < cacheHostEnd)) {
-    return cacheHostList;
-  }
+  return (allowCached && cacheHostList && (now < cacheHostEnd))
+    ? cacheHostList
+    : fetchJson<HostList>(ALL_JSON).then((list) => {
+      cacheHostEnd = now + CACHE_TIMEOUT;
+      cacheHostList = list;
 
-  const response = await fetchWithTimeout(ALL_JSON);
-  const list = (await response.json()) as HostList;
-
-  cacheHostEnd = now + CACHE_TIMEOUT;
-  cacheHostList = list;
-
-  return list;
+      return list;
+    });
 }
 
 /**
@@ -108,13 +110,13 @@ export function checkHost (items: string[], host: string): boolean {
  */
 export async function checkAddress (address: string | Uint8Array, allowCached = true): Promise<string | null> {
   try {
-    const all = await retrieveAddrU8a(allowCached);
     const u8a = decodeAddress(address);
+    const all = await retrieveAddrU8a(allowCached);
     const entry = all.find(([, all]) => all.some((a) => u8aEq(a, u8a))) || [null];
 
     return entry[0];
   } catch (error) {
-    console.error('Exception while checking address, assuming non-phishing', (error as Error).message);
+    log(error, 'address');
 
     return null;
   }
@@ -130,7 +132,7 @@ export async function checkIfDenied (host: string, allowCached = true): Promise<
 
     return checkHost(deny, host);
   } catch (error) {
-    console.error(`Exception while checking ${host}, assuming non-phishing`, (error as Error).message);
+    log(error, host);
 
     return false;
   }

packages/phishing/src/fetch.ts

@@ -4,7 +4,7 @@
 import { fetch } from '@polkadot/x-fetch';
 
 // a fetch with a 2s timeout
-export async function fetchWithTimeout (url: string, timeout = 2000): Promise<Response> {
+async function fetchWithTimeout (url: string, timeout = 2000): Promise<Response> {
   const controller = new AbortController();
   let isAborted = false;
   const id = setTimeout((): void => {
@@ -28,3 +28,11 @@ export async function fetchWithTimeout (url: string, timeout = 2000): Promise<Re
     throw error;
   }
 }
+
+export function fetchJson <T> (url: string, timeout = 2000): Promise<T> {
+  return fetchWithTimeout(url, timeout).then<T>((r) => r.json());
+}
+
+export function fetchText (url: string, timeout = 2000): Promise<string> {
+  return fetchWithTimeout(url, timeout).then((r) => r.text());
+}