Extend phishing with full URL support (#11)

* Extend phishing with full URL support

* Cleanups

* false on network exception

package.json

@@ -8,7 +8,7 @@
   ],
   "resolutions": {
     "babel-core": "^7.0.0-bridge.0",
-    "typescript": "^4.0.2"
+    "typescript": "^4.0.3"
   },
   "scripts": {
     "build": "polkadot-dev-build-ts",
@@ -16,11 +16,11 @@
     "lint": "polkadot-dev-run-lint",
     "clean": "polkadot-dev-clean-build",
     "postinstall": "polkadot-dev-yarn-only",
-    "test": "echo \"no tests\""
+    "test": "polkadot-dev-run-test --coverage --runInBand"
   },
   "devDependencies": {
-    "@babel/core": "^7.11.6",
-    "@polkadot/dev": "^0.58.1",
+    "@babel/core": "^7.12.3",
+    "@polkadot/dev": "^0.58.3",
     "@types/jest": "^26.0.14"
   },
   "version": "0.1.6"

packages/phishing/package.json

@@ -22,7 +22,7 @@
   },
   "homepage": "https://github.com/polkadot-js/common/tree/master/packages/phishing#readme",
   "dependencies": {
-    "@babel/runtime": "^7.11.2",
-    "@polkadot/x-fetch": "0.3.2"
+    "@babel/runtime": "^7.12.1",
+    "@polkadot/x-fetch": "0.3.3"
   }
 }

packages/phishing/src/index.spec.ts

@@ -0,0 +1,36 @@
+// Copyright 2020 @polkadot/phishing authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import checkIfDenied from '.';
+
+describe('checkIfDenied', (): void => {
+  it('returns false when host in list', async (): Promise<void> => {
+    expect(
+      await checkIfDenied('polkadot.network')
+    ).toEqual(false);
+  });
+
+  it('returns false when host in list (with protocol)', async (): Promise<void> => {
+    expect(
+      await checkIfDenied('https://polkadot.network')
+    ).toEqual(false);
+  });
+
+  it('returns true when host in list', async (): Promise<void> => {
+    expect(
+      await checkIfDenied('polkawallets.site')
+    ).toEqual(true);
+  });
+
+  it('returns true when host in list (protocol)', async (): Promise<void> => {
+    expect(
+      await checkIfDenied('https://polkawallets.site')
+    ).toEqual(true);
+  });
+
+  it('returns true when host in list (path)', async (): Promise<void> => {
+    expect(
+      await checkIfDenied('https://polkawallets.site/something/index.html')
+    ).toEqual(true);
+  });
+});

packages/phishing/src/index.ts

@@ -5,17 +5,30 @@ import { HostList } from './types';
 
 import fetch from '@polkadot/x-fetch';
 
-// retrieve allow/deny from our list provider
+const ALL_JSON = 'https://polkadot.js.org/phishing/all.json';
+
+// gets the host-only part for a host
+function extractHost (path: string): string {
+  return path
+    .replace(/https:\/\/|http:\/\/|wss:\/\/|ws:\/\//, '')
+    .split('/')[0];
+}
+
+/**
+ * Retrieve allow/deny from our list provider
+ */
 export async function retrieveHostList (): Promise<HostList> {
-  const response = await fetch('https://polkadot.js.org/phishing/all.json');
+  const response = await fetch(ALL_JSON);
   const list = (await response.json()) as HostList;
 
   return list;
 }
 
-// checks a host to see if it appears in the list
+/**
+ * Checks a host to see if it appears in the provided list
+ */
 export function checkHost (items: string[], host: string): boolean {
-  const hostParts = host.split('.').reverse();
+  const hostParts = extractHost(host).split('.').reverse();
 
   return items.some((item): boolean => {
     const checkParts = item.split('.').reverse();
@@ -30,9 +43,19 @@ export function checkHost (items: string[], host: string): boolean {
   });
 }
 
-// retrieve the deny list and check the host against it
-export default async function retrieveCheckDeny (host: string): Promise<boolean> {
-  const { deny } = await retrieveHostList();
+/**
+ * Determines if a host is in our deny list. Returns true if host is a problematic one. Returns
+ * true if the host provided is in our list of less-than-honest sites.
+ */
+export default async function checkIfDenied (host: string): Promise<boolean> {
+  try {
+    const { deny } = await retrieveHostList();
 
-  return checkHost(deny, host);
+    return checkHost(deny, host);
+  } catch (error) {
+    console.error('Exception while checking host, assuming false');
+    console.error(error);
+
+    return false;
+  }
 }

packages/unused/package.json

@@ -1,16 +1,8 @@
 {
   "name": "@polkadot/unused",
   "version": "0.1.6",
-  "description": "Simple phishing scripts from a central source",
+  "description": "Dummy",
   "main": "index.js",
-  "publishConfig": {
-    "access": "public",
-    "registry": "https://registry.npmjs.org"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/polkadot-js/phishing.git"
-  },
   "author": "Jaco Greeff <jacogr@gmail.com>",
   "maintainers": [
     "Jaco Greeff <jacogr@gmail.com>"
@@ -20,9 +12,5 @@
   "bugs": {
     "url": "https://github.com/polkadot-js/phishing/issues"
   },
-  "homepage": "https://github.com/polkadot-js/common/tree/master/packages/phishing#readme",
-  "dependencies": {
-    "@babel/runtime": "^7.11.2",
-    "@polkadot/x-fetch": "0.3.2"
-  }
+  "homepage": "https://github.com/polkadot-js/phishing"
 }