Add crosscheck action (#13)

.github/crosscheck.md

@@ -0,0 +1,8 @@
+---
+title: Phishing missing cross-check entries
+labels: ci
+---
+
+Additional entries have been detected in other databases.
+
+Check the nightly cron output or run `yarn phishing:crosscheck` locally. The output as found from the test includes:

.github/workflows/crosscheck.yml

@@ -0,0 +1,24 @@
+name: crosscheck
+on:
+  schedule:
+    - cron:  '45 4 * * *'
+
+jobs:
+  ss58:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: check
+      env:
+        CI_LOG: 123
+      run: |
+        yarn install --immutable | grep -v 'YN0013'
+        yarn phishing:crosscheck
+
+    - name: issue
+      if: ${{ failure() }}
+      uses: JasonEtco/create-an-issue@v2
+      env:
+        GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+      with:
+        filename: .github/crosscheck.md

package.json

@@ -15,8 +15,9 @@
     "build:release": "polkadot-ci-ghact-build --skip-beta",
     "lint": "polkadot-dev-run-lint",
     "clean": "polkadot-dev-clean-build",
+    "phishing:crosscheck": "polkadot-dev-run-test packages/phishing/src/crosscheck",
     "postinstall": "polkadot-dev-yarn-only",
-    "test": "polkadot-dev-run-test --coverage --runInBand"
+    "test": "polkadot-dev-run-test --coverage --runInBand --testPathIgnorePatterns crosscheck"
   },
   "devDependencies": {
     "@babel/core": "^7.12.3",

packages/phishing/package.json

@@ -23,6 +23,10 @@
   "homepage": "https://github.com/polkadot-js/common/tree/master/packages/phishing#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.1",
-    "@polkadot/x-fetch": "0.3.3"
+    "@polkadot/x-fetch": "^0.3.3"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^3.12.5",
+    "js-yaml": "^3.14.0"
   }
 }

packages/phishing/src/crosscheck.spec.ts

@@ -0,0 +1,52 @@
+// Copyright 2020 @polkadot/phishing authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from 'fs';
+import { safeLoad } from 'js-yaml';
+import fetch from '@polkadot/x-fetch';
+
+import { retrieveHostList } from '.';
+
+interface CryptoScamEntry {
+  addresses: Record<string, string[]>;
+  category: string;
+  description: string;
+  name: string;
+  resporter: string;
+  subcategory: string;
+  url: string;
+}
+
+function assertAndLog (check: boolean, error: string): void {
+  if (!check) {
+    process.env.CI_LOG && fs.appendFileSync('./.github/crosscheck.md', `
+
+${error}
+`);
+
+    throw new Error(error);
+  }
+}
+
+const CRYPTOSCAM = 'https://raw.githubusercontent.com/CryptoScamDB/blacklist/master/data/urls.yaml';
+
+describe('crosscheck', (): void => {
+  let scamDb: CryptoScamEntry[];
+  let ours: string[];
+
+  beforeAll(async (): Promise<void> => {
+    ours = (await retrieveHostList()).deny;
+    scamDb = safeLoad(await (await fetch(CRYPTOSCAM)).text()) as CryptoScamEntry[];
+  });
+
+  it('has all the relevant entries from CryptoScamDb', (): void => {
+    const filtered = scamDb.filter(({ subcategory }) => subcategory === 'Polkadot');
+    const missing = filtered.filter(({ url }) =>
+      !ours.includes(url.replace(/https:\/\/|http:\/\//, '').split('/')[0])
+    );
+
+    console.log(JSON.stringify(filtered, null, 2));
+
+    assertAndLog(missing.length === 0, `Missing entries found from CryptoScamDB: ${JSON.stringify(missing, null, 2)}`);
+  });
+});

yarn.lock

@@ -1823,7 +1823,9 @@ __metadata:
   resolution: "@polkadot/phishing@workspace:packages/phishing"
   dependencies:
     "@babel/runtime": ^7.12.1
-    "@polkadot/x-fetch": 0.3.3
+    "@polkadot/x-fetch": ^0.3.3
+    "@types/js-yaml": ^3.12.5
+    js-yaml: ^3.14.0
   languageName: unknown
   linkType: soft
 
@@ -1833,7 +1835,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@polkadot/x-fetch@npm:0.3.3":
+"@polkadot/x-fetch@npm:^0.3.3":
   version: 0.3.3
   resolution: "@polkadot/x-fetch@npm:0.3.3"
   dependencies:
@@ -1980,6 +1982,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/js-yaml@npm:^3.12.5":
+  version: 3.12.5
+  resolution: "@types/js-yaml@npm:3.12.5"
+  checksum: 82ca4b40ee1baf82961f75b7db8fae5a1d63410c066475424659c3bd1873cf02bf015ebdeb8751e7b6d0c87b4d3f09f20d2cc2075c4bdd4e7d58409ddd6ad925
+  languageName: node
+  linkType: hard
+
 "@types/json-schema@npm:^7.0.3":
   version: 7.0.6
   resolution: "@types/json-schema@npm:7.0.6"
@@ -7221,7 +7230,7 @@ fsevents@^1.2.7:
   languageName: node
   linkType: hard
 
-"js-yaml@npm:^3.13.1":
+"js-yaml@npm:^3.13.1, js-yaml@npm:^3.14.0":
   version: 3.14.0
   resolution: "js-yaml@npm:3.14.0"
   dependencies: