diff --git a/.github/crosscheck.md b/.github/crosscheck.md new file mode 100644 index 000000000..80359a13e --- /dev/null +++ b/.github/crosscheck.md @@ -0,0 +1,8 @@ +--- +title: Phishing missing cross-check entries +labels: ci +--- + +Additional entries have been detected in other databases. + +Check the nightly cron output or run `yarn phishing:crosscheck` locally. The output as found from the test includes: diff --git a/.github/workflows/crosscheck.yml b/.github/workflows/crosscheck.yml new file mode 100644 index 000000000..06facc17a --- /dev/null +++ b/.github/workflows/crosscheck.yml @@ -0,0 +1,24 @@ +name: crosscheck +on: + schedule: + - cron: '45 4 * * *' + +jobs: + ss58: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: check + env: + CI_LOG: 123 + run: | + yarn install --immutable | grep -v 'YN0013' + yarn phishing:crosscheck + + - name: issue + if: ${{ failure() }} + uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + with: + filename: .github/crosscheck.md diff --git a/package.json b/package.json index 214f9d39e..00dd32f9f 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,9 @@ "build:release": "polkadot-ci-ghact-build --skip-beta", "lint": "polkadot-dev-run-lint", "clean": "polkadot-dev-clean-build", + "phishing:crosscheck": "polkadot-dev-run-test packages/phishing/src/crosscheck", "postinstall": "polkadot-dev-yarn-only", - "test": "polkadot-dev-run-test --coverage --runInBand" + "test": "polkadot-dev-run-test --coverage --runInBand --testPathIgnorePatterns crosscheck" }, "devDependencies": { "@babel/core": "^7.12.3", diff --git a/packages/phishing/package.json b/packages/phishing/package.json index e2f884e5c..27d8ee5f4 100644 --- a/packages/phishing/package.json +++ b/packages/phishing/package.json @@ -23,6 +23,10 @@ "homepage": "https://github.com/polkadot-js/common/tree/master/packages/phishing#readme", "dependencies": { "@babel/runtime": "^7.12.1", - "@polkadot/x-fetch": "0.3.3" + "@polkadot/x-fetch": "^0.3.3" + }, + "devDependencies": { + "@types/js-yaml": "^3.12.5", + "js-yaml": "^3.14.0" } } diff --git a/packages/phishing/src/crosscheck.spec.ts b/packages/phishing/src/crosscheck.spec.ts new file mode 100644 index 000000000..5347ec73d --- /dev/null +++ b/packages/phishing/src/crosscheck.spec.ts @@ -0,0 +1,52 @@ +// Copyright 2020 @polkadot/phishing authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import fs from 'fs'; +import { safeLoad } from 'js-yaml'; +import fetch from '@polkadot/x-fetch'; + +import { retrieveHostList } from '.'; + +interface CryptoScamEntry { + addresses: Record; + category: string; + description: string; + name: string; + resporter: string; + subcategory: string; + url: string; +} + +function assertAndLog (check: boolean, error: string): void { + if (!check) { + process.env.CI_LOG && fs.appendFileSync('./.github/crosscheck.md', ` + +${error} +`); + + throw new Error(error); + } +} + +const CRYPTOSCAM = 'https://raw.githubusercontent.com/CryptoScamDB/blacklist/master/data/urls.yaml'; + +describe('crosscheck', (): void => { + let scamDb: CryptoScamEntry[]; + let ours: string[]; + + beforeAll(async (): Promise => { + ours = (await retrieveHostList()).deny; + scamDb = safeLoad(await (await fetch(CRYPTOSCAM)).text()) as CryptoScamEntry[]; + }); + + it('has all the relevant entries from CryptoScamDb', (): void => { + const filtered = scamDb.filter(({ subcategory }) => subcategory === 'Polkadot'); + const missing = filtered.filter(({ url }) => + !ours.includes(url.replace(/https:\/\/|http:\/\//, '').split('/')[0]) + ); + + console.log(JSON.stringify(filtered, null, 2)); + + assertAndLog(missing.length === 0, `Missing entries found from CryptoScamDB: ${JSON.stringify(missing, null, 2)}`); + }); +}); diff --git a/yarn.lock b/yarn.lock index 8451ae4eb..de26a54f6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1823,7 +1823,9 @@ __metadata: resolution: "@polkadot/phishing@workspace:packages/phishing" dependencies: "@babel/runtime": ^7.12.1 - "@polkadot/x-fetch": 0.3.3 + "@polkadot/x-fetch": ^0.3.3 + "@types/js-yaml": ^3.12.5 + js-yaml: ^3.14.0 languageName: unknown linkType: soft @@ -1833,7 +1835,7 @@ __metadata: languageName: unknown linkType: soft -"@polkadot/x-fetch@npm:0.3.3": +"@polkadot/x-fetch@npm:^0.3.3": version: 0.3.3 resolution: "@polkadot/x-fetch@npm:0.3.3" dependencies: @@ -1980,6 +1982,13 @@ __metadata: languageName: node linkType: hard +"@types/js-yaml@npm:^3.12.5": + version: 3.12.5 + resolution: "@types/js-yaml@npm:3.12.5" + checksum: 82ca4b40ee1baf82961f75b7db8fae5a1d63410c066475424659c3bd1873cf02bf015ebdeb8751e7b6d0c87b4d3f09f20d2cc2075c4bdd4e7d58409ddd6ad925 + languageName: node + linkType: hard + "@types/json-schema@npm:^7.0.3": version: 7.0.6 resolution: "@types/json-schema@npm:7.0.6" @@ -7221,7 +7230,7 @@ fsevents@^1.2.7: languageName: node linkType: hard -"js-yaml@npm:^3.13.1": +"js-yaml@npm:^3.13.1, js-yaml@npm:^3.14.0": version: 3.14.0 resolution: "js-yaml@npm:3.14.0" dependencies: