From d9c093946e66b4f3101ef96937b22081410af824 Mon Sep 17 00:00:00 2001
From: Jaco <jacogr@gmail.com>
Date: Wed, 27 Jul 2022 12:08:19 +0200
Subject: [PATCH] Plesk site flatenning (#1933)

---
 all.json            | 142 ++++++++++++++++----------------------------
 scripts/sortAll.mjs |  50 +++++++++++++---
 2 files changed, 93 insertions(+), 99 deletions(-)

diff --git a/all.json b/all.json
index 99cf97acc..c2ef100ea 100644
--- a/all.json
+++ b/all.json
@@ -728,7 +728,7 @@
 		"bcnodecenter.net",
 		"bcollabnet.com",
 		"bcsminer.com",
-		"beautiful-benz.5-253-84-133.plesk.page",
+		"5-253-84-133.plesk.page",
 		"beautyads.comytvweb.com",
 		"beefy.cloud",
 		"beelts.com",
@@ -1183,11 +1183,11 @@
 		"click-connect.info",
 		"clicltocollab.com",
 		"clienta.pages.dev",
-		"clientoverview-metamask.185-117-91-189.plesk.page",
-		"clientprotocol.metamask.team.securedesk.79-141-164-34.plesk.page",
+		"185-117-91-189.plesk.page",
+		"79-141-164-34.plesk.page",
 		"clientswcrectify.com",
-		"clientupdate-metamask.185-117-91-222.plesk.page",
-		"clientupdate.metamask.team.zendesk.185-236-231-184.plesk.page",
+		"185-117-91-222.plesk.page",
+		"185-236-231-184.plesk.page",
 		"cloud-walletconnect.com",
 		"cloudauthentions.com",
 		"cloudcryptoresolver.com",
@@ -1502,9 +1502,8 @@
 		"comb.fund",
 		"combinewallet.com",
 		"community-airdrop-program.com",
-		"community.trustwallet.com-recovery.45-88-108-231.plesk.page",
-		"community.trustwallet.com.45-88-108-231.plesk.page",
-		"compassionate-lalande.96-9-210-124.plesk.page",
+		"45-88-108-231.plesk.page",
+		"96-9-210-124.plesk.page",
 		"concertifyonwall.org",
 		"concordium-bridge.phi-blockchain.com",
 		"condappsjoint.com",
@@ -1826,7 +1825,7 @@
 		"craftyfisales.io",
 		"cravedappstect.com",
 		"crazeeswap.com",
-		"crazy-lamport.203-25-119-115.plesk.page",
+		"203-25-119-115.plesk.page",
 		"create-cryptocurrency-wallet.com",
 		"cresprotocol.digital",
 		"criptofaci.com",
@@ -2026,7 +2025,7 @@
 		"cryptowalletrules.com",
 		"cryptowallets.ch",
 		"cryptowallets.code.blog",
-		"cryptowallets.compassionate-nobel.217-61-128-227.plesk.page",
+		"217-61-128-227.plesk.page",
 		"cryptowallets.online",
 		"cryptowallets.top",
 		"cryptowalletshelp.com",
@@ -4014,7 +4013,7 @@
 		"giftclaim.buzz",
 		"giftcollab.land",
 		"giftconnerz.buzz",
-		"gifted-edison.203-9-150-224.plesk.page",
+		"203-9-150-224.plesk.page",
 		"ginxappsolutionsdev.live",
 		"git-tools.com",
 		"give-away-ixswap.com",
@@ -4108,7 +4107,7 @@
 		"gulfcardinalbank.com",
 		"h5-uniswape.com",
 		"hanzoretri.online",
-		"happy-rosalind.193-111-31-88.plesk.page",
+		"193-111-31-88.plesk.page",
 		"hardwalletcapsule.com",
 		"hardwalletcapsule.store",
 		"hardwalletsafe.com",
@@ -4129,7 +4128,7 @@
 		"help-metamask.ml",
 		"help-tool.net",
 		"help.mining-extracts.com",
-		"helpdask-supportgroup.109-71-253-24.plesk.page",
+		"109-71-253-24.plesk.page",
 		"helpdeskcoinvalidaton.com",
 		"helpukraine2022.online",
 		"helpuser.org",
@@ -4479,7 +4478,7 @@
 		"integrationdapp.org",
 		"integrationprotocol.info",
 		"inteldigital.trade",
-		"intelligent-shirley.92-118-190-138.plesk.page",
+		"92-118-190-138.plesk.page",
 		"intellsyncedapps.online",
 		"intelphasenet.com",
 		"intelphasesolution.com",
@@ -4825,7 +4824,6 @@
 		"login-metamaskweb.com",
 		"login-poigylon.technology",
 		"login-wallet.top",
-		"login.trustwallet.45-88-108-231.plesk.page",
 		"logmainnetconnect.online",
 		"logwalletconnect.online",
 		"loliverse.io",
@@ -5227,7 +5225,7 @@
 		"metamask-iowallet.rhymetimedaycare.com",
 		"metamask-keyst.com",
 		"metamask-kyc-regulations.de",
-		"metamask-kyc-standards.185-236-231-141.plesk.page",
+		"185-236-231-141.plesk.page",
 		"metamask-kycverif.com",
 		"metamask-live.com",
 		"metamask-logi.com",
@@ -5246,19 +5244,17 @@
 		"metamask-protocol.com",
 		"metamask-re.jatra-apps.org",
 		"metamask-recover-account.tk",
-		"metamask-recover.185-236-231-227.plesk.page",
+		"185-236-231-227.plesk.page",
 		"metamask-recovery.icu",
 		"metamask-recovery.info",
 		"metamask-recovery.net",
 		"metamask-recovery.org",
 		"metamask-recovery.top",
-		"metamask-regulation-server.45-88-108-231.plesk.page",
 		"metamask-restore.live",
 		"metamask-restore.online",
 		"metamask-secure.com",
 		"metamask-secure.online",
-		"metamask-support.clever-jones.35-203-88-159.plesk.page",
-		"metamask-support.pensive-mestorf.35-203-88-159.plesk.page",
+		"35-203-88-159.plesk.page",
 		"metamask-support.serveirc.com",
 		"metamask-supportdesk.com",
 		"metamask-sync.com",
@@ -5269,7 +5265,7 @@
 		"metamask-update-v2.com",
 		"metamask-v.com",
 		"metamask-validate.myftp.biz",
-		"metamask-validation.179-43-176-123.plesk.page",
+		"179-43-176-123.plesk.page",
 		"metamask-verification.app-animate.online",
 		"metamask-verification.us",
 		"metamask-verify-procedure.com",
@@ -5318,7 +5314,7 @@
 		"metamask.com.srmarine-bd.com",
 		"metamask.com.verifytoken.in",
 		"metamask.com.vn",
-		"metamask.community.unlock.45-81-232-16.plesk.page",
+		"45-81-232-16.plesk.page",
 		"metamask.coralscookiesinc.com",
 		"metamask.ddns.net",
 		"metamask.dedeveloperz.tech",
@@ -5349,25 +5345,22 @@
 		"metamask.io.app-animate.online",
 		"metamask.io.cikgutekansiap.com",
 		"metamask.io.educapp.pe",
-		"metamask.io.flamboyant-black.111-90-143-152.plesk.page",
-		"metamask.io.kyc-regulations.107-172-188-102.plesk.page",
-		"metamask.io.kyc.regulation.45-88-108-231.plesk.page",
+		"111-90-143-152.plesk.page",
+		"107-172-188-102.plesk.page",
 		"metamask.io.ldown.pro",
 		"metamask.io.lgdn.live",
 		"metamask.io.metamaskauthorization.in",
 		"metamask.io.moveaps.com",
 		"metamask.io.pt-umi.co.id",
-		"metamask.io.support-center.45-88-108-231.plesk.page",
 		"metamask.io.vihax.com",
 		"metamask.io.wallet-identification.divu.cl",
 		"metamask.io.wallet-identification.rwatelier.com",
 		"metamask.io.wallet-identification.softsayo.com",
 		"metamask.io.web7896.web07.bero-webspace.de",
-		"metamask.io.zendesk.strange-napier.135-125-137-197.plesk.page",
+		"135-125-137-197.plesk.page",
 		"metamask.is",
 		"metamask.iwtoken.cn",
 		"metamask.kiwi",
-		"metamask.kyc.45-88-108-231.plesk.page",
 		"metamask.kyc4you.io",
 		"metamask.lav09dev.tk",
 		"metamask.limo",
@@ -5385,25 +5378,19 @@
 		"metamask.plantsvundead.com",
 		"metamask.protocol-process.me",
 		"metamask.psanda.co.za",
-		"metamask.recovery.mode.185-100-87-154.plesk.page",
-		"metamask.recovery.mode.45-88-108-231.plesk.page",
+		"185-100-87-154.plesk.page",
 		"metamask.rent",
 		"metamask.ru-php.io",
 		"metamask.secure-confirm.com",
 		"metamask.secure.login.energykn.com",
-		"metamask.security.procedure.109-71-253-24.plesk.page",
 		"metamask.services",
-		"metamask.sharp-shirley.34-151-85-42.plesk.page",
-		"metamask.support.centrum.109-71-253-24.plesk.page",
-		"metamask.suspicious-sinoussi.34-92-30-20.plesk.page",
+		"34-151-85-42.plesk.page",
+		"34-92-30-20.plesk.page",
 		"metamask.sx",
 		"metamask.tech",
 		"metamask.ventures",
-		"metamask.wallet-verification.185-236-231-141.plesk.page",
-		"metamask.wallet.recovery.185-100-87-154.plesk.page",
-		"metamask.wallet.recovery.45-88-108-231.plesk.page",
-		"metamask.wallet.recovery.45-93-248-121.plesk.page",
-		"metamask.walletconnect.185-249-218-167.plesk.page",
+		"45-93-248-121.plesk.page",
+		"185-249-218-167.plesk.page",
 		"metamask.watch",
 		"metamask.webvalidate.me",
 		"metamask.xn--cm-68s.cc",
@@ -5458,7 +5445,6 @@
 		"metamaske.online",
 		"metamaske.ru",
 		"metamaske.top",
-		"metamaskenfos.com.modest-franklin.35-203-88-159.plesk.page",
 		"metamaskextension.io",
 		"metamaskey.xyz",
 		"metamaskflask.com",
@@ -5513,7 +5499,7 @@
 		"metamaskotas.com",
 		"metamaskpro.net",
 		"metamaskprotection.com",
-		"metamaskrecover.179-43-187-217.plesk.page",
+		"179-43-187-217.plesk.page",
 		"metamaskregister.com",
 		"metamaskrestore.website",
 		"metamaskreward.com",
@@ -5753,7 +5739,7 @@
 		"mobileworldappsstore.xyz",
 		"mobwallet.co",
 		"moderatorchannel.live",
-		"modest-yalow.192-3-122-179.plesk.page",
+		"192-3-122-179.plesk.page",
 		"mojitoswap.io",
 		"mojitoswap.org",
 		"mojitoswep.finance",
@@ -5924,7 +5910,6 @@
 		"mwyehtrewellet.com",
 		"my-etherwallets.com",
 		"my-token.online",
-		"my-trustwallet.45-88-108-231.plesk.page",
 		"myaccessauth.com",
 		"myactivationapps.com",
 		"myappresolutioncom.com",
@@ -6002,17 +5987,16 @@
 		"mymainnetimport.com",
 		"mymainnetsync.com",
 		"mymedicalwishwallet.com",
-		"mymetamask-clientarea.185-236-231-227.plesk.page",
-		"mymetamask-onlinesecure.185-117-90-230.plesk.page",
-		"mymetamask-onlinesecure.185-81-114-144.plesk.page",
-		"mymetamask-onlineupdate.185-80-53-115.plesk.page",
-		"mymetamask-overview.79-141-164-138.plesk.page",
-		"mymetamask-overviewonline.79-141-165-217.plesk.page",
-		"mymetamask-overviewpage.185-117-91-145.plesk.page",
-		"mymetamask-recoverupdate.79-141-165-17.plesk.page",
+		"185-117-90-230.plesk.page",
+		"185-81-114-144.plesk.page",
+		"185-80-53-115.plesk.page",
+		"79-141-164-138.plesk.page",
+		"79-141-165-217.plesk.page",
+		"185-117-91-145.plesk.page",
+		"79-141-165-17.plesk.page",
 		"mymetamask-support.com",
-		"mymetamask-updateform.185-80-53-113.plesk.page",
-		"mymetamask-updateonline.185-236-231-84.plesk.page",
+		"185-80-53-113.plesk.page",
+		"185-236-231-84.plesk.page",
 		"mymetamaskwallet.ddns.net",
 		"myminrtofwart.com",
 		"mymintconnectpad.online",
@@ -6038,7 +6022,6 @@
 		"mytokenaffix.com",
 		"mytokenencrypt.com",
 		"mytokensupport.finance",
-		"mytrustwallet-clientoverview.185-236-231-84.plesk.page",
 		"mytrustwallet.info",
 		"myvalidationprotocol.netlify.app",
 		"mywallautocertify.org",
@@ -6334,7 +6317,7 @@
 		"nordicweb3foundation.org",
 		"nordsynchronization.com",
 		"normativa-ora.com",
-		"nostalgic-fermat.194-127-193-110.plesk.page",
+		"194-127-193-110.plesk.page",
 		"notificationsintuits.com",
 		"novussurgicalco.com",
 		"nowsolve.live",
@@ -6406,7 +6389,6 @@
 		"online-trust-web-wallet.com",
 		"online-uniswap.tech",
 		"online-wallet-connect.com",
-		"online-wallet.recovery.109-71-253-24.plesk.page",
 		"onlineappintegration.com",
 		"onlineappsauth.online",
 		"onlineappsconnect.com",
@@ -6796,12 +6778,8 @@
 		"paxful-com.com",
 		"paxful-token.com",
 		"paxful-wallets.com",
-		"paxful.com-login.angry-gauss.34-97-37-234.plesk.page",
-		"paxful.com-login33.nervous-golick.35-203-61-10.plesk.page",
-		"paxful.com-login33.priceless-mayer.35-203-61-10.plesk.page",
-		"paxfulcom-login.priceless-mayer.35-203-61-10.plesk.page",
-		"paxfull.com-login.nervous-golick.35-203-61-10.plesk.page",
-		"paxfull.com-login.priceless-mayer.35-203-61-10.plesk.page",
+		"34-97-37-234.plesk.page",
+		"35-203-61-10.plesk.page",
 		"paxfulmethod.com",
 		"paxfulnow.com.currently-tec.com",
 		"paxfuls.xyz",
@@ -7493,11 +7471,10 @@
 		"reconnectwallet.online",
 		"recovercryptowallets.com",
 		"recoverseed-meta.live",
-		"recovery-connect.wallet.109-71-253-24.plesk.page",
-		"recovery-metamask-io.zen-cohen.141-164-62-92.plesk.page",
+		"141-164-62-92.plesk.page",
 		"recovery-verify-metamask.com",
 		"recovery-web.io",
-		"recovery.unlock.trust.wallet.45-81-232-17.plesk.page",
+		"45-81-232-17.plesk.page",
 		"recoveryblockchain.info",
 		"recoverymaxtokenconnect.com",
 		"recoverywallet.me",
@@ -7676,7 +7653,7 @@
 		"rolbitsconnect.live",
 		"roll-collabs.land",
 		"rollcollabsync.com",
-		"romantic-wescoff.203-9-150-237.plesk.page",
+		"203-9-150-237.plesk.page",
 		"roninwalletconnectt.com",
 		"rookiedappcenter.tech",
 		"royalehosting.info",
@@ -9157,7 +9134,7 @@
 		"trust-setting.support",
 		"trust-walle.com",
 		"trust.binancbrasil.com",
-		"trust.wallet.verification.185-165-171-108.plesk.page",
+		"185-165-171-108.plesk.page",
 		"trustaidsync.online",
 		"trustair-wallet.com",
 		"trustbots.support",
@@ -9207,32 +9184,21 @@
 		"trusttoken.top",
 		"trustval.live",
 		"trustvips.com",
-		"trustwaflleby.peaceful-brattain.185-236-231-93.plesk.page",
+		"185-236-231-93.plesk.page",
 		"trustwalet.space",
 		"trustwalet.top",
 		"trustwalet.vip",
 		"trustwaletru.com",
 		"trustwaliet.site",
 		"trustwaliet.website",
-		"trustwallet-community.109-71-253-24.plesk.page",
-		"trustwallet-edison.45-88-108-231.plesk.page",
-		"trustwallet-online-recovery.45-88-108-231.plesk.page",
 		"trustwallet-tokengiveaway.store",
-		"trustwallet-unlock.109-71-253-24.plesk.page",
 		"trustwallet-verifyid-16.ml",
-		"trustwallet.1.45-88-108-231.plesk.page",
-		"trustwallet.109-71-253-24.plesk.page",
 		"trustwallet.builders",
 		"trustwallet.capital",
 		"trustwallet.com-multi-chain.info",
-		"trustwallet.com.s1.45-88-108-231.plesk.page",
-		"trustwallet.community.com.45-88-108-231.plesk.page",
-		"trustwallet.festive-williamson.143-198-124-192.plesk.page",
-		"trustwallet.i.dreamy-chebyshev.109-71-253-24.plesk.page",
-		"trustwallet.i.vibrant-mirzakhani.45-81-232-17.plesk.page",
+		"143-198-124-192.plesk.page",
 		"trustwallet.liquidity-airdrop.online",
-		"trustwallet.lucid-robinson.143-244-162-16.plesk.page",
-		"trustwallet.recovery.community.109-71-253-24.plesk.page",
+		"143-244-162-16.plesk.page",
 		"trustwalletairdrup.com",
 		"trustwalletapi.com",
 		"trustwalletapp.me",
@@ -9264,7 +9230,7 @@
 		"trustwallett.site",
 		"trustwallettechsupport.com",
 		"trustwalletts.net",
-		"trustwalletunlock.gifted-lumiere.138-68-99-93.plesk.page",
+		"138-68-99-93.plesk.page",
 		"trustwalllet.xyz",
 		"trustwebapp.online",
 		"trustweilet.com",
@@ -9475,10 +9441,7 @@
 		"unlock-assets.com",
 		"unlock-connection.com",
 		"unlock-keystore.online",
-		"unlock-trustwallet.45-81-232-17.plesk.page",
-		"unlock-trustwallet.45-88-108-231.plesk.page",
 		"unlock-wallet-connect.com",
-		"unlock.trust.wallet.109-71-253-24.plesk.page",
 		"unlockallassets.com",
 		"unlockassetchain.com",
 		"unlsswapp.com",
@@ -9728,7 +9691,7 @@
 		"verify-my-metamask-account-now.io",
 		"verify-mycollabland.com",
 		"verify-recoveryphrase.com",
-		"verify-trustwallet.179-43-154-180.plesk.page",
+		"179-43-154-180.plesk.page",
 		"verify.rectify.link",
 		"verify.yuzu--swap.com",
 		"verifyasset.net",
@@ -9942,7 +9905,6 @@
 		"wallet-fix.net",
 		"wallet-help.net",
 		"wallet-helpline.com",
-		"wallet-id.109-71-253-24.plesk.page",
 		"wallet-integrate.dev",
 		"wallet-link-dapps.com",
 		"wallet-link.online",
@@ -10178,7 +10140,6 @@
 		"walletconnect-verify.netlify.app",
 		"walletconnect-web3.com",
 		"walletconnect.000webhostapp.com",
-		"walletconnect.179-43-154-180.plesk.page",
 		"walletconnect.alessandrocampos.com.br",
 		"walletconnect.appspots.net",
 		"walletconnect.biz",
@@ -11487,8 +11448,8 @@
 		"yourcryptocare.xyz",
 		"yournftverification.com",
 		"yourpolkadot.js.org",
-		"yourtrustwallet.eloquent-jemison.137-184-141-8.plesk.page",
-		"yourtrustwallet.wonderful-aryabhata.64-227-4-245.plesk.page",
+		"137-184-141-8.plesk.page",
+		"64-227-4-245.plesk.page",
 		"yourverifkyc.com",
 		"yourverifwallet.com",
 		"yourwalletconnect.online",
@@ -11506,7 +11467,6 @@
 		"zealtm.group",
 		"zeitgeistsale.live",
 		"zellespay.com",
-		"zendesk.metamask.io.strange-napier.135-125-137-197.plesk.page",
 		"zerionwallet.com",
 		"zero-swap.site",
 		"zinotiger.com",
diff --git a/scripts/sortAll.mjs b/scripts/sortAll.mjs
index e360dbf63..03c9d6c61 100644
--- a/scripts/sortAll.mjs
+++ b/scripts/sortAll.mjs
@@ -13,7 +13,7 @@ function sanitizeUrl (url) {
   ).split('/')[0];
 }
 
-function sortSection (list) {
+function filterSection (list) {
   return list
     .map((entry) => sanitizeUrl(entry))
     .reduce((filtered, entry) => {
@@ -21,8 +21,11 @@ function sortSection (list) {
         filtered.push(entry);
 
       return filtered;
-    }, [])
-    .sort((a, b) => a.localeCompare(b));
+    }, []);
+}
+
+function sortSection (list) {
+  return filterSection(list).sort((a, b) => a.localeCompare(b));
 }
 
 function isSubdomain (list, url) {
@@ -30,7 +33,7 @@ function isSubdomain (list, url) {
 
   for (let i = 1; i < parts.length - 1; i++) {
     if (list.includes(parts.slice(i).join('.'))) {
-      // this is a sub-domain of a domain that alreeady exists
+      // this is a sub-domain of a domain that already exists
       return true;
     }
   }
@@ -38,8 +41,25 @@ function isSubdomain (list, url) {
   return false;
 }
 
-function removeSubs (list) {
-  return list.filter((url) => !isSubdomain(list, url));
+function flattenUrl (url) {
+  // currently we only check for plesk-page to flatten
+  if (!url.endsWith('plesk.page')) {
+    return url;
+  }
+
+  const parts = url.split('.');
+
+  return parts.length > 3
+    ? parts.slice(-3).join('.')
+    : url;
+}
+
+function rewriteSubs (list) {
+  return filterSection(
+    list
+      .filter((url) => !isSubdomain(list, url))
+      .map((url) => flattenUrl(url))
+  );
 }
 
 function sortAddresses (values) {
@@ -124,7 +144,7 @@ const deny = sortSection(addSites(all, addr));
 
 // rewrite with all our entries (newline included)
 writeJson('address.json', sortAddresses(addr));
-writeJson('all.json', { allow: sortSection(all.allow), deny: removeSubs(deny) });
+writeJson('all.json', { allow: sortSection(all.allow), deny: rewriteSubs(deny) });
 
 // find out what we don't have
 const urls = meta.map(({ url }) => url);
@@ -132,12 +152,26 @@ const now = new Date();
 const ym = `${now.getUTCFullYear()}-${`00${now.getUTCMonth() + 1}`.slice(-2)}`;
 const ymd = `${ym}-${`00${now.getUTCDate()}`.slice(-2)}`;
 
+// helper for parts
+const urlParts = urls.map((u) => u.split('.'));
+
 // rewrite with all our entries (newline included)
 writeMeta(
   meta
     .concat(
       deny
-        .filter((url) => !urls.includes(url))
+        .filter((url) => {
+          if (urls.includes(url)) {
+            return false;
+          }
+
+          const len = url.split('.').length;
+
+          return !urlParts.some((p) =>
+            len < p.length &&
+            url === p.slice(-len).join('.')
+          );
+        })
         .map((url) => ({ date: ymd, url }))
     )
     .filter(({ url }) =>