CI: chore (#3957)

* CI: chore

* CI: lsof and jq are installed

* CI: generate docs with deps

* CI: remove check width, fmt checks it

* CI: simplify rules

* CI: make CI image variable

* CI: more chore

* rm check_line_width.sh

* CI: fix spellcheck; fix check-transaction

* CI: return installing jq

* lint: remove dupes

polkadot/.gitlab-ci.yml

@@ -6,7 +6,7 @@
 # setting DEPLOY_TAG will only deploy the tagged image
 #
 # please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
-# &rules-pr-only preset. And "kubernetes-env" with "docker-env" to set a runner
+# &rules-test-and-rococo preset. And "kubernetes-env" with "docker-env" to set a runner
 # which executes the job.
 
 stages:
@@ -15,8 +15,6 @@ stages:
   - publish
   - deploy
 
-image:                             paritytech/ci-linux:production
-
 workflow:
   rules:
     - if: $CI_COMMIT_TAG
@@ -26,6 +24,7 @@ variables:
   GIT_STRATEGY:                    fetch
   GIT_DEPTH:                       100
   CI_SERVER_NAME:                  "GitLab CI"
+  CI_IMAGE:                        "paritytech/ci-linux:production"
   DOCKER_OS:                       "debian:stretch"
   ARCH:                            "x86_64"
   VAULT_SERVER_URL:                "https://vault.parity-mgmt-vault.parity.io"
@@ -57,6 +56,7 @@ default:
     - kubernetes-parity-build
 
 .docker-env:                       &docker-env
+  image:                           "${CI_IMAGE}"
   retry:
     max: 2
     when:
@@ -75,20 +75,20 @@ default:
 
 .rules-test:                       &rules-test
   # these jobs run always*
-  rules:
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
-      when: never
-    - when: always
-
-.pr-only:                          &rules-pr-only
-  # these jobs run only on PRs
   rules:
     - if: $CI_COMMIT_REF_NAME == "rococo-v1"
       when: never
     - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: never
+    - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
 
+.rules-test-and-rococo:            &rules-test-and-rococo
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
+
 #### Vault secrets
 .vault-secrets:                    &vault-secrets
   secrets:
@@ -144,7 +144,7 @@ check-runtime:
   stage:                           test
   image:                           paritytech/tools:latest
   <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
+  <<:                              *rules-test
   <<:                              *vault-secrets
   variables:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
@@ -153,19 +153,10 @@ check-runtime:
     - ./scripts/gitlab/check_runtime.sh
   allow_failure:                   true
 
-check-line-width:
-  stage:                           test
-  image:                           paritytech/tools:latest
-  <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
-  script:
-    - ./scripts/gitlab/check_line_width.sh
-  allow_failure:                   true
-
 cargo-fmt:
   stage:                           test
-  <<:                              *kubernetes-env
+  <<:                              *docker-env
-  <<:                              *rules-pr-only
+  <<:                              *rules-test
   script:
     - cargo +nightly fmt --all -- --check
   allow_failure:                   true
@@ -239,11 +230,12 @@ check-no-default-features:
 spellcheck:
   stage:                           test
   <<:                              *docker-env
-  <<:                              *rules-pr-only
+  <<:                              *rules-test
   script:
     - cargo spellcheck --version
     # compare with the commit parent to the PR, given it's from a default branch
     - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
+    - echo "___Spellcheck is going to check your diff___"
     - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
     - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
         $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
@@ -254,11 +246,7 @@ build-adder-collator:
   <<:                              *collect-artifacts
   <<:                              *docker-env
   <<:                              *compiler-info
-  rules:
+  <<:                              *rules-test-and-rococo
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
   script:
     - time cargo build --release --verbose -p test-parachain-adder-collator
     - sccache -s
@@ -294,18 +282,19 @@ check-dependent-cumulus:
     DEPENDENT_REPO: cumulus
 
 check-transaction-versions:
-  # image must be ubuntu:20.04 based to match the linkers, this image has npm installed
-  image:                           paritytech/contracts-ci-linux:production
   stage:                           build
   <<:                              *rules-test
   <<:                              *docker-env
   <<:                              *vault-secrets
+  image:                           node:15
   needs:
     - job:                         test-build-linux-stable
       artifacts:                   true
   before_script:
     - apt-get -y update; apt-get -y install jq lsof
     - npm install --ignore-scripts -g @polkadot/metadata-cmp
+    # Set git config
+    - git config remote.origin.url "https://github.com/paritytech/polkadot.git"
     - git fetch origin release
   script:
     - scripts/gitlab/check_extrinsics_ordering.sh
@@ -334,7 +323,7 @@ build-rustdoc:
     - ./crate-docs/
   script:
     # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
-    - time cargo doc --no-deps --workspace --verbose
+    - time cargo doc --workspace --verbose
     - rm -f ./target/doc/.lock
     - mv ./target/doc ./crate-docs
     # FIXME: remove me after CI image gets nonroot
@@ -416,16 +405,12 @@ publish-adder-collator-image:
   # service image for Simnet
   stage:                           build
   <<:                              *build-push-image
+  <<:                              *rules-test-and-rococo
   variables:
     <<:                            *image-variables
     # scripts/dockerfiles/collator_injected.Dockerfile
     DOCKERFILE:                    dockerfiles/collator_injected.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/colander
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
   needs:
     - job:                         build-adder-collator
       artifacts:                   true
@@ -439,22 +424,20 @@ publish-adder-collator-image:
       # this artifact is used in simnet-tests job
       dotenv: ./artifacts/collator.env
 
-.update_weights:                   &update-weights
+update_polkadot_weights:           &update-weights
   stage:                           build
   when:                            manual
-  tags:
-    - weights
   variables:
     RUNTIME:                       polkadot
   artifacts:
     paths:
       - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
-  script: |
+  script:
-    ./scripts/run_benches_for_runtime.sh $RUNTIME
+    - ./scripts/run_benches_for_runtime.sh $RUNTIME
-    git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+    - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
-
+  # uses the "shell" executors
-update_polkadot_weights:
+  tags:
-  <<:                              *update-weights
+    - weights
 
 update_kusama_weights:
   <<:                              *update-weights
@@ -547,6 +530,7 @@ publish-rustdoc:
     - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
         echo "___Nothing to commit___"
     - git push origin gh-pages --force
+    - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___"
   after_script:
     - rm -rf .git/ ./*
 
@@ -555,7 +539,6 @@ publish-rustdoc:
 deploy-polkasync-kusama:
   stage:                           deploy
   rules:
-    # former .rules-build
     - if: $CI_COMMIT_REF_NAME == "rococo-v1"
       when: never
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -570,13 +553,9 @@ simnet-tests:
   stage:                           deploy
   image:                           "${SIMNET_IMAGE}"
   <<:                              *kubernetes-env
+  <<:                              *rules-test-and-rococo
   variables:
     GH_DIR:                        "https://github.com/paritytech/polkadot/tree/master/simnet_tests"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
   needs:
     - job:                         publish-polkadot-image
     - job:                         publish-adder-collator-image

polkadot/CONTRIBUTING.md

@@ -1,11 +1,5 @@
 # Contributing
 
-`Substrate` projects is a **OPENISH Open Source Project**
-
-## What?
-
-Individuals making significant and valuable contributions are given commit-access to a project to contribute as they see fit. A project is more like an open wiki than a standard guarded open source project.
-
 ## Rules
 
 There are a few basic ground-rules for contributors (including the maintainer(s) of the project):
@@ -16,25 +10,25 @@ There are a few basic ground-rules for contributors (including the maintainer(s)
 - A pull-request _must not be merged until CI_ has finished successfully.
 - Contributors should adhere to the [house coding style](https://github.com/paritytech/polkadot/wiki/Style-Guide).
 
-#### Merging pull requests once CI is successful:
+### Merging pull requests once CI is successful
 
-- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/substrate/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author.
+- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/polkadot/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author.
 - A pull request with no large change to logic that is an urgent fix may be merged after a non-author contributor has reviewed it well.
-- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview) tag in order to garner feedback.
+- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/polkadot/pulls?q=is:pr+is:open+label:A0-pleasereview) tag in order to garner feedback.
 - No PR should be merged until all reviews' comments are addressed.
 
-#### Reviewing pull requests:
+### Reviewing pull requests
 
 When reviewing a pull request, the end-goal is to suggest useful changes to the author. Reviews should finish with approval unless there are issues that would result in:
 
-- Buggy behaviour.
+- Buggy behavior.
 - Undue maintenance burden.
 - Breaking with house coding style.
-- Pessimisation (i.e. reduction of speed as measured in the projects benchmarks).
+- Pessimization (i.e. reduction of speed as measured in the projects benchmarks).
 - Feature reduction (i.e. it removes some aspect of functionality that a significant minority of users rely on).
 - Uselessness (i.e. it does not strictly add a feature or fix a known issue).
 
-#### Reviews may not be used as an effective veto for a PR because:
+### Reviews may not be used as an effective veto for a PR because
 
 - There exists a somewhat cleaner/better/faster way of accomplishing the same feature/fix.
 - It does not fit well with some other contributors' longer-term vision for the project.
@@ -49,4 +43,4 @@ This is an experiment and feedback is welcome! This document may also be subject
 
 ## Heritage
 
-These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: https://github.com/Level/community/blob/master/CONTRIBUTING.md
+These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: <https://github.com/Level/community/blob/master/CONTRIBUTING.md>

polkadot/README.md

@@ -33,7 +33,7 @@ the command-line.
 Currently supports Debian 10 (Buster) and Ubuntu 20.04 (Focal), and
 derivatives. Run the following commands as the `root` user.
 
-```
+```bash
 # Import the security@parity.io GPG key
 gpg --recv-keys --keyserver hkps://keys.mailvelope.com 9D4B2B6EB8F97156D19669A9FF0812D491B96798
 gpg --export 9D4B2B6EB8F97156D19669A9FF0812D491B96798 > /usr/share/keyrings/parity.gpg
@@ -52,7 +52,7 @@ apt install polkadot
 
 Currently supports Fedora 32 and CentOS 8, and derivatives.
 
-```
+```bash
 # Install dnf-plugins-core (This might already be installed)
 dnf install dnf-plugins-core
 # Add the repository and enable it
@@ -244,9 +244,11 @@ polkadot --chain=polkadot-local --bob -d /tmp/bob --port 30334 --bootnodes '/ip4
 Ensure you replace `ALICE_BOOTNODE_ID_HERE` with the node ID from the output of the first terminal.
 
 ### Using Docker
+
 [Using Docker](doc/docker.md)
 
 ### Shell Completion
+
 [Shell Completion](doc/shell-completion.md)
 
 ## Contributing

polkadot/doc/shell-completion.md

@@ -1,7 +1,6 @@
+# Shell completion
 
-## Shell completion
+The Polkadot CLI command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system.
-
-The Polkadot cli command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system.
 
 Assuming you built a release version using `cargo build --release` and use `bash` run the following:
 
@@ -19,7 +18,7 @@ You can find completion scripts for:
 
 To make this change persistent, you can proceed as follow:
 
-### First install
+## First install
 
 ```bash
 COMPL_DIR=$HOME/.completion
@@ -29,7 +28,7 @@ echo "source $COMPL_DIR/polkadot.bash" >> $HOME/.bash_profile
 source $HOME/.bash_profile
 ```
 
-### Update
+## Update
 
 When you build a new version of Polkadot, the following will ensure you auto-completion script matches the current binary:
 

polkadot/doc/testing.md

@@ -183,51 +183,50 @@ The implementation is yet to be completed, see the [implementation PR](https://g
 struct BehaveMaleficient;
 
 impl OverseerGen for BehaveMaleficient {
-	fn generate<'a, Spawner, RuntimeClient>(
+ fn generate<'a, Spawner, RuntimeClient>(
-		&self,
+  &self,
-		connector: OverseerConnector,
+  args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
-		args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
+ ) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandler), Error>
-	) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandler), Error>
+ where
-	where
+  RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
-		RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
+  RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
-		RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
+  Spawner: 'static + SpawnNamed + Clone + Unpin,
-		Spawner: 'static + SpawnNamed + Clone + Unpin,
+ {
-	{
+  let spawner = args.spawner.clone();
-		let spawner = args.spawner.clone();
+  let leaves = args.leaves.clone();
-		let leaves = args.leaves.clone();
+  let runtime_client = args.runtime_client.clone();
-		let runtime_client = args.runtime_client.clone();
+  let registry = args.registry.clone();
-		let registry = args.registry.clone();
+  let candidate_validation_config = args.candidate_validation_config.clone();
-		let candidate_validation_config = args.candidate_validation_config.clone();
+  // modify the subsystem(s) as needed:
-		// modify the subsystem(s) as needed:
+  let all_subsystems = create_default_subsystems(args)?.
-		let all_subsystems = create_default_subsystems(args)?.
         // or spawn an entirely new set
 
         replace_candidate_validation(
-			// create the filtered subsystem
+   // create the filtered subsystem
-			FilteredSubsystem::new(
+   FilteredSubsystem::new(
-				CandidateValidationSubsystem::with_config(
+    CandidateValidationSubsystem::with_config(
-					candidate_validation_config,
+     candidate_validation_config,
-					Metrics::register(registry)?,
+     Metrics::register(registry)?,
-				),
+    ),
                 // an implementation of
-				Skippy::default(),
+    Skippy::default(),
-			),
+   ),
-		);
+  );
 
-		Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner, connector)
+  Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner)
-			.map_err(|e| e.into())
+   .map_err(|e| e.into())
 
         // A builder pattern will simplify this further
         // WIP https://github.com/paritytech/polkadot/pull/2962
-	}
+ }
 }
 
 fn main() -> eyre::Result<()> {
-	color_eyre::install()?;
+ color_eyre::install()?;
-	let cli = Cli::from_args();
+ let cli = Cli::from_args();
-	assert_matches::assert_matches!(cli.subcommand, None);
+ assert_matches::assert_matches!(cli.subcommand, None);
-	polkadot_cli::run_node(cli, BehaveMaleficient)?;
+ polkadot_cli::run_node(cli, BehaveMaleficient)?;
-	Ok(())
+ Ok(())
 }
 ```
 

polkadot/scripts/gitlab/check_line_width.sh

@@ -1,51 +0,0 @@
-#!/bin/sh
-#
-# check if line width of rust source files is not beyond x characters
-#
-set -e
-
-BASE_BRANCH="origin/master"
-LINE_WIDTH="121"
-GOOD_LINE_WIDTH="101"
-
-git diff --name-only "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- \*.rs :^bridges | ( while read -r file
-do
-  if [ ! -f "${file}" ];
-  then
-	echo "Skipping removed file."
-  elif git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${LINE_WIDTH}\}"
-  then
-    if [ -z "${FAIL}" ]
-    then
-      echo "| warning!"
-      echo "| Lines should not be longer than 120 characters."
-      echo "| "
-      echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
-      echo "|"
-      FAIL="true"
-    fi
-    echo "| file: ${file}"
-    git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \
-      | grep -n "^+.\{${LINE_WIDTH}\}"
-    echo "|"
-  else
-    if git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${GOOD_LINE_WIDTH}\}"
-    then
-      if [ -z "${FAIL}" ]
-      then
-        echo "| warning!"
-        echo "| Lines should be longer than 100 characters only in exceptional circumstances!"
-        echo "| "
-        echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
-        echo "|"
-      fi
-      echo "| file: ${file}"
-      git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \
-        | grep -n "^+.\{${LINE_WIDTH}\}"
-      echo "|"
-    fi
-  fi
-done
-
-test -z "${FAIL}"
-)

polkadot/scripts/gitlab/lingua.dic

@@ -1,4 +1,5 @@
 150
+A&V
 accessor/MS
 AccountId
 activations
@@ -15,7 +16,6 @@ assignee/SM
 async
 asynchrony
 autogenerated
-A&V
 backable
 backend/MS
 benchmark/DSMG
@@ -47,6 +47,7 @@ customizable/B
 DDoS
 Debian/M
 decodable/MS
+decrement
 deduplicated
 deinitializing
 dequeue/SD
@@ -56,6 +57,7 @@ DHT
 disincentivize/D
 dispatchable/SM
 DLEQ
+DM
 DMP/SM
 DMQ
 DoS
@@ -78,6 +80,7 @@ fedora/M
 finalize/B
 FRAME/MS
 FSMs
+functor
 fungibility
 gameable
 getter/MS
@@ -156,21 +159,24 @@ mutex
 natively
 NFA
 NFT/SM
+no_std
 nonces
 NPoS
-no_std
 NTB
 offboard/DMSG
 onboard/DMSG
 oneshot/MS
 onwards
 OOM/S
+OPENISH
 others'
 ourself
 overseer/MS
 ownerless
+p2p
 parablock/MS
 parachain/MS
+ParaId
 parameterization
 parameterize/D
 parathread/MS
@@ -178,6 +184,7 @@ passthrough
 PDK
 peerset/MS
 permission/D
+pessimization
 phragmen
 picosecond/SM
 PoA/MS
@@ -201,7 +208,6 @@ proxy/G
 proxying
 PRs
 PVF/S
-p2p
 README/MS
 redhat/M
 register/CD
@@ -217,10 +223,9 @@ rpc
 RPC/MS
 runtime/MS
 rustc/MS
-sybil
 SAFT
-scalable
 scalability
+scalable
 Schnorr
 schnorrkel
 SDF
@@ -230,8 +235,8 @@ shareable
 Simnet/MS
 spawn/SR
 spawner
-SS58
 sr25519
+SS58
 SSL
 startup/MS
 stateful
@@ -243,18 +248,19 @@ subsystem/MS
 subsystems'
 supermajority
 SURI
+sybil
 systemwide
 taskmanager/MS
 TCP
 teleport/D
 teleport/RG
-teleports
 teleportation/SM
 teleporter/SM
 teleporters
+teleports
 testnet/MS
-timestamp/MS
 timeframe
+timestamp/MS
 tradeoff
 transitionary
 trie/MS
@@ -309,7 +315,4 @@ XCM/S
 XCMP/M
 yeet
 yml
-decrement
+zsh
-DM
-ParaId
-functor