CI: chore (#3957)

* CI: chore * CI: lsof and jq are installed * CI: generate docs with deps * CI: remove check width, fmt checks it * CI: simplify rules * CI: make CI image variable * CI: more chore * rm check_line_width.sh * CI: fix spellcheck; fix check-transaction * CI: return installing jq * lint: remove dupes
2026-05-30 21:01:02 +00:00 · 2021-11-11 17:38:05 +01:00
parent 62146d849d
commit 9d5724f926
7 changed files with 96 additions and 171 deletions
@@ -6,7 +6,7 @@
 # setting DEPLOY_TAG will only deploy the tagged image
 #
 # please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
-# &rules-pr-only preset. And "kubernetes-env" with "docker-env" to set a runner
+# &rules-test-and-rococo preset. And "kubernetes-env" with "docker-env" to set a runner
 # which executes the job.

 stages:
@@ -15,8 +15,6 @@ stages:
  - publish
  - deploy

-image:                             paritytech/ci-linux:production
-
 workflow:
  rules:
    - if: $CI_COMMIT_TAG
@@ -26,6 +24,7 @@ variables:
  GIT_STRATEGY:                    fetch
  GIT_DEPTH:                       100
  CI_SERVER_NAME:                  "GitLab CI"
+  CI_IMAGE:                        "paritytech/ci-linux:production"
  DOCKER_OS:                       "debian:stretch"
  ARCH:                            "x86_64"
  VAULT_SERVER_URL:                "https://vault.parity-mgmt-vault.parity.io"
@@ -57,6 +56,7 @@ default:
    - kubernetes-parity-build

 .docker-env:                       &docker-env
+  image:                           "${CI_IMAGE}"
  retry:
    max: 2
    when:
@@ -75,20 +75,20 @@ default:

 .rules-test:                       &rules-test
  # these jobs run always*
-  rules:
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
-      when: never
-    - when: always
-
-.pr-only:                          &rules-pr-only
-  # these jobs run only on PRs
  rules:
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: never
+    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs

+.rules-test-and-rococo:            &rules-test-and-rococo
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
+
 #### Vault secrets
 .vault-secrets:                    &vault-secrets
  secrets:
@@ -144,7 +144,7 @@ check-runtime:
  stage:                           test
  image:                           paritytech/tools:latest
  <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
+  <<:                              *rules-test
  <<:                              *vault-secrets
  variables:
    GITLAB_API:                    "https://gitlab.parity.io/api/v4"
@@ -153,19 +153,10 @@ check-runtime:
    - ./scripts/gitlab/check_runtime.sh
  allow_failure:                   true

-check-line-width:
-  stage:                           test
-  image:                           paritytech/tools:latest
-  <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
-  script:
-    - ./scripts/gitlab/check_line_width.sh
-  allow_failure:                   true
-
 cargo-fmt:
  stage:                           test
-  <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
+  <<:                              *docker-env
+  <<:                              *rules-test
  script:
    - cargo +nightly fmt --all -- --check
  allow_failure:                   true
@@ -239,11 +230,12 @@ check-no-default-features:
 spellcheck:
  stage:                           test
  <<:                              *docker-env
-  <<:                              *rules-pr-only
+  <<:                              *rules-test
  script:
    - cargo spellcheck --version
    # compare with the commit parent to the PR, given it's from a default branch
    - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
+    - echo "___Spellcheck is going to check your diff___"
    - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
    - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
        $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
@@ -254,11 +246,7 @@ build-adder-collator:
  <<:                              *collect-artifacts
  <<:                              *docker-env
  <<:                              *compiler-info
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
+  <<:                              *rules-test-and-rococo
  script:
    - time cargo build --release --verbose -p test-parachain-adder-collator
    - sccache -s
@@ -294,18 +282,19 @@ check-dependent-cumulus:
    DEPENDENT_REPO: cumulus

 check-transaction-versions:
-  # image must be ubuntu:20.04 based to match the linkers, this image has npm installed
-  image:                           paritytech/contracts-ci-linux:production
  stage:                           build
  <<:                              *rules-test
  <<:                              *docker-env
  <<:                              *vault-secrets
+  image:                           node:15
  needs:
    - job:                         test-build-linux-stable
      artifacts:                   true
  before_script:
    - apt-get -y update; apt-get -y install jq lsof
    - npm install --ignore-scripts -g @polkadot/metadata-cmp
+    # Set git config
+    - git config remote.origin.url "https://github.com/paritytech/polkadot.git"
    - git fetch origin release
  script:
    - scripts/gitlab/check_extrinsics_ordering.sh
@@ -334,7 +323,7 @@ build-rustdoc:
    - ./crate-docs/
  script:
    # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
-    - time cargo doc --no-deps --workspace --verbose
+    - time cargo doc --workspace --verbose
    - rm -f ./target/doc/.lock
    - mv ./target/doc ./crate-docs
    # FIXME: remove me after CI image gets nonroot
@@ -416,16 +405,12 @@ publish-adder-collator-image:
  # service image for Simnet
  stage:                           build
  <<:                              *build-push-image
+  <<:                              *rules-test-and-rococo
  variables:
    <<:                            *image-variables
    # scripts/dockerfiles/collator_injected.Dockerfile
    DOCKERFILE:                    dockerfiles/collator_injected.Dockerfile
    IMAGE_NAME:                    docker.io/paritypr/colander
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
  needs:
    - job:                         build-adder-collator
      artifacts:                   true
@@ -439,22 +424,20 @@ publish-adder-collator-image:
      # this artifact is used in simnet-tests job
      dotenv: ./artifacts/collator.env

-.update_weights:                   &update-weights
+update_polkadot_weights:           &update-weights
  stage:                           build
  when:                            manual
-  tags:
-    - weights
  variables:
    RUNTIME:                       polkadot
  artifacts:
    paths:
      - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
-  script: |
-    ./scripts/run_benches_for_runtime.sh $RUNTIME
-    git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
-
-update_polkadot_weights:
-  <<:                              *update-weights
+  script:
+    - ./scripts/run_benches_for_runtime.sh $RUNTIME
+    - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+  # uses the "shell" executors
+  tags:
+    - weights

 update_kusama_weights:
  <<:                              *update-weights
@@ -547,6 +530,7 @@ publish-rustdoc:
    - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
        echo "___Nothing to commit___"
    - git push origin gh-pages --force
+    - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___"
  after_script:
    - rm -rf .git/ ./*

@@ -555,7 +539,6 @@ publish-rustdoc:
 deploy-polkasync-kusama:
  stage:                           deploy
  rules:
-    # former .rules-build
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -570,13 +553,9 @@ simnet-tests:
  stage:                           deploy
  image:                           "${SIMNET_IMAGE}"
  <<:                              *kubernetes-env
+  <<:                              *rules-test-and-rococo
  variables:
    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/master/simnet_tests"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
  needs:
    - job:                         publish-polkadot-image
    - job:                         publish-adder-collator-image
@@ -1,11 +1,5 @@
 # Contributing

-`Substrate` projects is a **OPENISH Open Source Project**
-
-## What?
-
-Individuals making significant and valuable contributions are given commit-access to a project to contribute as they see fit. A project is more like an open wiki than a standard guarded open source project.
-
 ## Rules

 There are a few basic ground-rules for contributors (including the maintainer(s) of the project):
@@ -16,25 +10,25 @@ There are a few basic ground-rules for contributors (including the maintainer(s)
 - A pull-request _must not be merged until CI_ has finished successfully.
 - Contributors should adhere to the [house coding style](https://github.com/paritytech/polkadot/wiki/Style-Guide).

-#### Merging pull requests once CI is successful:
+### Merging pull requests once CI is successful

- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/substrate/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author.
+- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/polkadot/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author.
 - A pull request with no large change to logic that is an urgent fix may be merged after a non-author contributor has reviewed it well.
- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview) tag in order to garner feedback.
+- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/polkadot/pulls?q=is:pr+is:open+label:A0-pleasereview) tag in order to garner feedback.
 - No PR should be merged until all reviews' comments are addressed.

-#### Reviewing pull requests:
+### Reviewing pull requests

 When reviewing a pull request, the end-goal is to suggest useful changes to the author. Reviews should finish with approval unless there are issues that would result in:

- Buggy behaviour.
+- Buggy behavior.
 - Undue maintenance burden.
 - Breaking with house coding style.
- Pessimisation (i.e. reduction of speed as measured in the projects benchmarks).
+- Pessimization (i.e. reduction of speed as measured in the projects benchmarks).
 - Feature reduction (i.e. it removes some aspect of functionality that a significant minority of users rely on).
 - Uselessness (i.e. it does not strictly add a feature or fix a known issue).

-#### Reviews may not be used as an effective veto for a PR because:
+### Reviews may not be used as an effective veto for a PR because

 - There exists a somewhat cleaner/better/faster way of accomplishing the same feature/fix.
 - It does not fit well with some other contributors' longer-term vision for the project.
@@ -49,4 +43,4 @@ This is an experiment and feedback is welcome! This document may also be subject

 ## Heritage

-These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: https://github.com/Level/community/blob/master/CONTRIBUTING.md
+These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: <https://github.com/Level/community/blob/master/CONTRIBUTING.md>
@@ -33,7 +33,7 @@ the command-line.
 Currently supports Debian 10 (Buster) and Ubuntu 20.04 (Focal), and
 derivatives. Run the following commands as the `root` user.

-```
+```bash
 # Import the security@parity.io GPG key
 gpg --recv-keys --keyserver hkps://keys.mailvelope.com 9D4B2B6EB8F97156D19669A9FF0812D491B96798
 gpg --export 9D4B2B6EB8F97156D19669A9FF0812D491B96798 > /usr/share/keyrings/parity.gpg
@@ -52,7 +52,7 @@ apt install polkadot

 Currently supports Fedora 32 and CentOS 8, and derivatives.

-```
+```bash
 # Install dnf-plugins-core (This might already be installed)
 dnf install dnf-plugins-core
 # Add the repository and enable it
@@ -67,7 +67,7 @@ dnf install polkadot

 ### Install via Cargo

-Make sure you have the support software installed from the **Build from Source** section 
+Make sure you have the support software installed from the **Build from Source** section
 below this section.

 If you want to install Polkadot in your PATH, you can do so with with:
@@ -244,9 +244,11 @@ polkadot --chain=polkadot-local --bob -d /tmp/bob --port 30334 --bootnodes '/ip4
 Ensure you replace `ALICE_BOOTNODE_ID_HERE` with the node ID from the output of the first terminal.

 ### Using Docker
+
 [Using Docker](doc/docker.md)

 ### Shell Completion
+
 [Shell Completion](doc/shell-completion.md)

 ## Contributing
@@ -1,7 +1,6 @@
+# Shell completion

-## Shell completion
-
-The Polkadot cli command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system.
+The Polkadot CLI command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system.

 Assuming you built a release version using `cargo build --release` and use `bash` run the following:

@@ -19,7 +18,7 @@ You can find completion scripts for:

 To make this change persistent, you can proceed as follow:

-### First install
+## First install

 ```bash
 COMPL_DIR=$HOME/.completion
@@ -29,7 +28,7 @@ echo "source $COMPL_DIR/polkadot.bash" >> $HOME/.bash_profile
 source $HOME/.bash_profile
 ```

-### Update
+## Update

 When you build a new version of Polkadot, the following will ensure you auto-completion script matches the current binary:

@@ -183,51 +183,50 @@ The implementation is yet to be completed, see the [implementation PR](https://g
 struct BehaveMaleficient;

 impl OverseerGen for BehaveMaleficient {
-	fn generate<'a, Spawner, RuntimeClient>(
-		&self,
-		connector: OverseerConnector,
-		args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
-	) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandler), Error>
-	where
-		RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
-		RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
-		Spawner: 'static + SpawnNamed + Clone + Unpin,
-	{
-		let spawner = args.spawner.clone();
-		let leaves = args.leaves.clone();
-		let runtime_client = args.runtime_client.clone();
-		let registry = args.registry.clone();
-		let candidate_validation_config = args.candidate_validation_config.clone();
-		// modify the subsystem(s) as needed:
-		let all_subsystems = create_default_subsystems(args)?.
+ fn generate<'a, Spawner, RuntimeClient>(
+  &self,
+  args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
+ ) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandler), Error>
+ where
+  RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
+  RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
+  Spawner: 'static + SpawnNamed + Clone + Unpin,
+ {
+  let spawner = args.spawner.clone();
+  let leaves = args.leaves.clone();
+  let runtime_client = args.runtime_client.clone();
+  let registry = args.registry.clone();
+  let candidate_validation_config = args.candidate_validation_config.clone();
+  // modify the subsystem(s) as needed:
+  let all_subsystems = create_default_subsystems(args)?.
        // or spawn an entirely new set

        replace_candidate_validation(
-			// create the filtered subsystem
-			FilteredSubsystem::new(
-				CandidateValidationSubsystem::with_config(
-					candidate_validation_config,
-					Metrics::register(registry)?,
-				),
+   // create the filtered subsystem
+   FilteredSubsystem::new(
+    CandidateValidationSubsystem::with_config(
+     candidate_validation_config,
+     Metrics::register(registry)?,
+    ),
                // an implementation of
-				Skippy::default(),
-			),
-		);
+    Skippy::default(),
+   ),
+  );

-		Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner, connector)
-			.map_err(|e| e.into())
+  Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner)
+   .map_err(|e| e.into())

        // A builder pattern will simplify this further
        // WIP https://github.com/paritytech/polkadot/pull/2962
-	}
+ }
 }

 fn main() -> eyre::Result<()> {
-	color_eyre::install()?;
-	let cli = Cli::from_args();
-	assert_matches::assert_matches!(cli.subcommand, None);
-	polkadot_cli::run_node(cli, BehaveMaleficient)?;
-	Ok(())
+ color_eyre::install()?;
+ let cli = Cli::from_args();
+ assert_matches::assert_matches!(cli.subcommand, None);
+ polkadot_cli::run_node(cli, BehaveMaleficient)?;
+ Ok(())
 }
 ```

@@ -1,51 +0,0 @@
-#!/bin/sh
-#
-# check if line width of rust source files is not beyond x characters
-#
-set -e
-
-BASE_BRANCH="origin/master"
-LINE_WIDTH="121"
-GOOD_LINE_WIDTH="101"
-
-git diff --name-only "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- \*.rs :^bridges | ( while read -r file
-do
-  if [ ! -f "${file}" ];
-  then
-	echo "Skipping removed file."
-  elif git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${LINE_WIDTH}\}"
-  then
-    if [ -z "${FAIL}" ]
-    then
-      echo "| warning!"
-      echo "| Lines should not be longer than 120 characters."
-      echo "| "
-      echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
-      echo "|"
-      FAIL="true"
-    fi
-    echo "| file: ${file}"
-    git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \
-      | grep -n "^+.\{${LINE_WIDTH}\}"
-    echo "|"
-  else
-    if git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${GOOD_LINE_WIDTH}\}"
-    then
-      if [ -z "${FAIL}" ]
-      then
-        echo "| warning!"
-        echo "| Lines should be longer than 100 characters only in exceptional circumstances!"
-        echo "| "
-        echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
-        echo "|"
-      fi
-      echo "| file: ${file}"
-      git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \
-        | grep -n "^+.\{${LINE_WIDTH}\}"
-      echo "|"
-    fi
-  fi
-done
-
-test -z "${FAIL}"
-)
@@ -1,4 +1,5 @@
 150
+A&V
 accessor/MS
 AccountId
 activations
@@ -15,7 +16,6 @@ assignee/SM
 async
 asynchrony
 autogenerated
-A&V
 backable
 backend/MS
 benchmark/DSMG
@@ -47,6 +47,7 @@ customizable/B
 DDoS
 Debian/M
 decodable/MS
+decrement
 deduplicated
 deinitializing
 dequeue/SD
@@ -56,6 +57,7 @@ DHT
 disincentivize/D
 dispatchable/SM
 DLEQ
+DM
 DMP/SM
 DMQ
 DoS
@@ -78,6 +80,7 @@ fedora/M
 finalize/B
 FRAME/MS
 FSMs
+functor
 fungibility
 gameable
 getter/MS
@@ -156,21 +159,24 @@ mutex
 natively
 NFA
 NFT/SM
+no_std
 nonces
 NPoS
-no_std
 NTB
 offboard/DMSG
 onboard/DMSG
 oneshot/MS
 onwards
 OOM/S
+OPENISH
 others'
 ourself
 overseer/MS
 ownerless
+p2p
 parablock/MS
 parachain/MS
+ParaId
 parameterization
 parameterize/D
 parathread/MS
@@ -178,6 +184,7 @@ passthrough
 PDK
 peerset/MS
 permission/D
+pessimization
 phragmen
 picosecond/SM
 PoA/MS
@@ -201,7 +208,6 @@ proxy/G
 proxying
 PRs
 PVF/S
-p2p
 README/MS
 redhat/M
 register/CD
@@ -217,10 +223,9 @@ rpc
 RPC/MS
 runtime/MS
 rustc/MS
-sybil
 SAFT
-scalable
 scalability
+scalable
 Schnorr
 schnorrkel
 SDF
@@ -230,8 +235,8 @@ shareable
 Simnet/MS
 spawn/SR
 spawner
-SS58
 sr25519
+SS58
 SSL
 startup/MS
 stateful
@@ -243,18 +248,19 @@ subsystem/MS
 subsystems'
 supermajority
 SURI
+sybil
 systemwide
 taskmanager/MS
 TCP
 teleport/D
 teleport/RG
-teleports
 teleportation/SM
 teleporter/SM
 teleporters
+teleports
 testnet/MS
-timestamp/MS
 timeframe
+timestamp/MS
 tradeoff
 transitionary
 trie/MS
@@ -309,7 +315,4 @@ XCM/S
 XCMP/M
 yeet
 yml
-decrement
-DM
-ParaId
-functor
+zsh