From 9d5724f926226d5f6207986631780e225436747c Mon Sep 17 00:00:00 2001 From: Denis Pisarev Date: Thu, 11 Nov 2021 17:38:05 +0100 Subject: [PATCH] CI: chore (#3957) * CI: chore * CI: lsof and jq are installed * CI: generate docs with deps * CI: remove check width, fmt checks it * CI: simplify rules * CI: make CI image variable * CI: more chore * rm check_line_width.sh * CI: fix spellcheck; fix check-transaction * CI: return installing jq * lint: remove dupes --- polkadot/.gitlab-ci.yml | 83 ++++++++------------- polkadot/CONTRIBUTING.md | 22 ++---- polkadot/README.md | 8 +- polkadot/doc/shell-completion.md | 9 +-- polkadot/doc/testing.md | 67 ++++++++--------- polkadot/scripts/gitlab/check_line_width.sh | 51 ------------- polkadot/scripts/gitlab/lingua.dic | 27 ++++--- 7 files changed, 96 insertions(+), 171 deletions(-) delete mode 100755 polkadot/scripts/gitlab/check_line_width.sh diff --git a/polkadot/.gitlab-ci.yml b/polkadot/.gitlab-ci.yml index 92f1fbdce2..0c810359b7 100644 --- a/polkadot/.gitlab-ci.yml +++ b/polkadot/.gitlab-ci.yml @@ -6,7 +6,7 @@ # setting DEPLOY_TAG will only deploy the tagged image # # please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything, -# &rules-pr-only preset. And "kubernetes-env" with "docker-env" to set a runner +# &rules-test-and-rococo preset. And "kubernetes-env" with "docker-env" to set a runner # which executes the job. stages: @@ -15,8 +15,6 @@ stages: - publish - deploy -image: paritytech/ci-linux:production - workflow: rules: - if: $CI_COMMIT_TAG @@ -26,6 +24,7 @@ variables: GIT_STRATEGY: fetch GIT_DEPTH: 100 CI_SERVER_NAME: "GitLab CI" + CI_IMAGE: "paritytech/ci-linux:production" DOCKER_OS: "debian:stretch" ARCH: "x86_64" VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io" @@ -57,6 +56,7 @@ default: - kubernetes-parity-build .docker-env: &docker-env + image: "${CI_IMAGE}" retry: max: 2 when: @@ -75,20 +75,20 @@ default: .rules-test: &rules-test # these jobs run always* - rules: - - if: $CI_COMMIT_REF_NAME == "rococo-v1" - when: never - - when: always - -.pr-only: &rules-pr-only - # these jobs run only on PRs rules: - if: $CI_COMMIT_REF_NAME == "rococo-v1" when: never - if: $CI_PIPELINE_SOURCE == "schedule" - when: never + - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs +.rules-test-and-rococo: &rules-test-and-rococo + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME == "rococo-v1" + #### Vault secrets .vault-secrets: &vault-secrets secrets: @@ -144,7 +144,7 @@ check-runtime: stage: test image: paritytech/tools:latest <<: *kubernetes-env - <<: *rules-pr-only + <<: *rules-test <<: *vault-secrets variables: GITLAB_API: "https://gitlab.parity.io/api/v4" @@ -153,19 +153,10 @@ check-runtime: - ./scripts/gitlab/check_runtime.sh allow_failure: true -check-line-width: - stage: test - image: paritytech/tools:latest - <<: *kubernetes-env - <<: *rules-pr-only - script: - - ./scripts/gitlab/check_line_width.sh - allow_failure: true - cargo-fmt: stage: test - <<: *kubernetes-env - <<: *rules-pr-only + <<: *docker-env + <<: *rules-test script: - cargo +nightly fmt --all -- --check allow_failure: true @@ -239,11 +230,12 @@ check-no-default-features: spellcheck: stage: test <<: *docker-env - <<: *rules-pr-only + <<: *rules-test script: - cargo spellcheck --version # compare with the commit parent to the PR, given it's from a default branch - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} + - echo "___Spellcheck is going to check your diff___" - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1 $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) @@ -254,11 +246,7 @@ build-adder-collator: <<: *collect-artifacts <<: *docker-env <<: *compiler-info - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - - if: $CI_COMMIT_REF_NAME == "rococo-v1" + <<: *rules-test-and-rococo script: - time cargo build --release --verbose -p test-parachain-adder-collator - sccache -s @@ -294,18 +282,19 @@ check-dependent-cumulus: DEPENDENT_REPO: cumulus check-transaction-versions: - # image must be ubuntu:20.04 based to match the linkers, this image has npm installed - image: paritytech/contracts-ci-linux:production stage: build <<: *rules-test <<: *docker-env <<: *vault-secrets + image: node:15 needs: - job: test-build-linux-stable artifacts: true before_script: - apt-get -y update; apt-get -y install jq lsof - npm install --ignore-scripts -g @polkadot/metadata-cmp + # Set git config + - git config remote.origin.url "https://github.com/paritytech/polkadot.git" - git fetch origin release script: - scripts/gitlab/check_extrinsics_ordering.sh @@ -334,7 +323,7 @@ build-rustdoc: - ./crate-docs/ script: # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` - - time cargo doc --no-deps --workspace --verbose + - time cargo doc --workspace --verbose - rm -f ./target/doc/.lock - mv ./target/doc ./crate-docs # FIXME: remove me after CI image gets nonroot @@ -416,16 +405,12 @@ publish-adder-collator-image: # service image for Simnet stage: build <<: *build-push-image + <<: *rules-test-and-rococo variables: <<: *image-variables # scripts/dockerfiles/collator_injected.Dockerfile DOCKERFILE: dockerfiles/collator_injected.Dockerfile IMAGE_NAME: docker.io/paritypr/colander - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - - if: $CI_COMMIT_REF_NAME == "rococo-v1" needs: - job: build-adder-collator artifacts: true @@ -439,22 +424,20 @@ publish-adder-collator-image: # this artifact is used in simnet-tests job dotenv: ./artifacts/collator.env -.update_weights: &update-weights +update_polkadot_weights: &update-weights stage: build when: manual - tags: - - weights variables: RUNTIME: polkadot artifacts: paths: - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - script: | - ./scripts/run_benches_for_runtime.sh $RUNTIME - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - -update_polkadot_weights: - <<: *update-weights + script: + - ./scripts/run_benches_for_runtime.sh $RUNTIME + - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch + # uses the "shell" executors + tags: + - weights update_kusama_weights: <<: *update-weights @@ -547,6 +530,7 @@ publish-rustdoc: - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || echo "___Nothing to commit___" - git push origin gh-pages --force + - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" after_script: - rm -rf .git/ ./* @@ -555,7 +539,6 @@ publish-rustdoc: deploy-polkasync-kusama: stage: deploy rules: - # former .rules-build - if: $CI_COMMIT_REF_NAME == "rococo-v1" when: never - if: $CI_PIPELINE_SOURCE == "schedule" @@ -570,13 +553,9 @@ simnet-tests: stage: deploy image: "${SIMNET_IMAGE}" <<: *kubernetes-env + <<: *rules-test-and-rococo variables: GH_DIR: "https://github.com/paritytech/polkadot/tree/master/simnet_tests" - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - - if: $CI_COMMIT_REF_NAME == "rococo-v1" needs: - job: publish-polkadot-image - job: publish-adder-collator-image diff --git a/polkadot/CONTRIBUTING.md b/polkadot/CONTRIBUTING.md index 47ae56b90c..47c14240e7 100644 --- a/polkadot/CONTRIBUTING.md +++ b/polkadot/CONTRIBUTING.md @@ -1,11 +1,5 @@ # Contributing -`Substrate` projects is a **OPENISH Open Source Project** - -## What? - -Individuals making significant and valuable contributions are given commit-access to a project to contribute as they see fit. A project is more like an open wiki than a standard guarded open source project. - ## Rules There are a few basic ground-rules for contributors (including the maintainer(s) of the project): @@ -16,25 +10,25 @@ There are a few basic ground-rules for contributors (including the maintainer(s) - A pull-request _must not be merged until CI_ has finished successfully. - Contributors should adhere to the [house coding style](https://github.com/paritytech/polkadot/wiki/Style-Guide). -#### Merging pull requests once CI is successful: +### Merging pull requests once CI is successful -- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/substrate/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author. +- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/polkadot/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author. - A pull request with no large change to logic that is an urgent fix may be merged after a non-author contributor has reviewed it well. -- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview) tag in order to garner feedback. +- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/polkadot/pulls?q=is:pr+is:open+label:A0-pleasereview) tag in order to garner feedback. - No PR should be merged until all reviews' comments are addressed. -#### Reviewing pull requests: +### Reviewing pull requests When reviewing a pull request, the end-goal is to suggest useful changes to the author. Reviews should finish with approval unless there are issues that would result in: -- Buggy behaviour. +- Buggy behavior. - Undue maintenance burden. - Breaking with house coding style. -- Pessimisation (i.e. reduction of speed as measured in the projects benchmarks). +- Pessimization (i.e. reduction of speed as measured in the projects benchmarks). - Feature reduction (i.e. it removes some aspect of functionality that a significant minority of users rely on). - Uselessness (i.e. it does not strictly add a feature or fix a known issue). -#### Reviews may not be used as an effective veto for a PR because: +### Reviews may not be used as an effective veto for a PR because - There exists a somewhat cleaner/better/faster way of accomplishing the same feature/fix. - It does not fit well with some other contributors' longer-term vision for the project. @@ -49,4 +43,4 @@ This is an experiment and feedback is welcome! This document may also be subject ## Heritage -These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: https://github.com/Level/community/blob/master/CONTRIBUTING.md +These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: diff --git a/polkadot/README.md b/polkadot/README.md index 5ea7ac4335..34e3a7bd75 100644 --- a/polkadot/README.md +++ b/polkadot/README.md @@ -33,7 +33,7 @@ the command-line. Currently supports Debian 10 (Buster) and Ubuntu 20.04 (Focal), and derivatives. Run the following commands as the `root` user. -``` +```bash # Import the security@parity.io GPG key gpg --recv-keys --keyserver hkps://keys.mailvelope.com 9D4B2B6EB8F97156D19669A9FF0812D491B96798 gpg --export 9D4B2B6EB8F97156D19669A9FF0812D491B96798 > /usr/share/keyrings/parity.gpg @@ -52,7 +52,7 @@ apt install polkadot Currently supports Fedora 32 and CentOS 8, and derivatives. -``` +```bash # Install dnf-plugins-core (This might already be installed) dnf install dnf-plugins-core # Add the repository and enable it @@ -67,7 +67,7 @@ dnf install polkadot ### Install via Cargo -Make sure you have the support software installed from the **Build from Source** section +Make sure you have the support software installed from the **Build from Source** section below this section. If you want to install Polkadot in your PATH, you can do so with with: @@ -244,9 +244,11 @@ polkadot --chain=polkadot-local --bob -d /tmp/bob --port 30334 --bootnodes '/ip4 Ensure you replace `ALICE_BOOTNODE_ID_HERE` with the node ID from the output of the first terminal. ### Using Docker + [Using Docker](doc/docker.md) ### Shell Completion + [Shell Completion](doc/shell-completion.md) ## Contributing diff --git a/polkadot/doc/shell-completion.md b/polkadot/doc/shell-completion.md index 986609392e..9c53cf43a1 100644 --- a/polkadot/doc/shell-completion.md +++ b/polkadot/doc/shell-completion.md @@ -1,7 +1,6 @@ +# Shell completion -## Shell completion - -The Polkadot cli command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system. +The Polkadot CLI command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system. Assuming you built a release version using `cargo build --release` and use `bash` run the following: @@ -19,7 +18,7 @@ You can find completion scripts for: To make this change persistent, you can proceed as follow: -### First install +## First install ```bash COMPL_DIR=$HOME/.completion @@ -29,7 +28,7 @@ echo "source $COMPL_DIR/polkadot.bash" >> $HOME/.bash_profile source $HOME/.bash_profile ``` -### Update +## Update When you build a new version of Polkadot, the following will ensure you auto-completion script matches the current binary: diff --git a/polkadot/doc/testing.md b/polkadot/doc/testing.md index 6709d1b9bd..363085ab46 100644 --- a/polkadot/doc/testing.md +++ b/polkadot/doc/testing.md @@ -183,51 +183,50 @@ The implementation is yet to be completed, see the [implementation PR](https://g struct BehaveMaleficient; impl OverseerGen for BehaveMaleficient { - fn generate<'a, Spawner, RuntimeClient>( - &self, - connector: OverseerConnector, - args: OverseerGenArgs<'a, Spawner, RuntimeClient>, - ) -> Result<(Overseer>, OverseerHandler), Error> - where - RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, - RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Spawner: 'static + SpawnNamed + Clone + Unpin, - { - let spawner = args.spawner.clone(); - let leaves = args.leaves.clone(); - let runtime_client = args.runtime_client.clone(); - let registry = args.registry.clone(); - let candidate_validation_config = args.candidate_validation_config.clone(); - // modify the subsystem(s) as needed: - let all_subsystems = create_default_subsystems(args)?. + fn generate<'a, Spawner, RuntimeClient>( + &self, + args: OverseerGenArgs<'a, Spawner, RuntimeClient>, + ) -> Result<(Overseer>, OverseerHandler), Error> + where + RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, + RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, + Spawner: 'static + SpawnNamed + Clone + Unpin, + { + let spawner = args.spawner.clone(); + let leaves = args.leaves.clone(); + let runtime_client = args.runtime_client.clone(); + let registry = args.registry.clone(); + let candidate_validation_config = args.candidate_validation_config.clone(); + // modify the subsystem(s) as needed: + let all_subsystems = create_default_subsystems(args)?. // or spawn an entirely new set replace_candidate_validation( - // create the filtered subsystem - FilteredSubsystem::new( - CandidateValidationSubsystem::with_config( - candidate_validation_config, - Metrics::register(registry)?, - ), + // create the filtered subsystem + FilteredSubsystem::new( + CandidateValidationSubsystem::with_config( + candidate_validation_config, + Metrics::register(registry)?, + ), // an implementation of - Skippy::default(), - ), - ); + Skippy::default(), + ), + ); - Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner, connector) - .map_err(|e| e.into()) + Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner) + .map_err(|e| e.into()) // A builder pattern will simplify this further // WIP https://github.com/paritytech/polkadot/pull/2962 - } + } } fn main() -> eyre::Result<()> { - color_eyre::install()?; - let cli = Cli::from_args(); - assert_matches::assert_matches!(cli.subcommand, None); - polkadot_cli::run_node(cli, BehaveMaleficient)?; - Ok(()) + color_eyre::install()?; + let cli = Cli::from_args(); + assert_matches::assert_matches!(cli.subcommand, None); + polkadot_cli::run_node(cli, BehaveMaleficient)?; + Ok(()) } ``` diff --git a/polkadot/scripts/gitlab/check_line_width.sh b/polkadot/scripts/gitlab/check_line_width.sh deleted file mode 100755 index b147488d92..0000000000 --- a/polkadot/scripts/gitlab/check_line_width.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# -# check if line width of rust source files is not beyond x characters -# -set -e - -BASE_BRANCH="origin/master" -LINE_WIDTH="121" -GOOD_LINE_WIDTH="101" - -git diff --name-only "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- \*.rs :^bridges | ( while read -r file -do - if [ ! -f "${file}" ]; - then - echo "Skipping removed file." - elif git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${LINE_WIDTH}\}" - then - if [ -z "${FAIL}" ] - then - echo "| warning!" - echo "| Lines should not be longer than 120 characters." - echo "| " - echo "| see more https://wiki.parity.io/Substrate-Style-Guide" - echo "|" - FAIL="true" - fi - echo "| file: ${file}" - git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \ - | grep -n "^+.\{${LINE_WIDTH}\}" - echo "|" - else - if git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${GOOD_LINE_WIDTH}\}" - then - if [ -z "${FAIL}" ] - then - echo "| warning!" - echo "| Lines should be longer than 100 characters only in exceptional circumstances!" - echo "| " - echo "| see more https://wiki.parity.io/Substrate-Style-Guide" - echo "|" - fi - echo "| file: ${file}" - git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \ - | grep -n "^+.\{${LINE_WIDTH}\}" - echo "|" - fi - fi -done - -test -z "${FAIL}" -) diff --git a/polkadot/scripts/gitlab/lingua.dic b/polkadot/scripts/gitlab/lingua.dic index 3d65a40c81..9c5a29d8ba 100644 --- a/polkadot/scripts/gitlab/lingua.dic +++ b/polkadot/scripts/gitlab/lingua.dic @@ -1,4 +1,5 @@ 150 +A&V accessor/MS AccountId activations @@ -15,7 +16,6 @@ assignee/SM async asynchrony autogenerated -A&V backable backend/MS benchmark/DSMG @@ -47,6 +47,7 @@ customizable/B DDoS Debian/M decodable/MS +decrement deduplicated deinitializing dequeue/SD @@ -56,6 +57,7 @@ DHT disincentivize/D dispatchable/SM DLEQ +DM DMP/SM DMQ DoS @@ -78,6 +80,7 @@ fedora/M finalize/B FRAME/MS FSMs +functor fungibility gameable getter/MS @@ -156,21 +159,24 @@ mutex natively NFA NFT/SM +no_std nonces NPoS -no_std NTB offboard/DMSG onboard/DMSG oneshot/MS onwards OOM/S +OPENISH others' ourself overseer/MS ownerless +p2p parablock/MS parachain/MS +ParaId parameterization parameterize/D parathread/MS @@ -178,6 +184,7 @@ passthrough PDK peerset/MS permission/D +pessimization phragmen picosecond/SM PoA/MS @@ -201,7 +208,6 @@ proxy/G proxying PRs PVF/S -p2p README/MS redhat/M register/CD @@ -217,10 +223,9 @@ rpc RPC/MS runtime/MS rustc/MS -sybil SAFT -scalable scalability +scalable Schnorr schnorrkel SDF @@ -230,8 +235,8 @@ shareable Simnet/MS spawn/SR spawner -SS58 sr25519 +SS58 SSL startup/MS stateful @@ -243,18 +248,19 @@ subsystem/MS subsystems' supermajority SURI +sybil systemwide taskmanager/MS TCP teleport/D teleport/RG -teleports teleportation/SM teleporter/SM teleporters +teleports testnet/MS -timestamp/MS timeframe +timestamp/MS tradeoff transitionary trie/MS @@ -309,7 +315,4 @@ XCM/S XCMP/M yeet yml -decrement -DM -ParaId -functor +zsh