pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-07-21 17:55:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduce rusty-cachier (#11462)

Browse Source 
* Introduce `rusty-cachier`

* Return LF at the end of file

* Use `entrypoint` to `unshare(1)` into a new mount namespace

* Use `rusty-cachier`-provided absolute path for `CARGO_TARGET_DIR` everywhere

* Debug single `build-rustdoc` job

* CI: debug

* CI: debug

* CI: debug

* `unshare(1)` is no longer needed

* CI: remove debug

* Revert "Debug single `build-rustdoc` job"

* Formatiing

* Update scripts/ci/gitlab/pipeline/build.yml

Co-authored-by: Denis Pisarev <denis.pisarev@parity.io>
This commit is contained in:
Vlad
2022-05-31 18:10:04 +03:00
committed by GitHub
parent c193fa9a60
commit c91c1c793e
3 changed files with 95 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/.gitlab-ci.yml
+14
View File
@@ -47,6 +47,8 @@ variables: &default-vars
DOCKER_OS: "debian:stretch" DOCKER_OS: "debian:stretch"
ARCH: "x86_64" ARCH: "x86_64"
CI_IMAGE: "paritytech/ci-linux:production" CI_IMAGE: "paritytech/ci-linux:production"
RUSTY_CACHIER_SINGLE_BRANCH: master
RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true"
default: default:
retry: retry:
@@ -86,10 +88,22 @@ default:
- rustup +nightly show - rustup +nightly show
- cargo +nightly --version - cargo +nightly --version
.rusty-cachier:
before_script:
- curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
- rusty-cachier environment check --gracefully
- $(rusty-cachier environment inject)
- rusty-cachier project mtime
after_script:
- rusty-cachier snapshot destroy
.docker-env: .docker-env:
image: "${CI_IMAGE}" image: "${CI_IMAGE}"
before_script: before_script:
- !reference [.rust-info-script, script] - !reference [.rust-info-script, script]
- !reference [.rusty-cachier, before_script]
after_script:
- !reference [.rusty-cachier, after_script]
tags: tags:
- linux-docker - linux-docker
substrate/scripts/ci/gitlab/pipeline/build.yml
+33 -9
View File
@@ -55,9 +55,11 @@ build-linux-substrate:
artifacts: false artifacts: false
before_script: before_script:
- mkdir -p ./artifacts/substrate/ - mkdir -p ./artifacts/substrate/
- !reference [.rusty-cachier, before_script]
script: script:
- rusty-cachier snapshot create
- WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
- mv ./target/release/substrate ./artifacts/substrate/. - mv /cargo_target_dir/release/substrate ./artifacts/substrate/.
- echo -n "Substrate version = " - echo -n "Substrate version = "
- if [ "${CI_COMMIT_TAG}" ]; then - if [ "${CI_COMMIT_TAG}" ]; then
echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
@@ -69,6 +71,7 @@ build-linux-substrate:
- cp -r ./scripts/ci/docker/substrate.Dockerfile ./artifacts/substrate/ - cp -r ./scripts/ci/docker/substrate.Dockerfile ./artifacts/substrate/
- printf '\n# building node-template\n\n' - printf '\n# building node-template\n\n'
- ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
- rusty-cachier cache upload
.build-subkey: .build-subkey:
stage: build stage: build
@@ -79,6 +82,30 @@ build-linux-substrate:
needs: needs:
- job: cargo-check-subkey - job: cargo-check-subkey
artifacts: false artifacts: false
before_script:
- mkdir -p ./artifacts/subkey
- !reference [.rusty-cachier, before_script]
script:
- rusty-cachier snapshot create
- cd ./bin/utils/subkey
- SKIP_WASM_BUILD=1 time cargo build --release --verbose
- cd -
- mv /cargo_target_dir/release/subkey ./artifacts/subkey/.
- echo -n "Subkey version = "
- ./artifacts/subkey/subkey --version |
sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
tee ./artifacts/subkey/VERSION;
- sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
- cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
- rusty-cachier cache upload
build-subkey-linux:
extends: .build-subkey
build-subkey-macos:
extends: .build-subkey
# duplicating before_script & script sections from .build-subkey hidden job
# to overwrite rusty-cachier integration as it doesn't work on macos
before_script: before_script:
- mkdir -p ./artifacts/subkey - mkdir -p ./artifacts/subkey
script: script:
@@ -92,12 +119,6 @@ build-linux-substrate:
tee ./artifacts/subkey/VERSION; tee ./artifacts/subkey/VERSION;
- sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
- cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/ - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
build-subkey-linux:
extends: .build-subkey
build-subkey-macos:
extends: .build-subkey
tags: tags:
- osx - osx
@@ -109,6 +130,7 @@ build-rustdoc:
variables: variables:
SKIP_WASM_BUILD: 1 SKIP_WASM_BUILD: 1
DOC_INDEX_PAGE: "sc_service/index.html" # default redirected page DOC_INDEX_PAGE: "sc_service/index.html" # default redirected page
RUSTY_CACHIER_TOOLCHAIN: nightly
artifacts: artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
when: on_success when: on_success
@@ -116,9 +138,11 @@ build-rustdoc:
paths: paths:
- ./crate-docs/ - ./crate-docs/
script: script:
- rusty-cachier snapshot create
- time cargo +nightly doc --workspace --all-features --verbose - time cargo +nightly doc --workspace --all-features --verbose
- rm -f ./target/doc/.lock - rm -f /cargo_target_dir/doc/.lock
- mv ./target/doc ./crate-docs - mv /cargo_target_dir/doc ./crate-docs
# FIXME: remove me after CI image gets nonroot # FIXME: remove me after CI image gets nonroot
- chown -R nonroot:nonroot ./crate-docs - chown -R nonroot:nonroot ./crate-docs
- echo "<meta http-equiv=refresh content=0;url=${DOC_INDEX_PAGE}>" > ./crate-docs/index.html - echo "<meta http-equiv=refresh content=0;url=${DOC_INDEX_PAGE}>" > ./crate-docs/index.html
- rusty-cachier cache upload
substrate/scripts/ci/gitlab/pipeline/test.yml
+48 -1
View File
@@ -7,8 +7,11 @@ cargo-deny:
- .docker-env - .docker-env
- .nightly-pipeline - .nightly-pipeline
script: script:
- rusty-cachier snapshot create
- cargo deny check --hide-inclusion-graph -c ./scripts/ci/deny.toml - cargo deny check --hide-inclusion-graph -c ./scripts/ci/deny.toml
- rusty-cachier cache upload
after_script: after_script:
- !reference [.rusty-cachier, after_script]
- echo "___The complete log is in the artifacts___" - echo "___The complete log is in the artifacts___"
- cargo deny check -c ./scripts/ci/deny.toml 2> deny.log - cargo deny check -c ./scripts/ci/deny.toml 2> deny.log
artifacts: artifacts:
@@ -22,11 +25,15 @@ cargo-deny:
cargo-fmt: cargo-fmt:
stage: test stage: test
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends: extends:
- .docker-env - .docker-env
- .test-refs - .test-refs
script: script:
- rusty-cachier snapshot create
- cargo +nightly fmt --all -- --check - cargo +nightly fmt --all -- --check
- rusty-cachier cache upload
cargo-clippy: cargo-clippy:
stage: test stage: test
@@ -34,11 +41,15 @@ cargo-clippy:
needs: needs:
- job: cargo-fmt - job: cargo-fmt
artifacts: false artifacts: false
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends: extends:
- .docker-env - .docker-env
- .test-refs - .test-refs
script: script:
- rusty-cachier snapshot create
- SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy --all-targets - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy --all-targets
- rusty-cachier cache upload
cargo-check-nixos: cargo-check-nixos:
stage: test stage: test
@@ -62,6 +73,8 @@ cargo-check-nixos:
cargo-check-benches: cargo-check-benches:
stage: test stage: test
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends: extends:
- .docker-env - .docker-env
- .test-refs - .test-refs
@@ -76,7 +89,9 @@ cargo-check-benches:
git merge $CI_COMMIT_REF_NAME --verbose --no-edit; git merge $CI_COMMIT_REF_NAME --verbose --no-edit;
fi fi
- !reference [.rust-info-script, script] - !reference [.rust-info-script, script]
- !reference [.rusty-cachier, before_script]
script: script:
- rusty-cachier snapshot create
- mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
- SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
- 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json - 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
@@ -84,6 +99,7 @@ cargo-check-benches:
- 'cargo run --release -p node-bench -- ::trie::read::small --json - 'cargo run --release -p node-bench -- ::trie::read::small --json
| tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json' | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
- sccache -s - sccache -s
- rusty-cachier cache upload
tags: tags:
- linux-docker-benches - linux-docker-benches
@@ -120,8 +136,10 @@ cargo-check-subkey:
- .docker-env - .docker-env
- .test-refs - .test-refs
script: script:
- rusty-cachier snapshot create
- cd ./bin/utils/subkey - cd ./bin/utils/subkey
- SKIP_WASM_BUILD=1 time cargo check --release - SKIP_WASM_BUILD=1 time cargo check --release
- rusty-cachier cache upload
cargo-check-try-runtime: cargo-check-try-runtime:
stage: test stage: test
@@ -133,7 +151,9 @@ cargo-check-try-runtime:
- .docker-env - .docker-env
- .test-refs - .test-refs
script: script:
- rusty-cachier snapshot create
- time cargo check --features try-runtime - time cargo check --features try-runtime
- rusty-cachier cache upload
cargo-check-wasmer-sandbox: cargo-check-wasmer-sandbox:
stage: test stage: test
@@ -145,7 +165,9 @@ cargo-check-wasmer-sandbox:
- .docker-env - .docker-env
- .test-refs - .test-refs
script: script:
- rusty-cachier snapshot create
- time cargo check --features wasmer-sandbox - time cargo check --features wasmer-sandbox
- rusty-cachier cache upload
test-deterministic-wasm: test-deterministic-wasm:
stage: test stage: test
@@ -159,16 +181,18 @@ test-deterministic-wasm:
variables: variables:
WASM_BUILD_NO_COLOR: 1 WASM_BUILD_NO_COLOR: 1
script: script:
- rusty-cachier snapshot create
# build runtime # build runtime
- cargo build --verbose --release -p node-runtime - cargo build --verbose --release -p node-runtime
# make checksum # make checksum
- sha256sum ./target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256 - sha256sum /cargo_target_dir/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
# clean up – FIXME: can we reuse some of the artifacts? # clean up – FIXME: can we reuse some of the artifacts?
- cargo clean - cargo clean
# build again # build again
- cargo build --verbose --release -p node-runtime - cargo build --verbose --release -p node-runtime
# confirm checksum # confirm checksum
- sha256sum -c ./checksum.sha256 - sha256sum -c ./checksum.sha256
- rusty-cachier cache upload
test-linux-stable: test-linux-stable:
stage: test stage: test
@@ -184,14 +208,18 @@ test-linux-stable:
# Ensure we run the UI tests. # Ensure we run the UI tests.
RUN_UI_TESTS: 1 RUN_UI_TESTS: 1
script: script:
- rusty-cachier snapshot create
# this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
- time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml
- time cargo test -p frame-support-test --features=conditional-storage,no-metadata-docs --manifest-path ./frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec - time cargo test -p frame-support-test --features=conditional-storage,no-metadata-docs --manifest-path ./frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec
- SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
- rusty-cachier cache upload
test-frame-examples-compile-to-wasm: test-frame-examples-compile-to-wasm:
# into one job # into one job
stage: test stage: test
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends: extends:
- .docker-env - .docker-env
- .test-refs - .test-refs
@@ -201,10 +229,12 @@ test-frame-examples-compile-to-wasm:
RUSTFLAGS: "-Cdebug-assertions=y" RUSTFLAGS: "-Cdebug-assertions=y"
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
script: script:
- rusty-cachier snapshot create
- cd ./frame/examples/offchain-worker/ - cd ./frame/examples/offchain-worker/
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
- cd ../basic - cd ../basic
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
- rusty-cachier cache upload
test-linux-stable-int: test-linux-stable-int:
stage: test stage: test
@@ -220,13 +250,16 @@ test-linux-stable-int:
# Ensure we run the UI tests. # Ensure we run the UI tests.
RUN_UI_TESTS: 1 RUN_UI_TESTS: 1
script: script:
- rusty-cachier snapshot create
- echo "___Logs will be partly shown at the end in case of failure.___" - echo "___Logs will be partly shown at the end in case of failure.___"
- echo "___Full log will be saved to the job artifacts only in case of failure.___" - echo "___Full log will be saved to the job artifacts only in case of failure.___"
- WASM_BUILD_NO_COLOR=1 - WASM_BUILD_NO_COLOR=1
RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
time cargo test -p node-cli --release --verbose --locked -- --ignored time cargo test -p node-cli --release --verbose --locked -- --ignored
&> ${CI_COMMIT_SHORT_SHA}_int_failure.log &> ${CI_COMMIT_SHORT_SHA}_int_failure.log
- rusty-cachier cache upload
after_script: after_script:
- !reference [.rusty-cachier, after_script]
- awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
artifacts: artifacts:
name: $CI_COMMIT_SHORT_SHA name: $CI_COMMIT_SHORT_SHA
@@ -241,13 +274,17 @@ check-tracing:
needs: needs:
- job: test-linux-stable-int - job: test-linux-stable-int
artifacts: false artifacts: false
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends: extends:
- .docker-env - .docker-env
- .test-refs - .test-refs
script: script:
- rusty-cachier snapshot create
# with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
- time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features
- time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
- rusty-cachier cache upload
test-full-crypto-feature: test-full-crypto-feature:
stage: test stage: test
@@ -255,6 +292,8 @@ test-full-crypto-feature:
needs: needs:
- job: check-tracing - job: check-tracing
artifacts: false artifacts: false
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends: extends:
- .docker-env - .docker-env
- .test-refs - .test-refs
@@ -264,10 +303,12 @@ test-full-crypto-feature:
RUSTFLAGS: "-Cdebug-assertions=y" RUSTFLAGS: "-Cdebug-assertions=y"
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
script: script:
- rusty-cachier snapshot create
- cd primitives/core/ - cd primitives/core/
- time cargo +nightly build --verbose --no-default-features --features full_crypto - time cargo +nightly build --verbose --no-default-features --features full_crypto
- cd ../application-crypto - cd ../application-crypto
- time cargo +nightly build --verbose --no-default-features --features full_crypto - time cargo +nightly build --verbose --no-default-features --features full_crypto
- rusty-cachier cache upload
test-wasmer-sandbox: test-wasmer-sandbox:
stage: test stage: test
@@ -275,7 +316,9 @@ test-wasmer-sandbox:
- .docker-env - .docker-env
- .test-refs-wasmer-sandbox - .test-refs-wasmer-sandbox
script: script:
- rusty-cachier snapshot create
- time cargo test --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests - time cargo test --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests
- rusty-cachier cache upload
cargo-check-macos: cargo-check-macos:
stage: test stage: test
@@ -289,6 +332,8 @@ cargo-check-macos:
check-rustdoc: check-rustdoc:
stage: test stage: test
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends: extends:
- .docker-env - .docker-env
- .test-refs - .test-refs
@@ -296,4 +341,6 @@ check-rustdoc:
SKIP_WASM_BUILD: 1 SKIP_WASM_BUILD: 1
RUSTDOCFLAGS: "-Dwarnings" RUSTDOCFLAGS: "-Dwarnings"
script: script:
- rusty-cachier snapshot create
- time cargo +nightly doc --workspace --all-features --verbose --no-deps - time cargo +nightly doc --workspace --all-features --verbose --no-deps
- rusty-cachier cache upload