mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-06-18 01:21:01 +00:00
Dmitry Kashitsyn
61606a0b3b
* Add comments and refactor Sandbox module * Adds some comments * Add wasmtime instance to the sandbox and delegate calls * Adds module imports stub * WIP state holder via *mut * My take at the problem * Brings back invoke and instantiate implementation details * Removes redundant bound * Code cleanup * Fixes invoke closure * Refactors FunctionExecutor to eliminate lifetime * Wraps `FunctionExecutor::sandbox_store` in `RefCell` * Renames `FunctionExecutor::heap` to `allocator` * Wraps `FunctionExecutor::allocator` in `RefCell` * Refactors FunctionExecutor to `Rc<Inner>` pattern * Implements scoped TLS for FunctionExecutor * Fixes wasmi instancing * Fixes sandbox asserts * Makes sandbox compile after wasmtime API change * Uses Vurich/wasmtime for the Lightbeam backend * Uses wasmtime instead of wasmi for sandbox API results * Refactors sandbox to use one of the execution backends at a time * Fixes wasmtime module instantiation * TEMP vurich branch stuff * Adds wasmer impl stub * Adds get global * Fixes warnings * Adds wasmer invoke impl * Implements host function interface for wasmer * Fixes wasmer instantiation result * Adds workaround to remove debug_assert * Fixes import object generation for wasmer * Attempt to propagate wasmer::Store through sandbox::Store * Wraps `sandbox::Store::memories` in `RefCell` * Moves `sandbox::instantiate` to `sandbox::Store` * Eliminate `RefCell<memories>` * Implements `HostState::memory_get/set`, removes accidental `borrow_mut` * Fixes sandbox memory handling for wasmi * Fix memory allocation * Resets Cargo.lock to match master * Fixes compilation * Refactors sandbox to use TLS for dispatch_thunk propagation to wasmer * Pass dispatch thunk to the sandbox as a TLS * Initialize dispatch thunk holder in `SandboxInstance` * Comment out Wasmtime/Lightbeam sandbox backend * Revert wasmtime back to mainstream * Adds SandboxExecutionMethod enum for cli param * Cleanup sandbox code * Allow wasmi to access wasmer memory regions * More cleanup * Remove debug logging, replace asserts with runtime errors * Revert "Adds SandboxExecutionMethod enum for cli param" This reverts commit dcb2b1d3b54145ab51ad2e3fef0d980ba215b596. * Fixes warnings * Fixes indentation and line width * Fix return types condition * Puts everything related under the `wasmer-sandbox` feature flag * Fixes warnings * Address grumbles * Split instantiate per backend * More splits * Refacmemory allocation * Nitpicks * Attempt to wrap wasmer memory in protoco enforcing type * Revert renaming * WIP wasm buffer proxy API * Reimplement util::wasmer::MemoryRef to use buffers instead of memory slices * Adds WasmiMemoryWrapper and MemoryTransfer trait * Refactor naming * Perform all memory transfers using MemoryTransfer * Adds allocating `read` * Adds comments * Removes unused imports * Removes now unused function * Pulls Cargo.lock from origin/master * Fix rustdoc * Removes unused `TransferError` * Update Cargo.lock * Removes unused import * cargo fmt * Fix feature dependency graph * Feature should flow from the top level crate * We should not assume a specific workspace structure * sc-executor-wasmi does not use the feature * sc-executor-wasmtime should not know about the feature * Fix doc typo * Enable wasmer-sandbox by default (for now) It will be removed before merge. It is so that the benchbot uses the wasmer sandbox. * cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_contracts --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/contracts/src/weights.rs --template=./.maintain/frame-weight-template.hbs * Revert "cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_contracts --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/contracts/src/weights.rs --template=./.maintain/frame-weight-template.hbs" This reverts commit d713590ba45387c4204b2ad97c8bd6f6ebabda4e. * cargo fmt * Add ci-check to prevent wasmer sandbox build breaking * Run tests with wasmer-sandbox enabled * Revert "Run tests with wasmer-sandbox enabled" This reverts commit cff63156a162f9ffdab23e7cb94a30f44e320f8a. Co-authored-by: Sergei Shulepov <s.pepyakin@gmail.com> Co-authored-by: Andrew Jones <ascjones@gmail.com> Co-authored-by: Alexander Theißen <alex.theissen@me.com> Co-authored-by: Parity Benchmarking Bot <admin@parity.io>
751 lines
29 KiB
YAML
751 lines
29 KiB
YAML
# .gitlab-ci.yml
|
||
#
|
||
# substrate
|
||
#
|
||
# pipelines can be triggered manually in the web
|
||
|
||
# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
|
||
# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
|
||
#
|
||
# my-example-job:
|
||
# stage: test # One of the stages listed below this job (required)
|
||
# image: paritytech/tools:latest # Any docker image (required)
|
||
# allow_failure: true # Allow the pipeline to continue if this job fails (default: false)
|
||
# needs:
|
||
# - job: test-linux # Any jobs that are required to run before this job (optional)
|
||
# variables:
|
||
# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional)
|
||
# script:
|
||
# - echo "List of shell commands to run in your job"
|
||
# - echo "You can also just specify a script here, like so:"
|
||
# - ./.maintain/gitlab/my_amazing_script.sh
|
||
|
||
stages:
|
||
- check
|
||
- test
|
||
- build
|
||
- publish
|
||
- deploy
|
||
|
||
workflow:
|
||
rules:
|
||
- if: $CI_COMMIT_TAG
|
||
- if: $CI_COMMIT_BRANCH
|
||
|
||
variables: &default-vars
|
||
GIT_STRATEGY: fetch
|
||
GIT_DEPTH: 100
|
||
CARGO_INCREMENTAL: 0
|
||
DOCKER_OS: "debian:stretch"
|
||
ARCH: "x86_64"
|
||
CI_IMAGE: "paritytech/ci-linux:production"
|
||
# FIXME set to release
|
||
CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.12"
|
||
CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder"
|
||
|
||
default:
|
||
cache: {}
|
||
|
||
.collect-artifacts: &collect-artifacts
|
||
artifacts:
|
||
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
|
||
when: on_success
|
||
expire_in: 7 days
|
||
paths:
|
||
- artifacts/
|
||
|
||
.kubernetes-env: &kubernetes-env
|
||
retry:
|
||
max: 2
|
||
when:
|
||
- runner_system_failure
|
||
- unknown_failure
|
||
- api_failure
|
||
interruptible: true
|
||
tags:
|
||
- kubernetes-parity-build
|
||
|
||
.rust-info-script: &rust-info-script
|
||
- rustup show
|
||
- cargo --version
|
||
- rustup +nightly show
|
||
- cargo +nightly --version
|
||
- sccache -s
|
||
|
||
.docker-env: &docker-env
|
||
image: "${CI_IMAGE}"
|
||
before_script:
|
||
- *rust-info-script
|
||
retry:
|
||
max: 2
|
||
when:
|
||
- runner_system_failure
|
||
- unknown_failure
|
||
- api_failure
|
||
interruptible: true
|
||
tags:
|
||
- linux-docker
|
||
|
||
.test-refs: &test-refs
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
|
||
.test-refs-no-trigger: &test-refs-no-trigger
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
|
||
.test-refs-no-trigger-prs-only: &test-refs-no-trigger-prs-only
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
|
||
.build-refs: &build-refs
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
|
||
.nightly-pipeline: &nightly-pipeline
|
||
rules:
|
||
# this job runs only on nightly pipeline with the mentioned variable, against `master` branch
|
||
- if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
|
||
|
||
.merge-ref-into-master-script: &merge-ref-into-master-script
|
||
- if [ $CI_COMMIT_REF_NAME != "master" ]; then
|
||
git fetch origin +master:master;
|
||
git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME;
|
||
git checkout master;
|
||
git config user.email "ci@gitlab.parity.io";
|
||
git merge $CI_COMMIT_REF_NAME --verbose --no-edit;
|
||
fi
|
||
|
||
.cargo-check-benches-script: &cargo-check-benches-script
|
||
- mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
|
||
- SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
|
||
- 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
|
||
| tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
|
||
- 'cargo run --release -p node-bench -- ::trie::read::small --json
|
||
| tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
|
||
- sccache -s
|
||
|
||
#### stage: .pre
|
||
|
||
skip-if-draft:
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
stage: .pre
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
script:
|
||
- echo "Commit message is ${CI_COMMIT_MESSAGE}"
|
||
- echo "Ref is ${CI_COMMIT_REF_NAME}"
|
||
- echo "pipeline source is ${CI_PIPELINE_SOURCE}"
|
||
- ./.maintain/gitlab/skip_if_draft.sh
|
||
|
||
#### stage: check
|
||
|
||
check-runtime:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
variables:
|
||
<<: *default-vars
|
||
GITLAB_API: "https://gitlab.parity.io/api/v4"
|
||
GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api"
|
||
script:
|
||
- ./.maintain/gitlab/check_runtime.sh
|
||
allow_failure: true
|
||
|
||
check-signed-tag:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
script:
|
||
- ./.maintain/gitlab/check_signed.sh
|
||
|
||
check-line-width:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
script:
|
||
- ./.maintain/gitlab/check_line_width.sh
|
||
allow_failure: true
|
||
|
||
test-dependency-rules:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
script:
|
||
- .maintain/ensure-deps.sh
|
||
|
||
test-prometheus-alerting-rules:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_COMMIT_BRANCH
|
||
changes:
|
||
- .gitlab-ci.yml
|
||
- .maintain/monitoring/**/*
|
||
script:
|
||
- promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml
|
||
- cat .maintain/monitoring/alerting-rules/alerting-rules.yaml |
|
||
promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
|
||
|
||
#### stage: test
|
||
|
||
cargo-deny:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *nightly-pipeline
|
||
script:
|
||
- cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
|
||
after_script:
|
||
- echo "___The complete log is in the artifacts___"
|
||
- cargo deny check -c .maintain/deny.toml 2> deny.log
|
||
artifacts:
|
||
name: $CI_COMMIT_SHORT_SHA
|
||
expire_in: 3 days
|
||
when: always
|
||
paths:
|
||
- deny.log
|
||
# FIXME: Temorarily allow to fail.
|
||
allow_failure: true
|
||
|
||
cargo-fmt:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- cargo +nightly fmt --all -- --check
|
||
allow_failure: true
|
||
|
||
cargo-check-benches:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
<<: *collect-artifacts
|
||
before_script:
|
||
# merges in the master branch on PRs
|
||
- *merge-ref-into-master-script
|
||
- *rust-info-script
|
||
script:
|
||
- *cargo-check-benches-script
|
||
|
||
node-bench-regression-guard:
|
||
# it's not belong to `build` semantically, but dag jobs can't depend on each other
|
||
# within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
|
||
# more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
|
||
stage: build
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger-prs-only
|
||
needs:
|
||
# this is a DAG
|
||
- job: cargo-check-benches
|
||
artifacts: true
|
||
# this does not like a DAG, just polls the artifact
|
||
- project: $CI_PROJECT_PATH
|
||
job: cargo-check-benches
|
||
ref: master
|
||
artifacts: true
|
||
variables:
|
||
CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
|
||
before_script: [""]
|
||
script:
|
||
- 'node-bench-regression-guard --reference artifacts/benches/master-*
|
||
--compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
|
||
|
||
cargo-check-subkey:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- cd ./bin/utils/subkey
|
||
- SKIP_WASM_BUILD=1 time cargo check --release
|
||
- sccache -s
|
||
|
||
cargo-check-try-runtime:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- time cargo check --features try-runtime
|
||
- sccache -s
|
||
|
||
cargo-check-wasmer-sandbox:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- time cargo check --features wasmer-sandbox
|
||
- sccache -s
|
||
|
||
test-deterministic-wasm:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
WASM_BUILD_NO_COLOR: 1
|
||
script:
|
||
# build runtime
|
||
- cargo build --verbose --release -p node-runtime
|
||
# make checksum
|
||
- sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
|
||
# clean up – FIXME: can we reuse some of the artifacts?
|
||
- cargo clean
|
||
# build again
|
||
- cargo build --verbose --release -p node-runtime
|
||
# confirm checksum
|
||
- sha256sum -c checksum.sha256
|
||
- sccache -s
|
||
|
||
test-linux-stable: &test-linux
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
# Enable debug assertions since we are running optimized builds for testing
|
||
# but still want to have debug assertions.
|
||
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
|
||
RUST_BACKTRACE: 1
|
||
WASM_BUILD_NO_COLOR: 1
|
||
script:
|
||
# this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
|
||
- time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
|
||
- time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec
|
||
- SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
|
||
- sccache -s
|
||
|
||
unleash-check:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger
|
||
script:
|
||
- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
|
||
- cargo unleash de-dev-deps
|
||
# Reuse build artifacts when running checks (cuts down check time by 3x)
|
||
# TODO: Implement this optimization in cargo-unleash rather than here
|
||
- mkdir -p target/unleash
|
||
- export CARGO_TARGET_DIR=target/unleash
|
||
- cargo unleash check ${CARGO_UNLEASH_PKG_DEF}
|
||
# FIXME: this job must not fail, or unleash-to-crates-io will publish broken stuff
|
||
allow_failure: true
|
||
|
||
test-frame-examples-compile-to-wasm:
|
||
# into one job
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
# Enable debug assertions since we are running optimized builds for testing
|
||
# but still want to have debug assertions.
|
||
RUSTFLAGS: "-Cdebug-assertions=y"
|
||
RUST_BACKTRACE: 1
|
||
script:
|
||
- cd frame/example-offchain-worker/
|
||
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
|
||
- cd ../example
|
||
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
|
||
- sccache -s
|
||
|
||
test-linux-stable-int:
|
||
<<: *test-linux
|
||
stage: test
|
||
script:
|
||
- echo "___Logs will be partly shown at the end in case of failure.___"
|
||
- echo "___Full log will be saved to the job artifacts only in case of failure.___"
|
||
- WASM_BUILD_NO_COLOR=1
|
||
RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
|
||
time cargo test -p node-cli --release --verbose --locked -- --ignored
|
||
&> ${CI_COMMIT_SHORT_SHA}_int_failure.log
|
||
- sccache -s
|
||
after_script:
|
||
- awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
|
||
artifacts:
|
||
name: $CI_COMMIT_SHORT_SHA
|
||
when: on_failure
|
||
expire_in: 3 days
|
||
paths:
|
||
- ${CI_COMMIT_SHORT_SHA}_int_failure.log
|
||
|
||
check-tracing:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
# with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
|
||
- time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
|
||
- time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
|
||
- sccache -s
|
||
|
||
test-full-crypto-feature:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
# Enable debug assertions since we are running optimized builds for testing
|
||
# but still want to have debug assertions.
|
||
RUSTFLAGS: "-Cdebug-assertions=y"
|
||
RUST_BACKTRACE: 1
|
||
script:
|
||
- cd primitives/core/
|
||
- time cargo +nightly build --verbose --no-default-features --features full_crypto
|
||
- cd ../application-crypto
|
||
- time cargo +nightly build --verbose --no-default-features --features full_crypto
|
||
- sccache -s
|
||
|
||
cargo-check-macos:
|
||
stage: test
|
||
# shell runner on mac ignores the image set in *docker-env
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger
|
||
script:
|
||
- SKIP_WASM_BUILD=1 time cargo check --release
|
||
- sccache -s
|
||
tags:
|
||
- osx
|
||
|
||
#### stage: build
|
||
|
||
check-polkadot-companion-status:
|
||
stage: build
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
script:
|
||
- ./.maintain/gitlab/check_polkadot_companion_status.sh
|
||
|
||
check-polkadot-companion-build:
|
||
stage: build
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger
|
||
needs:
|
||
- job: test-linux-stable-int
|
||
artifacts: false
|
||
script:
|
||
- ./.maintain/gitlab/check_polkadot_companion_build.sh
|
||
after_script:
|
||
- cd polkadot && git rev-parse --abbrev-ref HEAD
|
||
allow_failure: true
|
||
|
||
build-linux-substrate: &build-binary
|
||
stage: build
|
||
<<: *collect-artifacts
|
||
<<: *docker-env
|
||
<<: *build-refs
|
||
needs:
|
||
- job: test-linux-stable
|
||
artifacts: false
|
||
before_script:
|
||
- mkdir -p ./artifacts/substrate/
|
||
script:
|
||
- WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
|
||
- mv ./target/release/substrate ./artifacts/substrate/.
|
||
- echo -n "Substrate version = "
|
||
- if [ "${CI_COMMIT_TAG}" ]; then
|
||
echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
|
||
else
|
||
./artifacts/substrate/substrate --version |
|
||
sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
|
||
tee ./artifacts/substrate/VERSION;
|
||
fi
|
||
- sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
|
||
- printf '\n# building node-template\n\n'
|
||
- ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
|
||
- cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
|
||
- sccache -s
|
||
|
||
build-linux-subkey: &build-subkey
|
||
stage: build
|
||
<<: *collect-artifacts
|
||
<<: *docker-env
|
||
<<: *build-refs
|
||
needs:
|
||
- job: cargo-check-subkey
|
||
artifacts: false
|
||
before_script:
|
||
- mkdir -p ./artifacts/subkey
|
||
script:
|
||
- cd ./bin/utils/subkey
|
||
- SKIP_WASM_BUILD=1 time cargo build --release --verbose
|
||
- cd -
|
||
- mv ./target/release/subkey ./artifacts/subkey/.
|
||
- echo -n "Subkey version = "
|
||
- ./artifacts/subkey/subkey --version |
|
||
sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
|
||
tee ./artifacts/subkey/VERSION;
|
||
- sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
|
||
- cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/
|
||
- sccache -s
|
||
|
||
build-macos-subkey:
|
||
<<: *build-subkey
|
||
tags:
|
||
- osx
|
||
|
||
build-rustdoc:
|
||
stage: build
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
SKIP_WASM_BUILD: 1
|
||
artifacts:
|
||
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
|
||
when: on_success
|
||
expire_in: 7 days
|
||
paths:
|
||
- ./crate-docs/
|
||
script:
|
||
# FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"`
|
||
- RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
|
||
time cargo +nightly doc --no-deps --workspace --all-features --verbose
|
||
- rm -f ./target/doc/.lock
|
||
- mv ./target/doc ./crate-docs
|
||
# FIXME: remove me after CI image gets nonroot
|
||
- chown -R nonroot:nonroot ./crate-docs
|
||
- echo "<meta http-equiv=refresh content=0;url=sc_service/index.html>" > ./crate-docs/index.html
|
||
- sccache -s
|
||
|
||
#### stage: publish
|
||
|
||
.build-push-docker-image: &build-push-docker-image
|
||
<<: *build-refs
|
||
<<: *kubernetes-env
|
||
image: quay.io/buildah/stable
|
||
variables: &docker-build-vars
|
||
<<: *default-vars
|
||
GIT_STRATEGY: none
|
||
DOCKERFILE: $PRODUCT.Dockerfile
|
||
IMAGE_NAME: docker.io/parity/$PRODUCT
|
||
before_script:
|
||
- cd ./artifacts/$PRODUCT/
|
||
- VERSION="$(cat ./VERSION)"
|
||
- echo "${PRODUCT} version = ${VERSION}"
|
||
- test -z "${VERSION}" && exit 1
|
||
script:
|
||
- test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
|
||
( echo "no docker credentials provided"; exit 1 )
|
||
- buildah bud
|
||
--format=docker
|
||
--build-arg VCS_REF="${CI_COMMIT_SHA}"
|
||
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||
--tag "$IMAGE_NAME:$VERSION"
|
||
--tag "$IMAGE_NAME:latest"
|
||
--file "$DOCKERFILE" .
|
||
- echo "$Docker_Hub_Pass_Parity" |
|
||
buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
|
||
- buildah info
|
||
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
|
||
- buildah push --format=v2s2 "$IMAGE_NAME:latest"
|
||
after_script:
|
||
- buildah logout "$IMAGE_NAME"
|
||
# pass artifacts to the trigger-simnet job
|
||
- echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env
|
||
- IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)"
|
||
- echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env
|
||
- cat ./artifacts/$PRODUCT/build.env
|
||
|
||
publish-docker-substrate:
|
||
stage: publish
|
||
<<: *build-push-docker-image
|
||
<<: *build-refs
|
||
needs:
|
||
- job: build-linux-substrate
|
||
artifacts: true
|
||
variables:
|
||
<<: *docker-build-vars
|
||
PRODUCT: substrate
|
||
artifacts:
|
||
reports:
|
||
# this artifact is used in trigger-simnet job
|
||
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
|
||
dotenv: ./artifacts/substrate/build.env
|
||
|
||
publish-docker-subkey:
|
||
stage: publish
|
||
<<: *build-push-docker-image
|
||
needs:
|
||
- job: build-linux-subkey
|
||
artifacts: true
|
||
variables:
|
||
<<: *docker-build-vars
|
||
PRODUCT: subkey
|
||
|
||
publish-s3-release:
|
||
stage: publish
|
||
<<: *build-refs
|
||
<<: *kubernetes-env
|
||
needs:
|
||
- job: build-linux-substrate
|
||
artifacts: true
|
||
- job: build-linux-subkey
|
||
artifacts: true
|
||
image: paritytech/awscli:latest
|
||
variables:
|
||
GIT_STRATEGY: none
|
||
BUCKET: "releases.parity.io"
|
||
PREFIX: "substrate/${ARCH}-${DOCKER_OS}"
|
||
script:
|
||
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
|
||
- echo "update objects in latest path"
|
||
- aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
|
||
after_script:
|
||
- aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
|
||
--recursive --human-readable --summarize
|
||
|
||
publish-rustdoc:
|
||
stage: publish
|
||
<<: *kubernetes-env
|
||
image: paritytech/tools:latest
|
||
variables:
|
||
GIT_DEPTH: 100
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
# `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
|
||
# artifacts from being dowloaded by this job.
|
||
needs:
|
||
- job: build-rustdoc
|
||
artifacts: true
|
||
script:
|
||
- rm -rf /tmp/*
|
||
# Set git config
|
||
- rm -rf .git/config
|
||
- git config user.email "devops-team@parity.io"
|
||
- git config user.name "${GITHUB_USER}"
|
||
- git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/substrate.git"
|
||
- git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
|
||
- git fetch origin gh-pages
|
||
# Save README and docs
|
||
- cp -r ./crate-docs/ /tmp/doc/
|
||
- cp README.md /tmp/doc/
|
||
- git checkout gh-pages
|
||
# Remove everything and restore generated docs and README
|
||
- rm -rf ./*
|
||
- mv /tmp/doc/* .
|
||
# Upload files
|
||
- git add --all --force
|
||
# `git commit` has an exit code of > 0 if there is nothing to commit.
|
||
# This causes GitLab to exit immediately and marks this job failed.
|
||
# We don't want to mark the entire job failed if there's nothing to
|
||
# publish though, hence the `|| true`.
|
||
- git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
|
||
echo "___Nothing to commit___"
|
||
- git push origin gh-pages --force
|
||
after_script:
|
||
- rm -rf .git/ ./*
|
||
|
||
publish-draft-release:
|
||
stage: publish
|
||
image: paritytech/tools:latest
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
script:
|
||
- ./.maintain/gitlab/publish_draft_release.sh
|
||
allow_failure: true
|
||
|
||
unleash-to-crates-io:
|
||
stage: publish
|
||
<<: *docker-env
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
# FIXME: wait until https://github.com/paritytech/cargo-unleash/issues/50 is fixed, also
|
||
# remove allow_failure: true on the check job
|
||
# - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
script:
|
||
- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
|
||
- cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF}
|
||
allow_failure: true
|
||
|
||
#### stage: deploy
|
||
|
||
deploy-prometheus-alerting-rules:
|
||
stage: deploy
|
||
needs:
|
||
- job: test-prometheus-alerting-rules
|
||
artifacts: false
|
||
allow_failure: true
|
||
trigger:
|
||
project: parity/infrastructure/cloud-infra
|
||
variables:
|
||
SUBSTRATE_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}"
|
||
SUBSTRATE_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}"
|
||
UPSTREAM_TRIGGER_PROJECT: "${CI_PROJECT_PATH}"
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
changes:
|
||
- .gitlab-ci.yml
|
||
- .maintain/monitoring/**/*
|
||
|
||
# Runs "quick" and "long" tests on nightly schedule and on commit / merge to master
|
||
# A "quick" test is a smoke test where basic check-expect tests run by
|
||
# checking values from metrics exposed by the app.
|
||
# A "long" test is the load testing where we send 50K transactions into the
|
||
# network and check if all completed successfully
|
||
simnet-tests:
|
||
stage: deploy
|
||
image: docker.io/paritytech/simnet:${SIMNET_REF}
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
needs:
|
||
- job: publish-docker-substrate
|
||
# variables:
|
||
# `build.env` brings here `${SUBSTRATE_IMAGE_NAME}` and `${SUBSTRATE_IMAGE_TAG}`
|
||
# (`$VERSION` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
|
||
# ${SIMNET_REF} is a gitlab variable
|
||
before_script:
|
||
- echo "Simnet Tests Config
|
||
docker.io/paritytech/simnet:${SIMNET_REF}
|
||
${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IAMGE_TAG}"
|
||
script:
|
||
- /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh
|
||
--github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests"
|
||
--config="simnet_tests/configs/default_local_testnet.toml"
|
||
--image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}"
|
||
retry: 2
|
||
tags:
|
||
- parity-simnet
|