pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-05-30 22:11:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

CI: optimizations (#2885)

Browse Source 
* CI: do not publish unnecessary things to s3 and to release registry

* CI: remove the build-linux-release job

* CI: publish less

* CI: workspace = all

* CI: more logs for the pipeline trigger

* CI: chore

* CI: nicer

* CI: new bucket for bins

* CI: no need pushing every master image to product registry

* CI: default variables in image jobs

* CI: optimize the bin build
This commit is contained in:
Denis Pisarev
2021-04-22 10:48:15 +02:00
committed by GitHub
parent c25e79d8c4
commit 683211837b
3 changed files with 66 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/.gitlab-ci.yml
+48 -80
View File
@@ -109,23 +109,39 @@ test-deterministic-wasm:
script: script:
- ./scripts/gitlab/test_deterministic_wasm.sh - ./scripts/gitlab/test_deterministic_wasm.sh
test-linux-stable: test-build-linux-stable:
stage: test stage: test
<<: *rules-test
<<: *docker-env <<: *docker-env
<<: *compiler-info <<: *compiler-info
<<: *collect-artifacts
variables: variables:
RUST_TOOLCHAIN: stable RUST_TOOLCHAIN: stable
# Enable debug assertions since we are running optimized builds for testing # Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions. # but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
TARGET: native rules:
artifacts: - if: $CI_PIPELINE_SOURCE == "schedule"
paths: - if: $CI_COMMIT_REF_NAME == "master"
- ./target/release/polkadot - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
script: script:
- ./scripts/gitlab/test_linux_stable.sh - ./scripts/gitlab/test_linux_stable.sh
# we're using the bin built here, instead of having a parallel `build-linux-release`
- time cargo build --release --verbose --bin polkadot
- sccache -s - sccache -s
# pack-artifacts
- mkdir -p ./artifacts
- VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
- mv ./target/release/polkadot ./artifacts/.
- sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
- EXTRATAG="$(./artifacts/polkadot --version |
sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
- EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- echo -n ${VERSION} > ./artifacts/VERSION
- echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
- cp -r scripts/docker/* ./artifacts
check-web-wasm: check-web-wasm:
stage: test stage: test
@@ -148,43 +164,6 @@ check-runtime-benchmarks:
- ./scripts/gitlab/check_runtime_benchmarks.sh - ./scripts/gitlab/check_runtime_benchmarks.sh
- sccache -s - sccache -s
.pack-artifacts: &pack-artifacts
- mkdir -p ./artifacts
- VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
- mv ./target/release/polkadot ./artifacts/.
- sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
- if [ "${CI_COMMIT_TAG}" ]; then
EXTRATAG="latest";
else
EXTRATAG="$(./artifacts/polkadot --version |
sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')";
EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)";
fi
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- echo -n ${VERSION} > ./artifacts/VERSION
- echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
- cp -r scripts/docker/* ./artifacts
build-linux-release:
stage: test
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
RUSTFLAGS: "-Cdebug-assertions=y"
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
RUSTFLAGS: "-Cdebug-assertions=y"
script:
- time cargo build --release --verbose
- sccache -s
- *pack-artifacts
build-adder-collator: build-adder-collator:
stage: test stage: test
<<: *collect-artifacts <<: *collect-artifacts
@@ -213,7 +192,7 @@ check-transaction-versions:
<<: *rules-test <<: *rules-test
<<: *docker-env <<: *docker-env
needs: needs:
- job: test-linux-stable - job: test-build-linux-stable
artifacts: true artifacts: true
before_script: before_script:
- apt-get -y update; apt-get -y install jq lsof - apt-get -y update; apt-get -y install jq lsof
@@ -235,19 +214,16 @@ generate-impl-guide:
.build-push-image: &build-push-image .build-push-image: &build-push-image
<<: *kubernetes-env <<: *kubernetes-env
image: quay.io/buildah/stable image: quay.io/buildah/stable
variables: variables: &image-variables
GIT_STRATEGY: none GIT_STRATEGY: none
# scripts/docker/Dockerfile DOCKER_USER: ${PARITYPR_USER}
DOCKERFILE: Dockerfile DOCKER_PASS: ${PARITYPR_PASS}
IMAGE_NAME: docker.io/parity/polkadot
DOCKER_USER: ${Docker_Hub_User_Parity}
DOCKER_PASS: ${Docker_Hub_Pass_Parity}
before_script: &check-versions before_script: &check-versions
- test -s ./artifacts/VERSION || exit 1 - test -s ./artifacts/VERSION || exit 1
- test -s ./artifacts/EXTRATAG || exit 1 - test -s ./artifacts/EXTRATAG || exit 1
- VERSION="$(cat ./artifacts/VERSION)" - VERSION="$(cat ./artifacts/VERSION)"
- EXTRATAG="$(cat ./artifacts/EXTRATAG)" - EXTRATAG="$(cat ./artifacts/EXTRATAG)"
- echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})" - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
script: script:
- test "$DOCKER_USER" -a "$DOCKER_PASS" || - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
( echo "no docker credentials provided"; exit 1 ) ( echo "no docker credentials provided"; exit 1 )
@@ -280,6 +256,11 @@ generate-impl-guide:
publish-polkadot-image: publish-polkadot-image:
stage: build stage: build
<<: *build-push-image <<: *build-push-image
variables:
<<: *image-variables
# scripts/docker/Dockerfile
DOCKERFILE: Dockerfile
IMAGE_NAME: docker.io/paritypr/synth-wave
rules: rules:
# Don't run on releases - this is handled by the Github Action here: # Don't run on releases - this is handled by the Github Action here:
# .github/workflows/publish-docker-release.yml # .github/workflows/publish-docker-release.yml
@@ -288,15 +269,14 @@ publish-polkadot-image:
- if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
IMAGE_NAME: docker.io/paritypr/synth-wave
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
- if: $CI_COMMIT_REF_NAME == "rococo-v1" - if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables: variables:
<<: *image-variables
IMAGE_NAME: docker.io/parity/rococo IMAGE_NAME: docker.io/parity/rococo
DOCKER_USER: ${Docker_Hub_User_Parity}
DOCKER_PASS: ${Docker_Hub_Pass_Parity}
needs: needs:
- job: build-linux-release - job: test-build-linux-stable
artifacts: true artifacts: true
publish-adder-collator-image: publish-adder-collator-image:
@@ -304,15 +284,13 @@ publish-adder-collator-image:
stage: build stage: build
<<: *build-push-image <<: *build-push-image
variables: variables:
<<: *image-variables
# scripts/docker/collator.Dockerfile # scripts/docker/collator.Dockerfile
DOCKERFILE: collator.Dockerfile DOCKERFILE: collator.Dockerfile
IMAGE_NAME: docker.io/paritypr/colander IMAGE_NAME: docker.io/paritypr/colander
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
rules: rules:
- if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "master"
# FIXME: remove me after merging
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
needs: needs:
- job: build-adder-collator - job: build-adder-collator
@@ -332,51 +310,41 @@ publish-adder-collator-image:
publish-s3-release: &publish-s3 publish-s3-release: &publish-s3
stage: publish stage: publish
needs: needs:
- job: build-linux-release - job: test-build-linux-stable
artifacts: true artifacts: true
<<: *kubernetes-env <<: *kubernetes-env
image: paritytech/awscli:latest image: paritytech/awscli:latest
variables: variables:
GIT_STRATEGY: none GIT_STRATEGY: none
BUCKET: "releases.parity.io" PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}"
PREFIX: "polkadot/${ARCH}-${DOCKER_OS}"
rules: rules:
# publishing binaries nightly
- if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
before_script: before_script:
- *check-versions - *check-versions
script: script:
- echo "uploading objects to https://${BUCKET}/${PREFIX}/${VERSION}" - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}"
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/${VERSION}/ - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/
- echo "update objects at https://${BUCKET}/${PREFIX}/${EXTRATAG}" - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}"
- find ./artifacts -type f | while read file; do - find ./artifacts -type f | while read file; do
name="${file#./artifacts/}"; name="${file#./artifacts/}";
aws s3api copy-object aws s3api copy-object
--copy-source ${BUCKET}/${PREFIX}/${VERSION}/${name} --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name}
--bucket ${BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
done done
- | - |
cat <<-EOM cat <<-EOM
| |
| polkadot binary paths: | polkadot binary paths:
| |
| - https://${BUCKET}/${PREFIX}/${EXTRATAG}/polkadot | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot
| - https://${BUCKET}/${PREFIX}/${VERSION}/polkadot | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot
| |
EOM EOM
after_script: after_script:
- aws s3 ls s3://${BUCKET}/${PREFIX}/${EXTRATAG}/ - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/
--recursive --human-readable --summarize --recursive --human-readable --summarize
publish-s3-adder-collator:
<<: *publish-s3
variables:
PREFIX: "rococo/${ARCH}-${DOCKER_OS}"
needs:
- job: build-adder-collator
artifacts: true
#### stage: deploy #### stage: deploy
deploy-polkasync-kusama: deploy-polkasync-kusama:
polkadot/scripts/gitlab/test_linux_stable.sh
+1 -1
View File
@@ -4,4 +4,4 @@ set -e
#shellcheck source=../common/lib.sh #shellcheck source=../common/lib.sh
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh" source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
time cargo test --all --release --verbose --locked --features=runtime-benchmarks time cargo test --workspace --release --verbose --locked --features=runtime-benchmarks
polkadot/scripts/gitlab/trigger_pipeline.sh
+14 -8
View File
@@ -1,7 +1,10 @@
#!/bin/bash #!/bin/bash
set -eu set -eu
# API trigger another project's pipeline # API trigger another project's pipeline
echo "Triggering Simnet pipeline."
curl --silent \ curl --silent \
-X POST \ -X POST \
-F "token=${CI_JOB_TOKEN}" \ -F "token=${CI_JOB_TOKEN}" \
@@ -12,13 +15,14 @@ curl --silent \
-F "variables[IMAGE_TAG]=${IMAGE_TAG}" \ -F "variables[IMAGE_TAG]=${IMAGE_TAG}" \
-F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \ -F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \
"https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \ "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \
tee pipeline tee pipeline;
PIPELINE_ID=$(cat pipeline | jq ".id") PIPELINE_ID=$(cat pipeline | jq ".id")
echo "\nWaiting on ${PIPELINE_ID} status..." PIPELINE_URL=$(cat pipeline | jq ".web_url")
echo
echo "Simnet pipeline ${PIPELINE_URL} was successfully triggered."
echo "Now we're polling it to obtain the distinguished status."
# This part polls for the triggered pipeline status, the native
# `trigger` job does not return this status via API.
# This is a workaround for a Gitlab bug, waits here until # This is a workaround for a Gitlab bug, waits here until
# https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed. # https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
# The timeout is 360 curls with 8 sec interval, roughly an hour. # The timeout is 360 curls with 8 sec interval, roughly an hour.
@@ -30,17 +34,19 @@ function get_status() {
jq --raw-output ".status"; jq --raw-output ".status";
} }
echo "Waiting on ${PIPELINE_ID} status..."
for i in $(seq 1 360); do for i in $(seq 1 360); do
STATUS=$(get_status); STATUS=$(get_status);
echo "Triggered pipeline status is ${STATUS}"; echo "Triggered pipeline status is ${STATUS}";
if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
echo "Busy..."; echo;
elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
exit 1; echo "Something's broken in: ${PIPELINE_URL}"; exit 1;
elif [[ ${STATUS} =~ ^(success)$ ]]; then elif [[ ${STATUS} =~ ^(success)$ ]]; then
exit 0; echo "Look how green it is: ${PIPELINE_URL}"; exit 0;
else else
exit 1; echo "Something else has happened in ${PIPELINE_URL}"; exit 1;
fi fi
sleep 8; sleep 8;
done done