Cargo caching with forklift (#2466)

This PR adds cargo caching feature with custom ['forklift'
tool](https://gitlab.parity.io/parity/infrastructure/ci_cd/forklift/forklift)
Forklift acts as RUSTC_WRAPPER, intercepts rustc calls and stores
produced artifacts in S3 bucket (see [forklift
readme](https://gitlab.parity.io/parity/infrastructure/ci_cd/forklift/forklift/-/blob/main/README.MD?ref_type=heads)
for detailed description)

All settings are made in [`.forklift` job's
before_script](https://github.com/paritytech/polkadot-sdk/blob/es/forklift-test/.gitlab-ci.yml#L119)
and affect all jobs that extend `.docker-env` job

To disable feature set `FORKLIFT_BYPASS` variable to true in [project
settings in
gitlab](https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/settings/ci_cd)
This commit is contained in:
Evgeny Snitko
2023-12-04 20:07:22 +04:00
committed by GitHub
parent aa4754e30a
commit e6b9da132d
+26 -13
View File
@@ -106,27 +106,37 @@ default:
.docker-env:
image: "${CI_IMAGE}"
variables:
FL_FORKLIFT_VERSION: !reference [.forklift, variables, FL_FORKLIFT_VERSION]
before_script:
- !reference [.common-before-script, before_script]
- !reference [.prepare-env, before_script]
- !reference [.rust-info-script, script]
- !reference [.rusty-cachier, before_script]
- !reference [.forklift-cache, before_script]
tags:
- linux-docker
# rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier
# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
.rusty-cachier:
#
.forklift-cache:
before_script:
# - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
# - mkdir -p cargo_home cargo_target_dir
# - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home
# - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir
# - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} +
# - git restore-mtime
# - rusty-cachier --version
# - rusty-cachier project touch-changed
- echo tbd
- 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"'
- chmod +x forklift
- mkdir .forklift
- cp $FL_FORKLIFT_CONFIG .forklift/config.toml
- export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*}
- shopt -s expand_aliases
- export PATH=$PATH:$(pwd)
- |
if [ "$FORKLIFT_BYPASS" != "true" ]; then
echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"
alias cargo="forklift cargo"
fi
- ls -al
- rm -f forklift.sock
- forklift clean
#
- echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}"
- echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX"
.common-refs:
rules:
@@ -214,6 +224,9 @@ include:
- project: parity/infrastructure/ci_cd/shared
ref: main
file: /common/ci-unified.yml
- project: parity/infrastructure/ci_cd/shared
ref: main
file: /common/forklift.yml
# This job cancels the whole pipeline if any of provided jobs fail.
# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
# to fail the pipeline as soon as possible to shorten the feedback loop.