pezkuwi-subxt/substrate/scripts/ci/gitlab/pipeline/publish.yml

# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "publish" stage

.build-push-docker-image-common:
  extends:
    - .kubernetes-env
  stage: publish
  variables:
    CI_IMAGE: $BUILDAH_IMAGE
    GIT_STRATEGY: none
    DOCKERFILE: $PRODUCT.Dockerfile
    IMAGE_NAME: docker.io/$IMAGE_PATH
  before_script:
    - !reference [.kubernetes-env, before_script]
    - cd ./artifacts/$PRODUCT/
    - VERSION="$(cat ./VERSION)"
    - echo "${PRODUCT} version = ${VERSION}"
    - test -z "${VERSION}" && exit 1
  script:
    - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
      ( echo "no docker credentials provided"; exit 1 )
    - $BUILDAH_COMMAND build
      --format=docker
      --build-arg VCS_REF="${CI_COMMIT_SHA}"
      --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
      --build-arg IMAGE_NAME="${IMAGE_PATH}"
      --tag "$IMAGE_NAME:$VERSION"
      --tag "$IMAGE_NAME:latest"
      --file "$DOCKERFILE" .
    - echo "$DOCKER_PASS" |
      buildah login --username "$DOCKER_USER" --password-stdin docker.io
    - $BUILDAH_COMMAND info
    - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION"
    - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:latest"
  after_script:
    - buildah logout --all
    - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env
    - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)"
    - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}"   | tee -a ./artifacts/$PRODUCT/build.env
    - cat ./artifacts/$PRODUCT/build.env

.build-push-docker-image:
  extends:
    - .publish-refs
    - .build-push-docker-image-common
  variables:
    IMAGE_PATH: parity/$PRODUCT
    DOCKER_USER: $Docker_Hub_User_Parity
    DOCKER_PASS: $Docker_Hub_Pass_Parity

.push-docker-image-description:
  stage: publish
  extends:
    - .kubernetes-env
  variables:
    CI_IMAGE: paritytech/dockerhub-description
    DOCKERHUB_REPOSITORY: parity/$PRODUCT
    DOCKER_USERNAME: $Docker_Hub_User_Parity
    DOCKER_PASSWORD: $Docker_Hub_Pass_Parity
    README_FILEPATH: $CI_PROJECT_DIR/scripts/ci/docker/$PRODUCT.Dockerfile.README.md
  rules:
    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
      changes:
        - scripts/ci/docker/$PRODUCT.Dockerfile.README.md
  before_script:
    - echo
  script:
    - cd / && sh entrypoint.sh

# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
.build-push-image-temporary:
  extends:
    - .build-refs
    - .build-push-docker-image-common
  variables:
    IMAGE_PATH: paritypr/$PRODUCT
    DOCKER_USER: $PARITYPR_USER
    DOCKER_PASS: $PARITYPR_PASS

publish-docker-substrate:
  extends: .build-push-docker-image
  needs:
    - job: build-linux-substrate
      artifacts: true
  variables:
    PRODUCT: substrate

publish-docker-description-substrate:
  extends: .push-docker-image-description
  variables:
    PRODUCT: substrate
    SHORT_DESCRIPTION: "Substrate Docker Image."

publish-docker-substrate-temporary:
  extends: .build-push-image-temporary
  needs:
    - job: build-linux-substrate
      artifacts: true
  variables:
    PRODUCT: substrate
  artifacts:
    reports:
      # this artifact is used in zombienet-tests job
      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
      dotenv: ./artifacts/$PRODUCT/build.env
    expire_in: 24h

publish-docker-subkey:
  extends: .build-push-docker-image
  needs:
    - job: build-subkey-linux
      artifacts: true
  variables:
    PRODUCT: subkey

publish-docker-description-subkey:
  extends: .push-docker-image-description
  variables:
    PRODUCT: subkey
    SHORT_DESCRIPTION: "The subkey program is a key management utility for Substrate-based blockchains."

publish-s3-release:
  stage: publish
  extends:
    - .publish-refs
    - .kubernetes-env
  needs:
    - job: build-linux-substrate
      artifacts: true
    - job: build-subkey-linux
      artifacts: true
  image: paritytech/awscli:latest
  variables:
    GIT_STRATEGY: none
    BUCKET: "releases.parity.io"
    PREFIX: "substrate/${ARCH}-${DOCKER_OS}"
  script:
    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
    - echo "update objects in latest path"
    - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
  after_script:
    - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
      --recursive --human-readable --summarize

publish-rustdoc:
  stage: publish
  extends: .kubernetes-env
  variables:
    CI_IMAGE: node:16
    GIT_DEPTH: 100
    RUSTDOCS_DEPLOY_REFS: "master"
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^monthly-20[0-9]{2}-[0-9]{2}.*$/ # to support: monthly-2021-09+1
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
  # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
  # artifacts from being dowloaded by this job.
  needs:
    - job: build-rustdoc
      artifacts: true
  script:
    # If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we
    # exit immediately.
    # Putting spaces at the front and back to ensure we are not matching just any substring, but the
    # whole space-separated value.
    - '[[ " ${RUSTDOCS_DEPLOY_REFS} " =~ " ${CI_COMMIT_REF_NAME} " ]] || exit 0'
    # setup ssh
    - eval $(ssh-agent)
    - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
    - mkdir ~/.ssh && touch ~/.ssh/known_hosts
    - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
    # Set git config
    - git config user.email "devops-team@parity.io"
    - git config user.name "${GITHUB_USER}"
    - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git"
    - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
    - git fetch origin gh-pages
    # Save README and docs
    - cp -r ./crate-docs/ /tmp/doc/
    - cp README.md /tmp/doc/
    # we don't need to commit changes because we copy docs to /tmp
    - git checkout gh-pages --force
    # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS
    - which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud
    - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME}
    # Ensure the destination dir doesn't exist.
    - rm -rf ${CI_COMMIT_REF_NAME}
    - mv -f /tmp/doc ${CI_COMMIT_REF_NAME}
    # Upload files
    - git add --all
    # `git commit` has an exit code of > 0 if there is nothing to commit.
    # This causes GitLab to exit immediately and marks this job failed.
    # We don't want to mark the entire job failed if there's nothing to
    # publish though, hence the `|| true`.
    - git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
      echo "___Nothing to commit___"
    - git push origin gh-pages --force
  after_script:
    - rm -rf .git/ ./*

publish-draft-release:
  stage: publish
  image: paritytech/tools:latest
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
  script:
    - ./scripts/ci/gitlab/publish_draft_release.sh
  allow_failure: true

.publish-crates-template:
  stage: publish
  extends:
    - .crates-publishing-template
    - .crates-publishing-pipeline
  # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite
  # the releases of one another. Use resource_group to ensure that at most one instance of this job
  # is running at any given time.
  resource_group: crates-publishing
  # crates.io currently rate limits crate publishing at 1 per minute:
  # https://github.com/paritytech/release-engineering/issues/123#issuecomment-1335509748
  # Taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, in the worst
  # case, due to the rate limits alone, we'd have to wait through at least 202 minutes of delay.
  # Taking into account also the verification steps and extra synchronization delays after
  # publishing the crate, the job needs to have a much higher timeout than average.
  timeout: 9h
  # A custom publishing environment is used for us to be able to set up protected secrets
  # specifically for it
  environment: publish-crates
  script:
    - rusty-cachier snapshot create
    - git clone
      --depth 1
      --branch "$RELENG_SCRIPTS_BRANCH"
      https://github.com/paritytech/releng-scripts.git
    - CRATESIO_TARGET_INSTANCE=default ./releng-scripts/publish-crates
    - rusty-cachier cache upload

publish-crates:
  extends: .publish-crates-template
  # publish-crates should only be run if publish-crates-locally passes
  needs:
    - job: check-crate-publishing
      artifacts: false

publish-crates-manual:
  extends: .publish-crates-template
  when: manual
  interruptible: false

check-crate-publishing:
  stage: publish
  extends:
    - .crates-publishing-template
    - .crates-publishing-pipeline
  # When lots of crates are taken into account (for example on master where all crates are tested)
  # the job might take a long time, as evidenced by:
  # https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2269364
  timeout: 4h
  script:
    - rusty-cachier snapshot create
    - git clone
      --depth 1
      --branch "$RELENG_SCRIPTS_BRANCH"
      https://github.com/paritytech/releng-scripts.git
    - CRATESIO_TARGET_INSTANCE=local ./releng-scripts/publish-crates
    - rusty-cachier cache upload