Move scripts used in CI to the new location (#5198)

* Move CI scripts and update references

* Update paths in .gitlab-ci.yml

* Removed outdated entries from CODEOWNERS

polkadot/.github/CODEOWNERS

@@ -1,7 +1,6 @@
-bridges/ @tomusdrw @svyatonik @adoerr @acatangiu @antonio-dropulic
+bridges/ @tomusdrw @svyatonik @acatangiu
 
 # CI
 /.github/ @paritytech/ci @chevdor
-/scripts/github/ @paritytech/ci @chevdor
-/scripts/gitlab/ @paritytech/ci @chevdor
+/scripts/ci/ @paritytech/ci @chevdor
 /.gitlab-ci.yml @paritytech/ci

polkadot/.github/workflows/check-labels.yml

@@ -15,7 +15,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - name: Check labels
-        run: bash ${{ github.workspace }}/scripts/github/check_labels.sh
+        run: bash ${{ github.workspace }}/scripts/ci/github/check_labels.sh
         env:
           GITHUB_PR: ${{ github.event.pull_request.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

polkadot/.github/workflows/extrinsic-ordering-check-from-bin.yml

@@ -62,7 +62,7 @@ jobs:
           $CMD >> output.txt
           sed -z -i 's/\n\n/\n/g' output.txt
           cat output.txt | egrep -n -i ''
-          SUMMARY=$(./scripts/github/extrinsic-ordering-filter.sh output.txt)
+          SUMMARY=$(./scripts/ci/github/extrinsic-ordering-filter.sh output.txt)
           echo -e $SUMMARY
           echo -e $SUMMARY >> output.txt
 

polkadot/.github/workflows/honggfuzz.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Run fuzzer
         working-directory: xcm/xcm-simulator/fuzzer/
-        run: bash $GITHUB_WORKSPACE/scripts/github/run_fuzzer.sh xcm-fuzzer
+        run: bash $GITHUB_WORKSPACE/scripts/ci/github/run_fuzzer.sh xcm-fuzzer
 
   erasure-coding-round-trip:
     runs-on: ubuntu-latest
@@ -88,7 +88,7 @@ jobs:
 
       - name: Run fuzzer
         working-directory: erasure-coding/fuzzer
-        run: bash $GITHUB_WORKSPACE/scripts/github/run_fuzzer.sh round_trip
+        run: bash $GITHUB_WORKSPACE/scripts/ci/github/run_fuzzer.sh round_trip
 
   erasure-coding-reconstruct:
     runs-on: ubuntu-latest
@@ -134,4 +134,4 @@ jobs:
 
       - name: Run fuzzer
         working-directory: erasure-coding/fuzzer
-        run: bash $GITHUB_WORKSPACE/scripts/github/run_fuzzer.sh reconstruct
+        run: bash $GITHUB_WORKSPACE/scripts/ci/github/run_fuzzer.sh reconstruct

polkadot/.github/workflows/publish-docker-manual.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          file: scripts/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.inputs.version }}

polkadot/.github/workflows/publish-docker-release.yml

@@ -30,7 +30,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          file: scripts/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.release.tag_name }}

polkadot/.github/workflows/publish-draft-release.yml

@@ -86,7 +86,7 @@ jobs:
 
       - name: Prepare tooling
         run: |
-          cd polkadot/scripts/changelog
+          cd polkadot/scripts/ci/changelog
           gem install bundler changelogerator:0.9.1
           bundle install
           changelogerator --help
@@ -115,7 +115,7 @@ jobs:
           ls -al $KUSAMA_DIGEST
           ls -al $POLKADOT_DIGEST
 
-          cd polkadot/scripts/changelog
+          cd polkadot/scripts/ci/changelog
 
           ./bin/changelog ${GITHUB_REF}
           ls -al release-notes.md
@@ -126,7 +126,7 @@ jobs:
         with:
           name: release-notes-context
           path: |
-            polkadot/scripts/changelog/context.json
+            polkadot/scripts/ci/changelog/context.json
             **/*_srtool_output.json
 
       - name: Create draft release
@@ -137,7 +137,7 @@ jobs:
         with:
           tag_name: ${{ github.ref }}
           release_name: Polkadot ${{ github.ref }}
-          body_path: ./polkadot/scripts/changelog/release-notes.md
+          body_path: ./polkadot/scripts/ci/changelog/release-notes.md
           draft: true
 
   publish-runtimes:
@@ -160,7 +160,7 @@ jobs:
       - name: Get runtime version
         id: get-runtime-ver
         run: |
-          echo "require './scripts/github/lib.rb'" > script.rb
+          echo "require './scripts/ci/github/lib.rb'" > script.rb
           echo "puts get_runtime(runtime: \"${{ matrix.runtime }}\", runtime_dir: \"$RUNTIME_DIR\")" >> script.rb
 
           echo "Current folder: $PWD"

polkadot/.gitlab-ci.yml

@@ -204,7 +204,7 @@ check-runtime:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
     GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
   script:
-    - ./scripts/gitlab/check_runtime.sh
+    - ./scripts/ci/gitlab/check_runtime.sh
   allow_failure:                   true
 
 cargo-fmt:
@@ -273,7 +273,7 @@ spellcheck:
     - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
     - echo "___Spellcheck is going to check your diff___"
     - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
-    - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
+    - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1
         $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
   allow_failure:                   true
 
@@ -295,7 +295,7 @@ build-test-collators:
     - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
     - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
     - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
-    - cp -r scripts/* ./artifacts
+    - cp -r ./scripts/* ./artifacts
 
 build-malus:
   stage:                           stage1
@@ -312,7 +312,7 @@ build-malus:
     - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
     - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
     - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
-    - cp -r scripts/* ./artifacts
+    - cp -r ./scripts/* ./artifacts
 
 #### stage:                        stage2
 
@@ -371,7 +371,7 @@ test-deterministic-wasm:
   <<:                              *docker-env
   <<:                              *compiler-info
   script:
-    - ./scripts/gitlab/test_deterministic_wasm.sh
+    - ./scripts/ci/gitlab/test_deterministic_wasm.sh
 
 check-transaction-versions:
   stage:                           stage2
@@ -388,7 +388,7 @@ check-transaction-versions:
     - git config remote.origin.url "https://github.com/paritytech/polkadot.git"
     - git fetch origin release
   script:
-    - scripts/gitlab/check_extrinsics_ordering.sh
+    - ./scripts/ci/gitlab/check_extrinsics_ordering.sh
 
 
 
@@ -409,8 +409,8 @@ publish-polkadot-debug-image:
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   variables:
     <<:                            *image-variables
-    # scripts/dockerfiles/polkadot_injected_debug.Dockerfile
-    DOCKERFILE:                    dockerfiles/polkadot_injected_debug.Dockerfile
+    # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/polkadot_injected_debug.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/polkadot-debug
   needs:
     - job:                         build-linux-stable
@@ -433,8 +433,8 @@ publish-test-collators-image:
   <<:                              *zombienet-refs
   variables:
     <<:                            *image-variables
-    # scripts/dockerfiles/collator_injected.Dockerfile
-    DOCKERFILE:                    dockerfiles/collator_injected.Dockerfile
+    # scripts/ci/dockerfiles/collator_injected.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/collator_injected.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/colander
   needs:
     - job:                         build-test-collators
@@ -456,8 +456,8 @@ publish-malus-image:
   <<:                              *zombienet-refs
   variables:
     <<:                            *image-variables
-    # scripts/dockerfiles/malus_injected.Dockerfile
-    DOCKERFILE:                    dockerfiles/malus_injected.Dockerfile
+    # scripts/ci/dockerfiles/malus_injected.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/malus_injected.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/malus
   needs:
     - job:                         build-malus
@@ -522,7 +522,7 @@ update_polkadot_weights:           &update-weights
     paths:
       - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
   script:
-    - ./scripts/run_benches_for_runtime.sh $RUNTIME
+    - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME
     - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
   # uses the "shell" executors
   tags:
@@ -583,7 +583,7 @@ generate-impl-guide:
     name: michaelfbryan/mdbook-docker-image:v0.4.4
     entrypoint: [""]
   script:
-    - mdbook build roadmap/implementers-guide
+    - mdbook build ./roadmap/implementers-guide
 
 check-try-runtime:
   stage:                           stage3
@@ -610,8 +610,8 @@ check-no-default-features:
   <<:                              *compiler-info
   script:
     # Check that polkadot-cli will compile no default features.
-    - pushd node/service && cargo check --no-default-features && popd
-    - pushd cli && cargo check --no-default-features --features "service" && popd
+    - pushd ./node/service && cargo check --no-default-features && popd
+    - pushd ./cli && cargo check --no-default-features --features "service" && popd
     - sccache -s
 
 build-short-benchmark:
@@ -793,7 +793,7 @@ publish-rustdoc:
     - git fetch origin gh-pages
     # Save README and docs
     - cp -r ./crate-docs/ /tmp/doc/
-    - cp README.md /tmp/doc/
+    - cp ./README.md /tmp/doc/
     - git checkout gh-pages
     # Remove everything and restore generated docs and README
     - rm -rf ./*

polkadot/Cargo.toml

@@ -235,5 +235,4 @@ polkadot = { path = "/usr/bin/polkadot" }
 "../scripts/packaging/polkadot.service" = { path = "/usr/lib/systemd/system/polkadot.service", mode = "644" }
 
 [package.metadata.spellcheck]
-config = "./scripts/gitlab/spellcheck.toml"
-
+config = "./scripts/ci/gitlab/spellcheck.toml"

polkadot/doc/docker.md

@@ -118,7 +118,7 @@ To get up and running with the smallest footprint on your system, you may use th
 You can build it yourself (it takes a while...) in the shell session of the daemon:
 
 ```bash
-cd scripts/docker/polkadot
+cd scripts/ci/dockerfiles/polkadot
 ./build.sh
 ```
 

polkadot/scripts/ci/changelog/README.md

@@ -7,7 +7,7 @@ For now, a bit of preparation is required before you can run the script:
 - store them under the `digests` folder as `<chain>-srtool-digest.json`
 - ensure the `.env` file is up to date with correct information. See below for an example
 
-The content of the release notes is generated from the template files under the `scripts/changelog/templates` folder. For readability and maintenance, the template is split into several small snippets.
+The content of the release notes is generated from the template files under the `scripts/ci/changelog/templates` folder. For readability and maintenance, the template is split into several small snippets.
 
 Run:
 ```

polkadot/scripts/ci/dockerfiles/collator_injected.Dockerfile

@@ -1,4 +1,4 @@
-# this file copies from scripts/docker/Dockerfile and changes only the binary name
+# this file copies from scripts/ci/dockerfiles/Dockerfile and changes only the binary name
 FROM docker.io/library/ubuntu:20.04
 
 # metadata
@@ -10,7 +10,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Injected adder-collator Docker image" \
-	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/collator_injected.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/collator_injected.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/polkadot/"

polkadot/scripts/ci/dockerfiles/malus_injected.Dockerfile

@@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Malus - the nemesis of polkadot" \
-	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/dockerfiles/malus.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/malus.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/polkadot/"

polkadot/scripts/ci/dockerfiles/polkadot/build.sh

@@ -15,7 +15,7 @@ GITREPO=polkadot
 # Build the image
 echo "Building ${GITUSER}/${GITREPO}:latest docker image, hang on!"
 time docker build \
-    -f ./scripts/dockerfiles/polkadot/polkadot_builder.Dockerfile \
+    -f ./scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile \
     -t ${GITUSER}/${GITREPO}:latest \
     -t ${GITUSER}/${GITREPO}:v${VERSION} \
     .

polkadot/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile

@@ -14,7 +14,7 @@ LABEL description="Multistage Docker image for Polkadot: a platform for web3" \
 	io.parity.image.authors="chevdor@gmail.com, devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.description="Polkadot: a platform for web3" \
-	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/dockerfiles/polkadot/polkadot_builder.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile" \
 	io.parity.image.documentation="https://github.com/paritytech/polkadot/"
 
 COPY --from=builder /polkadot/target/release/polkadot /usr/local/bin

polkadot/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile

@@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Polkadot: a platform for web3" \
-	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/polkadot_injected_debug.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/polkadot/"

polkadot/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile

@@ -11,7 +11,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="parity/polkadot" \
 	io.parity.image.description="Polkadot: a platform for web3. This is the official Parity image with an injected binary." \
-	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/dockerfiles/polkadot_injected_release.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/polkadot/"

polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile

@@ -20,7 +20,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="${IMAGE_NAME} for substrate based chains" \
-	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/${IMAGE_NAME}/${IMAGE_NAME}_builder.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_builder.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/polkadot/"

polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile

@@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="${IMAGE_NAME} for substrate based chains" \
-	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/${IMAGE_NAME}/${IMAGE_NAME}_injected.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_injected.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/polkadot/"

polkadot/scripts/ci/github/generate_release_text.rb

@@ -45,7 +45,7 @@ polkadot_path = ENV['GITHUB_WORKSPACE'] + '/polkadot/'
 
 # Generate an ERB renderer based on the template .erb file
 renderer = ERB.new(
-  File.read(File.join(polkadot_path, 'scripts/github/polkadot_release.erb')),
+  File.read(File.join(polkadot_path, 'scripts/ci/github/polkadot_release.erb')),
   trim_mode: '<>'
 )
 

polkadot/utils/staking-miner/README.md

@@ -30,7 +30,7 @@ There are 2 options to build a staking-miner Docker image:
 First build the binary as documented [above](#building).
 You may then inject the binary into a Docker base image usingfrom the root of the Polkadot repository:
 ```
-docker build -t staking-miner -f scripts/docker/staking-miner/staking-miner_injected.Dockerfile target/release
+docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile target/release
 ```
 
 ### Building the multi-stage image
@@ -39,7 +39,7 @@ Unlike the injected image that requires a Linux pre-built binary, this option do
 The trade-off however is that it takes a little longer to build and this option is less ideal for CI tasks.
 You may build the multi-stage image the root of the Polkadot repository with:
 ```
-docker build -t staking-miner -f scripts/docker/staking-miner/staking-miner_builder.Dockerfile .
+docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile .
 ```
 
 ### Running