From 04c5e3aa5923b6056e8583e7553f6171279ba9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 9 Dec 2021 20:43:46 +0100 Subject: [PATCH] Remove simnet from CI (#10456) --- substrate/.gitlab-ci.yml | 97 -------------- substrate/simnet_tests/README.md | 39 ------ .../configs/default_local_testnet.toml | 14 -- substrate/simnet_tests/run_tests.sh | 126 ------------------ .../tests/long/002-loadtest.feature | 5 - .../tests/quick/001-smoketest.feature | 16 --- 6 files changed, 297 deletions(-) delete mode 100644 substrate/simnet_tests/README.md delete mode 100644 substrate/simnet_tests/configs/default_local_testnet.toml delete mode 100755 substrate/simnet_tests/run_tests.sh delete mode 100644 substrate/simnet_tests/tests/long/002-loadtest.feature delete mode 100644 substrate/simnet_tests/tests/quick/001-smoketest.feature diff --git a/substrate/.gitlab-ci.yml b/substrate/.gitlab-ci.yml index f4391cf7d3..7af3675f07 100644 --- a/substrate/.gitlab-ci.yml +++ b/substrate/.gitlab-ci.yml @@ -45,7 +45,6 @@ variables: &default-vars VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io" VAULT_AUTH_PATH: "gitlab-parity-io-jwt" VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}" - SIMNET_FEATURES_PATH: "simnet_tests/tests" PIPELINE_SCRIPTS_TAG: "v0.4" default: @@ -641,18 +640,6 @@ build-linux-substrate: - printf '\n# building node-template\n\n' - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - -#Build binary for simnet quick tests. -build-linux-substrate-simnet: - stage: build - <<: *collect-artifacts-short - <<: *docker-env - <<: *test-refs-no-trigger-prs-only - before_script: - - mkdir -p ./artifacts/substrate/ - script: - - *build-linux-substrate-script - build-linux-subkey: &build-subkey stage: build <<: *collect-artifacts @@ -739,7 +726,6 @@ build-rustdoc: - buildah push --format=v2s2 "$IMAGE_NAME:latest" after_script: - buildah logout --all - # pass artifacts to the trigger-simnet job - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)" - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env @@ -755,30 +741,6 @@ publish-docker-substrate: variables: <<: *docker-build-vars PRODUCT: substrate - artifacts: - reports: - # this artifact is used in trigger-simnet job - # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - dotenv: ./artifacts/substrate/build.env - -#publish artifact for quick simnet tests -publish-docker-substrate-simnet: - stage: publish - <<: *build-push-docker-image - <<: *test-refs-no-trigger-prs-only - needs: - - job: build-linux-substrate-simnet - artifacts: true - variables: - <<: *default-vars - GIT_STRATEGY: none - DOCKERFILE: $PRODUCT.Dockerfile - PRODUCT: substrate - IMAGE_NAME: docker.io/paritypr/$PRODUCT - artifacts: - reports: - # this artifact is used in simnet-tests-quick job - dotenv: ./artifacts/substrate/build.env publish-docker-subkey: stage: publish @@ -940,62 +902,3 @@ deploy-prometheus-alerting-rules: changes: - .gitlab-ci.yml - .maintain/monitoring/**/* - -# Runs "quick" and "long" tests on nightly schedule and on commit / merge to master -# A "quick" test is a smoke test where basic check-expect tests run by -# checking values from metrics exposed by the app. -# A "long" test is the load testing where we send 50K transactions into the -# network and check if all completed successfully -simnet-tests: - stage: deploy - image: docker.io/paritytech/simnet:${SIMNET_REF} - <<: *kubernetes-env - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - needs: - - job: publish-docker-substrate - # variables: - # `build.env` brings here `${SUBSTRATE_IMAGE_NAME}` and `${SUBSTRATE_IMAGE_TAG}` - # (`$VERSION` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`). - # ${SIMNET_REF} is a gitlab variable - before_script: - - echo "Simnet Tests Config - docker.io/paritytech/simnet:${SIMNET_REF} - ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}" - script: - - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh - --github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests" - --config="simnet_tests/configs/default_local_testnet.toml" - --image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}" - retry: 2 - tags: - - substrate-simnet - -#run quick simnet-tests for each PR -simnet-tests-quick: - stage: deploy - image: docker.io/paritytech/simnet:${SIMNET_REF} - <<: *kubernetes-env - <<: *test-refs-no-trigger-prs-only - variables: - SIMNET_FEATURES: "${SIMNET_FEATURES_PATH}/quick" - needs: - - job: publish-docker-substrate-simnet - before_script: - - echo "Simnet Tests Config - docker.io/paritytech/simnet:${SIMNET_REF} - ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}" - script: - - echo "Image=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}" - - echo "Features=${SIMNET_FEATURES}" - - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh - --github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests" - --config="simnet_tests/configs/default_local_testnet.toml" - --image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}" - --features="${SIMNET_FEATURES}" - tags: - - substrate-simnet - diff --git a/substrate/simnet_tests/README.md b/substrate/simnet_tests/README.md deleted file mode 100644 index cb1b13ae98..0000000000 --- a/substrate/simnet_tests/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Simulation tests, or high level integration tests. - - -_The content of this directory is meant to be used by Parity's private CI/CD -infrastructure with private tools. At the moment those tools are still early -stage of development and we don't when if / when they will available for -public use._ - - -## Content of this dir. - -`configs` dir contains config files in toml format that describe how to -configure the simulation network that you want to launch. - -`tests` dir contains [cucumber](https://cucumber.io/) files. Those are -Behavior-Driven Development test files that describe tests in plain English. -Under the hood there are assertions that specific metrics should have specific -values. - -At the moment we have 2 tests: `tests/quick/001-smoketest.feature` and -`tests/long/002-loadtest.feature` -The load test uses a JS script that we added to simnet image and it's launched -by this step in the cucumber file: -`Then launch 'node' with parameters '/usr/local/bin/sub-flood --finalization --url ws://localhost:11222'` - -`run_test.sh` is a script meant to ease up launching a test. -In order to use this script locally, you need to install -[gurke](https://github.com/paritytech/gurke) -This script also helps preparing the test environment. Once you have access to -a kubernetes cluster (meaning you can do `kubectl get pods`) you can run this -script with no arguments, like `./run_test.sh` and tests should run. -Kubernetes cluster can be local, spawned with -[kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) -or an instance living in the -[cloud](https://github.com/paritytech/gurke/blob/main/docs/How-to-setup-access-to-gke-k8s-cluster.md) - - -### [Here is link to barcamp presenation of simnet](https://www.crowdcast.io/e/ph49xu01) -### [Here is link to the simnet repo, hosted on private gitlab](https://gitlab.parity.io/parity/simnet/-/tree/master) diff --git a/substrate/simnet_tests/configs/default_local_testnet.toml b/substrate/simnet_tests/configs/default_local_testnet.toml deleted file mode 100644 index 066bd4c9e3..0000000000 --- a/substrate/simnet_tests/configs/default_local_testnet.toml +++ /dev/null @@ -1,14 +0,0 @@ -[settings] -bootnode-domain-name = "bootnode.{{get_env(name="NAMESPACE")}}.svc.cluster.local" - - -[settings.setup] -timeout = 300 - -[settings.defaults] -timeout = 300 - -[nodes] -alice = { extra-args = ["--alice"], validator = true } -bob = { extra-args = ["--bob"], validator = true } -charlie = { extra-args = ["--charlie"], validator = true } diff --git a/substrate/simnet_tests/run_tests.sh b/substrate/simnet_tests/run_tests.sh deleted file mode 100755 index 3b8ac8a71d..0000000000 --- a/substrate/simnet_tests/run_tests.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -### ARGS FOR THIS SCRIPT ### -# ./${SCRIPT_NAME} NAMESPACE IMAGE LOG_PATH FEATURES -# NAMESPACE the kubernetes namespace where the test will run -# IMAGE Substrate image used to spawn network -# LOG_PATH path to dir where to save logs from external JS script that is run as part -# of step in features file -# FEATURES directory containing cucumber files or single cucumber file that describes -# what to test. -# -# All args have default values, specify args to override -# e.g: ./${SCRIPT_NAME} test-name parity/substrate:latest logs quick - -set -eou pipefail -SCRIPT_NAME="$0" -SCRIPT_PATH=$(dirname "${SCRIPT_NAME}") # relative -SCRIPT_PATH=$(cd "${SCRIPT_PATH}" && pwd) # absolutized and normalized - -function random_string { - head -1 <(fold -w 30 <(tr -dc 'a-z0-9' < /dev/urandom)) - } - -# -### Script args -# - -NAMESPACE=${1:-gurke-"$(random_string)"-runtest} -IMAGE=${2:-"parity/substrate:latest"} -LOG_PATH=${3:-"${SCRIPT_PATH}/logs"} -FEATURES=${4:-"ALL"} - -mkdir -p "${SCRIPT_PATH}"/logs - -echo "Running tests in namespace: ${NAMESPACE}" -echo "Testing image: ${IMAGE}" -echo "Storing scripts logs to: ${LOG_PATH}" -echo "Using features files from: ${FEATURES}" - -# -### Script logic -# - -function forward_port { - # RUN_IN_CONTAINER is env var that is set in the dockerfile - # use the -v operator to explicitly test if a variable is set - if [[ ! -v RUN_IN_CONTAINER ]] ; then - if is_port_forward_running ; then - kill_previous_job - fi - fi - start_forwading_job -} - -FORWARD_GREP_FILTER='kubectl.*[p]ort-forward.*svc/rpc.*11222' - -function is_port_forward_running { - # shellcheck disable=SC2009 - ps aux | grep -qE "${FORWARD_GREP_FILTER}" -} - -function kill_previous_job { - # shellcheck disable=SC2009 - job_pid=$(ps aux | grep -E "${FORWARD_GREP_FILTER}" | awk '{ print $2 }') - echo "INFO Killed forwading port 9944 into bootnode" - kill "${job_pid}" -} - -function start_forwading_job { - kubectl -n "${NAMESPACE}" \ - expose pod bootnode \ - --name=rpc \ - --type=NodePort \ - --target-port=9944 \ - --port=9944 - kubectl -n "${NAMESPACE}" \ - port-forward svc/rpc 11222:9944 &> "${LOG_PATH}/forward-${NAMESPACE}.log" & - sleep 2 - echo "INFO Started forwading port 9944 into bootnode" -} - -function update_api { - echo "INFO: Updating Polkadot JS API" - pwd - cd "${SCRIPT_PATH}"/../../sub-flood/ - npm run build - cd - -} - -function run_test { - case "${FEATURES}" in - quick) - gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests/quick --log-path "${LOG_PATH}" - ;; - long) - gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests/long --log-path "${LOG_PATH}" - ;; - ALL ) - gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests --log-path "${LOG_PATH}" - ;; - ??* ) - gurke test \ - "${NAMESPACE}" \ - "${SCRIPT_PATH}"/"${FEATURES}" \ - --log-path "${LOG_PATH}" - ;; - esac -} - - -export NAMESPACE="${NAMESPACE}" - -set -x # echo the commands to stdout -gurke spawn --config "${SCRIPT_PATH}"/configs/default_local_testnet.toml \ - -n "${NAMESPACE}" \ - --image "${IMAGE}" - -echo "INFO: Checking if pods launched correctly" -kubectl -n "${NAMESPACE}" get pods -o wide - -update_api - -forward_port -run_test - - diff --git a/substrate/simnet_tests/tests/long/002-loadtest.feature b/substrate/simnet_tests/tests/long/002-loadtest.feature deleted file mode 100644 index 67d108ea55..0000000000 --- a/substrate/simnet_tests/tests/long/002-loadtest.feature +++ /dev/null @@ -1,5 +0,0 @@ -Feature: LoadTesting - - Scenario: spawn 50k transactions and wait their finalization - Given a test network - Then launch 'node' with parameters '/usr/local/bin/sub-flood --finalization --url ws://localhost:11222' diff --git a/substrate/simnet_tests/tests/quick/001-smoketest.feature b/substrate/simnet_tests/tests/quick/001-smoketest.feature deleted file mode 100644 index a07041e4ea..0000000000 --- a/substrate/simnet_tests/tests/quick/001-smoketest.feature +++ /dev/null @@ -1,16 +0,0 @@ -Feature: Smoketest - - Scenario: Minimal Example - Given a test network - Then alice is up - And alice reports substrate_node_roles is 4 - And alice reports substrate_sub_libp2p_is_major_syncing is 0 - When alice's best block should be above 30 - Then alice reports block height is greater than 30 - And alice reports peers count is at least 2 - Then bob is up - And bob reports block height is greater than 30 - And bob reports peers count is at least 2 - Then charlie is up - And charlie reports block height is greater than 30 - And charlie reports peers count is at least 2