diff --git a/substrate/simnet_tests/README.md b/substrate/simnet_tests/README.md new file mode 100644 index 0000000000..cb1b13ae98 --- /dev/null +++ b/substrate/simnet_tests/README.md @@ -0,0 +1,39 @@ +# Simulation tests, or high level integration tests. + + +_The content of this directory is meant to be used by Parity's private CI/CD +infrastructure with private tools. At the moment those tools are still early +stage of development and we don't when if / when they will available for +public use._ + + +## Content of this dir. + +`configs` dir contains config files in toml format that describe how to +configure the simulation network that you want to launch. + +`tests` dir contains [cucumber](https://cucumber.io/) files. Those are +Behavior-Driven Development test files that describe tests in plain English. +Under the hood there are assertions that specific metrics should have specific +values. + +At the moment we have 2 tests: `tests/quick/001-smoketest.feature` and +`tests/long/002-loadtest.feature` +The load test uses a JS script that we added to simnet image and it's launched +by this step in the cucumber file: +`Then launch 'node' with parameters '/usr/local/bin/sub-flood --finalization --url ws://localhost:11222'` + +`run_test.sh` is a script meant to ease up launching a test. +In order to use this script locally, you need to install +[gurke](https://github.com/paritytech/gurke) +This script also helps preparing the test environment. Once you have access to +a kubernetes cluster (meaning you can do `kubectl get pods`) you can run this +script with no arguments, like `./run_test.sh` and tests should run. +Kubernetes cluster can be local, spawned with +[kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) +or an instance living in the +[cloud](https://github.com/paritytech/gurke/blob/main/docs/How-to-setup-access-to-gke-k8s-cluster.md) + + +### [Here is link to barcamp presenation of simnet](https://www.crowdcast.io/e/ph49xu01) +### [Here is link to the simnet repo, hosted on private gitlab](https://gitlab.parity.io/parity/simnet/-/tree/master) diff --git a/substrate/simnet_tests/configs/default_local_testnet.toml b/substrate/simnet_tests/configs/default_local_testnet.toml new file mode 100644 index 0000000000..066bd4c9e3 --- /dev/null +++ b/substrate/simnet_tests/configs/default_local_testnet.toml @@ -0,0 +1,14 @@ +[settings] +bootnode-domain-name = "bootnode.{{get_env(name="NAMESPACE")}}.svc.cluster.local" + + +[settings.setup] +timeout = 300 + +[settings.defaults] +timeout = 300 + +[nodes] +alice = { extra-args = ["--alice"], validator = true } +bob = { extra-args = ["--bob"], validator = true } +charlie = { extra-args = ["--charlie"], validator = true } diff --git a/substrate/simnet_tests/run_tests.sh b/substrate/simnet_tests/run_tests.sh new file mode 100755 index 0000000000..3b8ac8a71d --- /dev/null +++ b/substrate/simnet_tests/run_tests.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +### ARGS FOR THIS SCRIPT ### +# ./${SCRIPT_NAME} NAMESPACE IMAGE LOG_PATH FEATURES +# NAMESPACE the kubernetes namespace where the test will run +# IMAGE Substrate image used to spawn network +# LOG_PATH path to dir where to save logs from external JS script that is run as part +# of step in features file +# FEATURES directory containing cucumber files or single cucumber file that describes +# what to test. +# +# All args have default values, specify args to override +# e.g: ./${SCRIPT_NAME} test-name parity/substrate:latest logs quick + +set -eou pipefail +SCRIPT_NAME="$0" +SCRIPT_PATH=$(dirname "${SCRIPT_NAME}") # relative +SCRIPT_PATH=$(cd "${SCRIPT_PATH}" && pwd) # absolutized and normalized + +function random_string { + head -1 <(fold -w 30 <(tr -dc 'a-z0-9' < /dev/urandom)) + } + +# +### Script args +# + +NAMESPACE=${1:-gurke-"$(random_string)"-runtest} +IMAGE=${2:-"parity/substrate:latest"} +LOG_PATH=${3:-"${SCRIPT_PATH}/logs"} +FEATURES=${4:-"ALL"} + +mkdir -p "${SCRIPT_PATH}"/logs + +echo "Running tests in namespace: ${NAMESPACE}" +echo "Testing image: ${IMAGE}" +echo "Storing scripts logs to: ${LOG_PATH}" +echo "Using features files from: ${FEATURES}" + +# +### Script logic +# + +function forward_port { + # RUN_IN_CONTAINER is env var that is set in the dockerfile + # use the -v operator to explicitly test if a variable is set + if [[ ! -v RUN_IN_CONTAINER ]] ; then + if is_port_forward_running ; then + kill_previous_job + fi + fi + start_forwading_job +} + +FORWARD_GREP_FILTER='kubectl.*[p]ort-forward.*svc/rpc.*11222' + +function is_port_forward_running { + # shellcheck disable=SC2009 + ps aux | grep -qE "${FORWARD_GREP_FILTER}" +} + +function kill_previous_job { + # shellcheck disable=SC2009 + job_pid=$(ps aux | grep -E "${FORWARD_GREP_FILTER}" | awk '{ print $2 }') + echo "INFO Killed forwading port 9944 into bootnode" + kill "${job_pid}" +} + +function start_forwading_job { + kubectl -n "${NAMESPACE}" \ + expose pod bootnode \ + --name=rpc \ + --type=NodePort \ + --target-port=9944 \ + --port=9944 + kubectl -n "${NAMESPACE}" \ + port-forward svc/rpc 11222:9944 &> "${LOG_PATH}/forward-${NAMESPACE}.log" & + sleep 2 + echo "INFO Started forwading port 9944 into bootnode" +} + +function update_api { + echo "INFO: Updating Polkadot JS API" + pwd + cd "${SCRIPT_PATH}"/../../sub-flood/ + npm run build + cd - +} + +function run_test { + case "${FEATURES}" in + quick) + gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests/quick --log-path "${LOG_PATH}" + ;; + long) + gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests/long --log-path "${LOG_PATH}" + ;; + ALL ) + gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests --log-path "${LOG_PATH}" + ;; + ??* ) + gurke test \ + "${NAMESPACE}" \ + "${SCRIPT_PATH}"/"${FEATURES}" \ + --log-path "${LOG_PATH}" + ;; + esac +} + + +export NAMESPACE="${NAMESPACE}" + +set -x # echo the commands to stdout +gurke spawn --config "${SCRIPT_PATH}"/configs/default_local_testnet.toml \ + -n "${NAMESPACE}" \ + --image "${IMAGE}" + +echo "INFO: Checking if pods launched correctly" +kubectl -n "${NAMESPACE}" get pods -o wide + +update_api + +forward_port +run_test + + diff --git a/substrate/simnet_tests/tests/long/002-loadtest.feature b/substrate/simnet_tests/tests/long/002-loadtest.feature new file mode 100644 index 0000000000..67d108ea55 --- /dev/null +++ b/substrate/simnet_tests/tests/long/002-loadtest.feature @@ -0,0 +1,5 @@ +Feature: LoadTesting + + Scenario: spawn 50k transactions and wait their finalization + Given a test network + Then launch 'node' with parameters '/usr/local/bin/sub-flood --finalization --url ws://localhost:11222' diff --git a/substrate/simnet_tests/tests/quick/001-smoketest.feature b/substrate/simnet_tests/tests/quick/001-smoketest.feature new file mode 100644 index 0000000000..a07041e4ea --- /dev/null +++ b/substrate/simnet_tests/tests/quick/001-smoketest.feature @@ -0,0 +1,16 @@ +Feature: Smoketest + + Scenario: Minimal Example + Given a test network + Then alice is up + And alice reports substrate_node_roles is 4 + And alice reports substrate_sub_libp2p_is_major_syncing is 0 + When alice's best block should be above 30 + Then alice reports block height is greater than 30 + And alice reports peers count is at least 2 + Then bob is up + And bob reports block height is greater than 30 + And bob reports peers count is at least 2 + Then charlie is up + And charlie reports block height is greater than 30 + And charlie reports peers count is at least 2